From 3424c40f333021416d1db7b735258fa8ce1e06bf Mon Sep 17 00:00:00 2001 From: gabewillen Date: Thu, 23 Jul 2026 22:25:09 -0500 Subject: [PATCH 1/6] feat: make PersonaPlex CPU speech generation realtime --- CMakeLists.txt | 6 +- README.md | 4 +- coroutine-plan.md | 2 +- docs/benchmarking.md | 19 +- docs/determinism.md | 62 +- docs/templates/README.md.j2 | 10 +- goal.md | 2 +- scripts/bench.sh | 14 + scripts/bench_diarization_compare.sh | 5 + scripts/bench_embedding_compare.sh | 5 + scripts/bench_embedding_reference_liquid.sh | 5 + scripts/bench_generation_compare.sh | 5 + .../bench_generation_reference_llama_cpp.sh | 5 + scripts/bench_mimi_compare.sh | 20 +- scripts/bench_moshi_lm_compare.sh | 3 + scripts/bench_personaplex_compare.sh | 70 +- scripts/bench_whisper_compare.sh | 3 + scripts/bench_whisper_single_thread.sh | 3 + scripts/build_with_zig.sh | 14 + scripts/check_determinism.sh | 3 + scripts/check_macos_binary_target.sh | 43 + scripts/embedded_size.sh | 5 + scripts/generate_docs.sh | 7 +- scripts/paritychecker.sh | 8 +- scripts/setup_moshi_cpp_reference.sh | 183 +- scripts/setup_personaplex_mlx_reference.sh | 32 +- scripts/setup_whisper_cpp_reference.sh | 3 + scripts/test_with_coverage.sh | 22 +- scripts/zig_toolchain.sh | 72 + snapshots/lint/clang_format.txt | 10 +- snapshots/quality_gates/timing.txt | 14 +- src/emel/kernel/aarch64/actions.hpp | 8 +- src/emel/kernel/any.hpp | 56 +- src/emel/kernel/attention/actions.hpp | 125 ++ src/emel/kernel/attention/any.hpp | 3 + src/emel/kernel/attention/context.hpp | 21 + src/emel/kernel/attention/events.hpp | 60 + src/emel/kernel/attention/guards.hpp | 143 ++ src/emel/kernel/attention/sm.hpp | 125 ++ src/emel/kernel/events.hpp | 52 + src/emel/kernel/f32_matvec/actions.hpp | 186 ++ src/emel/kernel/f32_matvec/any.hpp | 3 + src/emel/kernel/f32_matvec/context.hpp | 14 + src/emel/kernel/f32_matvec/events.hpp | 114 ++ src/emel/kernel/f32_matvec/guards.hpp | 246 +++ src/emel/kernel/f32_matvec/sm.hpp | 234 +++ src/emel/kernel/matmul/actions.hpp | 305 +++ src/emel/kernel/matmul/context.hpp | 55 + src/emel/kernel/matmul/detail.hpp | 58 + src/emel/kernel/matmul/events.hpp | 79 + src/emel/kernel/matmul/guards.hpp | 281 +++ src/emel/kernel/matmul/sm.hpp | 243 +++ src/emel/machines.hpp | 6 + src/emel/sm.hpp | 88 +- src/emel/speech/codec/mimi/actions.hpp | 53 +- src/emel/speech/codec/mimi/any.hpp | 11 +- src/emel/speech/codec/mimi/detail.cpp | 444 ++++- src/emel/speech/codec/mimi/detail.hpp | 32 + src/emel/speech/codec/mimi/events.hpp | 87 +- src/emel/speech/codec/mimi/guards.hpp | 320 ++- src/emel/speech/codec/mimi/sm.hpp | 75 +- src/emel/speech/generator/actions.hpp | 480 +++++ src/emel/speech/generator/any.hpp | 1 + src/emel/speech/generator/context.hpp | 107 +- src/emel/speech/generator/errors.hpp | 5 + src/emel/speech/generator/events.hpp | 140 ++ src/emel/speech/generator/frame/actions.hpp | 205 ++ src/emel/speech/generator/frame/any.hpp | 10 + src/emel/speech/generator/frame/context.hpp | 62 + src/emel/speech/generator/frame/errors.hpp | 20 + src/emel/speech/generator/frame/events.hpp | 59 + src/emel/speech/generator/frame/guards.hpp | 119 ++ src/emel/speech/generator/frame/sm.hpp | 216 +++ src/emel/speech/generator/guards.hpp | 326 ++++ src/emel/speech/generator/sm.hpp | 1064 +++++++++- .../predictor/moshi/executor/actions.hpp | 350 +++- .../predictor/moshi/executor/context.hpp | 35 +- .../predictor/moshi/executor/guards.hpp | 113 ++ .../speech/predictor/moshi/executor/sm.hpp | 108 +- src/emel/text/generator/actions.hpp | 86 +- src/emel/text/generator/context.hpp | 4 +- src/emel/text/generator/detail.hpp | 72 +- src/emel/text/generator/guards.hpp | 3 +- .../text/generator/initializer/actions.hpp | 3 +- src/emel/text/generator/layer/actions.hpp | 38 +- src/emel/text/generator/layer/sm.hpp | 36 +- src/emel/text/generator/matmul/actions.hpp | 153 -- src/emel/text/generator/matmul/context.hpp | 83 - src/emel/text/generator/matmul/detail.hpp | 201 -- src/emel/text/generator/matmul/events.hpp | 45 - src/emel/text/generator/matmul/guards.hpp | 116 -- src/emel/text/generator/matmul/sm.hpp | 156 -- src/emel/text/generator/prefill/guards.hpp | 3 +- src/emel/text/generator/sm.hpp | 9 +- tests/kernel/aarch64_tests.cpp | 143 +- tests/kernel/attention_tests.cpp | 360 ++++ tests/kernel/f32_matvec_tests.cpp | 384 ++++ tests/kernel/matmul_tests.cpp | 779 ++++++++ tests/sm/sm_policy_tests.cpp | 122 ++ tests/speech/codec/mimi_lifecycle_tests.cpp | 218 ++- .../generator/frame_lifecycle_tests.cpp | 696 +++++++ .../generator/wavefront_lifecycle_tests.cpp | 1376 +++++++++++++ .../moshi_executor_streaming_tests.cpp | 8 + .../predictor/moshi_lifecycle_tests.cpp | 415 +++- tests/text/generator/action_guard_tests.cpp | 11 +- tests/text/generator/detail_tests.cpp | 34 +- tests/text/generator/determinism_tests.cpp | 17 +- tests/text/generator/lifecycle_tests.cpp | 1723 ++++++++++------- .../text/generator/parallel_matmul_tests.cpp | 480 ----- tests/text/generator/stream_window_tests.cpp | 6 +- tools/bench/CMakeLists.txt | 4 +- .../moshi_gguf_convert.cpython-314.pyc | Bin 100099 -> 0 bytes .../moshi_make_tiny_fixture.cpython-314.pyc | Bin 25632 -> 0 bytes tools/bench/bench_dependency_manifest.cpp | 4 +- tools/bench/dependency_manifest.txt | 2 +- tools/bench/generation_bench.cpp | 4 +- tools/bench/kernel/aarch64_bench.cpp | 292 ++- tools/bench/kernel/f32_matvec_bench.cpp | 155 ++ .../model/tensor/window_streaming_bench.cpp | 4 +- tools/bench/personaplex_compare.py | 151 +- tools/bench/personaplex_compare_tests.cpp | 177 +- tools/bench/speech/codec_mimi_bench.cpp | 23 +- tools/bench/speech/lm_moshi_bench.cpp | 336 ++++ .../bench/speech/mimi_emel_parity_runner.cpp | 15 +- .../bench/speech/personaplex_emel_runner.cpp | 539 +++++- .../text/generator/parallel_matmul_bench.cpp | 255 ++- tools/decode_wavefront_eval/main.cpp | 4 +- tools/determinism_check/main.cpp | 4 +- tools/docsgen/docsgen.cpp | 4 + tools/embedded_size/emel_probe/main.cpp | 4 +- tools/paritychecker/parity_engines.cpp | 4 +- 131 files changed, 14526 insertions(+), 2841 deletions(-) create mode 100755 scripts/check_macos_binary_target.sh create mode 100644 scripts/zig_toolchain.sh create mode 100644 src/emel/kernel/attention/actions.hpp create mode 100644 src/emel/kernel/attention/any.hpp create mode 100644 src/emel/kernel/attention/context.hpp create mode 100644 src/emel/kernel/attention/events.hpp create mode 100644 src/emel/kernel/attention/guards.hpp create mode 100644 src/emel/kernel/attention/sm.hpp create mode 100644 src/emel/kernel/f32_matvec/actions.hpp create mode 100644 src/emel/kernel/f32_matvec/any.hpp create mode 100644 src/emel/kernel/f32_matvec/context.hpp create mode 100644 src/emel/kernel/f32_matvec/events.hpp create mode 100644 src/emel/kernel/f32_matvec/guards.hpp create mode 100644 src/emel/kernel/f32_matvec/sm.hpp create mode 100644 src/emel/kernel/matmul/actions.hpp create mode 100644 src/emel/kernel/matmul/context.hpp create mode 100644 src/emel/kernel/matmul/detail.hpp create mode 100644 src/emel/kernel/matmul/events.hpp create mode 100644 src/emel/kernel/matmul/guards.hpp create mode 100644 src/emel/kernel/matmul/sm.hpp create mode 100644 src/emel/speech/generator/frame/actions.hpp create mode 100644 src/emel/speech/generator/frame/any.hpp create mode 100644 src/emel/speech/generator/frame/context.hpp create mode 100644 src/emel/speech/generator/frame/errors.hpp create mode 100644 src/emel/speech/generator/frame/events.hpp create mode 100644 src/emel/speech/generator/frame/guards.hpp create mode 100644 src/emel/speech/generator/frame/sm.hpp delete mode 100644 src/emel/text/generator/matmul/actions.hpp delete mode 100644 src/emel/text/generator/matmul/context.hpp delete mode 100644 src/emel/text/generator/matmul/detail.hpp delete mode 100644 src/emel/text/generator/matmul/events.hpp delete mode 100644 src/emel/text/generator/matmul/guards.hpp delete mode 100644 src/emel/text/generator/matmul/sm.hpp create mode 100644 tests/kernel/attention_tests.cpp create mode 100644 tests/kernel/f32_matvec_tests.cpp create mode 100644 tests/kernel/matmul_tests.cpp create mode 100644 tests/speech/generator/frame_lifecycle_tests.cpp create mode 100644 tests/speech/generator/wavefront_lifecycle_tests.cpp delete mode 100644 tests/text/generator/parallel_matmul_tests.cpp delete mode 100644 tools/bench/__pycache__/moshi_gguf_convert.cpython-314.pyc delete mode 100644 tools/bench/__pycache__/moshi_make_tiny_fixture.cpython-314.pyc create mode 100644 tools/bench/kernel/f32_matvec_bench.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 915bf176..d8a12991 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,7 +171,6 @@ if(EMEL_ENABLE_TESTS) tests/text/generator/lifecycle_tests.cpp tests/text/generator/action_guard_tests.cpp tests/text/generator/detail_tests.cpp - tests/text/generator/parallel_matmul_tests.cpp tests/text/generator/determinism_tests.cpp tests/text/generator/stream_window_tests.cpp tests/text/generator/decode_wavefront/lifecycle_tests.cpp @@ -188,6 +187,8 @@ if(EMEL_ENABLE_TESTS) tests/speech/codec/mimi_detail_tests.cpp tests/speech/codec/mimi_lifecycle_tests.cpp tests/speech/generator/lifecycle_tests.cpp + tests/speech/generator/frame_lifecycle_tests.cpp + tests/speech/generator/wavefront_lifecycle_tests.cpp tests/speech/predictor/moshi_lifecycle_tests.cpp tests/speech/predictor/moshi_executor_detail_tests.cpp tests/speech/predictor/moshi_executor_streaming_tests.cpp @@ -218,6 +219,9 @@ if(EMEL_ENABLE_TESTS) tests/graph/processor/processor_action_branch_tests.cpp tests/kernel/aarch64_tests.cpp tests/kernel/lifecycle_tests.cpp + tests/kernel/attention_tests.cpp + tests/kernel/f32_matvec_tests.cpp + tests/kernel/matmul_tests.cpp tests/kernel/x86_64_tests.cpp # Legacy processor tests target pre-cutover API shape and are disabled until migrated. # Keep them out of gate builds during the hard cutover. diff --git a/README.md b/README.md index 86c73621..3d2e5bd3 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,9 @@ environments, while Zig remains the default for day-to-day builds. - [`.planning/architecture/io_read.md`](.planning/architecture/io_read.md) - [`.planning/architecture/io_staged_read.md`](.planning/architecture/io_staged_read.md) - [`.planning/architecture/kernel_aarch64.md`](.planning/architecture/kernel_aarch64.md) +- [`.planning/architecture/kernel_attention.md`](.planning/architecture/kernel_attention.md) +- [`.planning/architecture/kernel_f32_matvec.md`](.planning/architecture/kernel_f32_matvec.md) +- [`.planning/architecture/kernel_matmul.md`](.planning/architecture/kernel_matmul.md) - [`.planning/architecture/kernel_x86_64.md`](.planning/architecture/kernel_x86_64.md) - [`.planning/architecture/logits_sampler.md`](.planning/architecture/logits_sampler.md) - [`.planning/architecture/logits_validator.md`](.planning/architecture/logits_validator.md) @@ -200,7 +203,6 @@ environments, while Zig remains the default for day-to-day builds. - [`.planning/architecture/text_formatter.md`](.planning/architecture/text_formatter.md) - [`.planning/architecture/text_generator_decode_wavefront.md`](.planning/architecture/text_generator_decode_wavefront.md) - [`.planning/architecture/text_generator_initializer.md`](.planning/architecture/text_generator_initializer.md) -- [`.planning/architecture/text_generator_matmul.md`](.planning/architecture/text_generator_matmul.md) - [`.planning/architecture/text_generator_prefill.md`](.planning/architecture/text_generator_prefill.md) - [`.planning/architecture/text_generator.md`](.planning/architecture/text_generator.md) - [`.planning/architecture/text_jinja_formatter.md`](.planning/architecture/text_jinja_formatter.md) diff --git a/coroutine-plan.md b/coroutine-plan.md index 28098fc8..0b0de31f 100644 --- a/coroutine-plan.md +++ b/coroutine-plan.md @@ -527,7 +527,7 @@ matmul dispatch path rather than a standalone component. Correctness: - Row slices write disjoint dst rows and reorder no reductions, so parallel output is - bit-identical to serial; `tests/text/generator/parallel_matmul_tests.cpp` proves slice + bit-identical to serial; `tests/kernel/matmul_tests.cpp` proves slice arithmetic, group alignment, and f32/q8_0 serial-vs-parallel byte equality, and the full generator fixture suites pass with prompts >= 8 tokens taking the parallel prefill route. - The strict LFM2 x86_64 generation evidence failure is unchanged and pre-existing diff --git a/docs/benchmarking.md b/docs/benchmarking.md index d6f58261..31a98506 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -412,14 +412,21 @@ the pinned upstream JSON remains untouched for the reference lane. EMEL model co the separate `tools/bench/personaplex-inference.json` contract, keeping prompt tokens, public codebook count, silence framing, and sampling limits explicit and reproducible. the setup keeps the full 16-codebook Mimi parity artifact separate from PersonaPlex's 8-public-codebook Mimi artifact. +the maintained PersonaPlex artifact is generated with `--quantize q8_0`: only the Mimi transformer +and RVQ projection weight classes are quantized, while convolutions, norms, biases, scales, and +codebooks retain their source float dtype. setup freshness verifies the converter manifest's +quantization, exact 68-tensor contract, and `enriched_sha256` against the actual GGUF bytes, so a +newer legacy F32 artifact cannot silently satisfy the maintained default. the report and both audible WAVs are written under `build/personaplex_compare/`. the report records -the input SHA256, EMEL/reference LM and Mimi paths and SHA256 values, exact public Mimi input-code -match, sampled output-token match, exact text frame/token match, lagged 80 ms energy correlation, -activity ratio, non-silence, output hashes, and end-to-end wall time. A text frame-count or token -drift fails the comparison. sampled audio output is compared as behavior rather than raw-waveform -equality because a one-rank floating-point difference can avalanche later fixed-seed draws even -when the RNG state and sampling policy agree. +the input SHA256, EMEL/reference LM and Mimi paths and SHA256 values, same-WAV public Mimi +input-code match, sampled output-token match, exact text frame/token match, lagged 80 ms energy +correlation, activity ratio, non-silence, output hashes, and end-to-end wall time. The current +maintained same-WAV input-code match is `0.961538`; it is reported separately from generated-output +acceptance and keeps exact semantic parity failed until it reaches `1.0`. A text frame-count or +token drift fails the bounded comparison. sampled audio output is compared as behavior rather than +raw-waveform equality because a one-rank floating-point difference can avalanche later fixed-seed +draws even when the RNG state and sampling policy agree. use `--emel-lm` and `--emel-mimi` (or `EMEL_PERSONAPLEX_EMEL_LM` and `EMEL_PERSONAPLEX_EMEL_MIMI`) to inject packed or quantized EMEL artifacts without changing the diff --git a/docs/determinism.md b/docs/determinism.md index 40abb034..9b084b62 100644 --- a/docs/determinism.md +++ b/docs/determinism.md @@ -99,47 +99,57 @@ statically determined order**: ## 4. Parallel-lane policy and lane-count invariance -The runtime has exactly three fork/join parallel sites, all bounded within a -single RTC dispatch, and none of them reorders a float reduction: - -1. **View-sliced parallel matmul lanes** - (`compute_mul_mat_sliced_parallel` in - `src/emel/text/generator/detail.hpp`; pool type - `emel::policy::thread_pool_scheduler` in `src/emel/sm.hpp`). - One logical `mul_mat` forks into per-lane row-slice events across kernel - actors on a warm thread pool. Slices are contiguous, aligned to the packed - storage group (`matmul_slice_group_rows`: 4 rows for q8_0_x4, 8 rows for - q4_k_x8/q6_k_x8, 1 otherwise), write **disjoint output rows**, and share - the read-only input operand. Because each output element's reduction runs - entirely inside one lane in the kernel's fixed order, slicing changes only - *which thread* computes a row, never *how* it is computed. Results are - therefore bitwise identical to serial dispatch and **bitwise invariant to - the lane count**. - - The lane count is a compile-time constant today - (`k_matmul_lanes = 8`); the effective per-dispatch lane count still varies - as `min(8, row_groups)`, so invariance across slice counts is load-bearing - and is proven directly: +The maintained runtime has five fork/join or overlap surfaces. Every compute +fork joins within the owning RTC dispatch, and none reorders a float reduction: + +1. **Kernel-owned row-sliced matmul** + (`src/emel/kernel/matmul/**`). A logical `mul_mat` is routed by explicit + state-machine guards to serial, 2-, 4-, or 8-lane effects. The owner runs + lane zero while a construction-time-sized `fork_join_lane_pool` runs the + remaining lanes. Slices are contiguous, aligned to the packed row group, + write disjoint output rows, and share read-only operands. Each output + element's reduction stays wholly within one lane in the kernel's fixed + order, so results are bitwise invariant across the supported lane counts. + The pool starts exactly `active_lanes - 1` workers on budgeted runtime paths; + unused capacity does not create runnable or spinning threads. + + Invariance is proven directly: `tests/text/generator/determinism_tests.cpp` sweeps slice counts {1, 2, 3, 5, 8} over f32/q8_0/q4_k/q6_k GEMV and f32 GEMM and asserts bitwise equality with the unsliced dispatch; - `tests/text/generator/parallel_matmul_tests.cpp` asserts serial vs + `tests/kernel/matmul_tests.cpp` asserts serial vs parallel fork/join bit-exactness on the production dispatch helper. -2. **Decode wavefront lanes** +2. **Moshi temporal-attention head lanes** + (`src/emel/speech/predictor/moshi/executor/actions.hpp`). The same budgeted + lane pool dispatches disjoint head ranges to independent attention actors. + Each head writes a disjoint output interval and performs its softmax and + weighted-value reductions locally in fixed order. The 1/2/4/8-lane result + and cache state are compared exactly by + `tests/speech/predictor/moshi_lifecycle_tests.cpp`. + +3. **Speech-generator frame wavefront** + (`src/emel/speech/generator/actions.hpp`). A fixed two-worker stage pool can + overlap encoder and decoder actors with the owner-thread Moshi middle stage. + The actors own disjoint frame lanes, preserve typed attribution, and join + before the action returns. Serial stage mode creates no stage pool. Lifecycle, + overlap, drain, attribution, and deterministic-trace contracts are covered by + `tests/speech/generator/wavefront_lifecycle_tests.cpp`. + +4. **Decode wavefront lanes** (`src/emel/text/generator/decode_wavefront/actions.hpp`). Forks *independent generation sessions* (each with its own graph, KV state, and buffers) across the pool. There is no shared mutable float state between lanes, so each session's stream is unchanged by running next to others (`tools/decode_wavefront_eval` asserts sequential == parallel outputs). -3. **Tensor-window stream I/O** +5. **Tensor-window stream I/O** (`src/emel/model/tensor/window/actions.hpp`). Overlaps weight prefetch I/O with compute; it moves bytes and performs no float arithmetic. **Current invariance status: no known gaps.** No parallel route performs a -cross-lane float reduction; flash and non-flash attention, softmax, and -norm reductions all run serially within a generator step. If a future +cross-lane float reduction; all softmax, norm, dot-product, and weighted-value +reductions remain local to one output element or attention head. If a future parallel route must split a reduction (e.g., intra-op flash-attention K-splitting), it MUST either merge partial results in a fixed, lane-count- independent order or record the resulting per-lane-count-only guarantee here, diff --git a/docs/templates/README.md.j2 b/docs/templates/README.md.j2 index 27081a90..357383ad 100644 --- a/docs/templates/README.md.j2 +++ b/docs/templates/README.md.j2 @@ -3,13 +3,13 @@ CPU-first C++ inference engine for deterministic, high-performance inference built around Stateforward.SML orchestration. -## Status: WIP +## Status: Discontinued -This repository is under active development. APIs, state machines, and formats will change. -If you’re evaluating EMEL, expect fast iteration and breaking changes until the core loader, -allocator, and execution pipelines stabilize. +Development of this C++ implementation has been discontinued in favor of +[stateforward/emel.rs](https://github.com/stateforward/emel.rs). Please use that repository for +current development and future updates. -This inference engine is being implemented by AI under human engineering and architecture direction. +This C++ implementation was developed by AI under human engineering and architecture direction. ## CPU-first scope diff --git a/goal.md b/goal.md index bcc3714d..30a366e2 100644 --- a/goal.md +++ b/goal.md @@ -43,7 +43,7 @@ path beats `llama.cpp` on this CPU. coroutine scheduler cutover but still has too much shared-queue wake/claim overhead for the hot row-sliced matmul actor fanout. The matmul route now uses an EMEL-owned fixed fork/join lane pool with one direct slot per worker - behind an explicit `emel::text::generator::matmul::sm` actor. The caller + behind an explicit `emel::kernel::matmul::sm` actor. The caller still joins every submitted lane actor before dispatch returns, so no deferred work escapes the RTC boundary. - Benchmark finding: the aarch64 f32 GEMV path has a 4-row RHS-reuse kernel, diff --git a/scripts/bench.sh b/scripts/bench.sh index 52b4d6fe..81fd3049 100755 --- a/scripts/bench.sh +++ b/scripts/bench.sh @@ -3,6 +3,8 @@ set -euo pipefail # shellcheck source=scripts/build_jobs.sh source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/build_jobs.sh" +# shellcheck source=scripts/zig_toolchain.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/zig_toolchain.sh" ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" TOOLS_DIR="$ROOT_DIR/tools/bench" @@ -303,6 +305,9 @@ configure_bench_build() { if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake "${cmake_args[@]}" >&2 cmake --build "$build_dir" --parallel "$EMEL_BUILD_JOBS" --target bench_runner >&2 @@ -617,6 +622,9 @@ if $SNAPSHOT; then if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake "${cmake_args[@]}" @@ -699,6 +707,9 @@ if $COMPARE; then if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake "${cmake_args[@]}" >&2 cmake --build "$compare_build_dir" --parallel "$EMEL_BUILD_JOBS" --target bench_runner >&2 @@ -757,6 +768,9 @@ if $RUN_ONLY; then if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake "${cmake_args[@]}" >&2 cmake --build "$run_only_build_dir" --parallel "$EMEL_BUILD_JOBS" --target bench_runner >&2 diff --git a/scripts/bench_diarization_compare.sh b/scripts/bench_diarization_compare.sh index 93141d80..b273b0e4 100755 --- a/scripts/bench_diarization_compare.sh +++ b/scripts/bench_diarization_compare.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" TOOLS_DIR="$ROOT_DIR/tools/bench" BUILD_DIR="${EMEL_BENCH_BUILD_DIR:-$ROOT_DIR/build/bench_tools_ninja}" OUTPUT_DIR="${EMEL_DIARIZATION_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/diarization_compare}" @@ -189,6 +191,9 @@ if ! $SKIP_EMEL_BUILD; then if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake "${cmake_args[@]}" cmake --build "$BUILD_DIR" --parallel --target bench_runner diff --git a/scripts/bench_embedding_compare.sh b/scripts/bench_embedding_compare.sh index 836f32b2..ff9e5269 100755 --- a/scripts/bench_embedding_compare.sh +++ b/scripts/bench_embedding_compare.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" TOOLS_DIR="$ROOT_DIR/tools/bench" BUILD_DIR="${EMEL_BENCH_BUILD_DIR:-$ROOT_DIR/build/bench_tools_ninja}" OUTPUT_DIR="${EMEL_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/embedding_compare}" @@ -137,6 +139,9 @@ if ! $SKIP_EMEL_BUILD; then if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake "${cmake_args[@]}" cmake --build "$BUILD_DIR" --parallel --target embedding_generator_bench_runner diff --git a/scripts/bench_embedding_reference_liquid.sh b/scripts/bench_embedding_reference_liquid.sh index fc9162f7..59d454a4 100755 --- a/scripts/bench_embedding_reference_liquid.sh +++ b/scripts/bench_embedding_reference_liquid.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" TOOLS_DIR="$ROOT_DIR/tools/bench" BUILD_DIR="${EMEL_REFERENCE_BUILD_DIR:-$ROOT_DIR/build/bench_tools_liquid_ninja}" REFERENCE_REPOSITORY="${EMEL_REFERENCE_REPOSITORY:-https://github.com/Liquid4All/benchmarks-llama.cpp.git}" @@ -169,6 +171,9 @@ fi if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi +if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") +fi if ! $RUN_ONLY; then cmake "${cmake_args[@]}" diff --git a/scripts/bench_generation_compare.sh b/scripts/bench_generation_compare.sh index 700f70a8..829551b1 100755 --- a/scripts/bench_generation_compare.sh +++ b/scripts/bench_generation_compare.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" TOOLS_DIR="$ROOT_DIR/tools/bench" BUILD_DIR="${EMEL_BENCH_BUILD_DIR:-$ROOT_DIR/build/bench_tools_ninja}" OUTPUT_DIR="${EMEL_GENERATION_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/generation_compare}" @@ -185,6 +187,9 @@ if ! $SKIP_EMEL_BUILD; then if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake "${cmake_args[@]}" cmake --build "$BUILD_DIR" --parallel "$EMEL_BUILD_JOBS" --target bench_runner diff --git a/scripts/bench_generation_reference_llama_cpp.sh b/scripts/bench_generation_reference_llama_cpp.sh index 4dfe792c..7a1c3ea4 100755 --- a/scripts/bench_generation_reference_llama_cpp.sh +++ b/scripts/bench_generation_reference_llama_cpp.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" TOOLS_DIR="$ROOT_DIR/tools/bench" BUILD_DIR="${EMEL_GENERATION_REFERENCE_BUILD_DIR:-$ROOT_DIR/build/bench_tools_ninja}" BUILD_ONLY=false @@ -203,6 +205,9 @@ if ! $RUN_ONLY; then if [[ -n "$bench_cxx_flags" ]]; then cmake_args+=("-DCMAKE_CXX_FLAGS=$bench_cxx_flags") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake "${cmake_args[@]}" cmake --build "$BUILD_DIR" --parallel "$EMEL_BUILD_JOBS" --target bench_runner diff --git a/scripts/bench_mimi_compare.sh b/scripts/bench_mimi_compare.sh index 83e4a482..88b638cb 100755 --- a/scripts/bench_mimi_compare.sh +++ b/scripts/bench_mimi_compare.sh @@ -14,7 +14,9 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/build_jobs.sh source "$ROOT_DIR/scripts/build_jobs.sh" -BUILD_DIR="${EMEL_MIMI_COMPARE_BUILD_DIR:-$ROOT_DIR/build/mimi_compare_tools}" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" +BUILD_DIR="${EMEL_MIMI_COMPARE_BUILD_DIR:-$ROOT_DIR/build/mimi_compare_tools_zig}" OUTPUT_DIR="${EMEL_MIMI_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/mimi_compare}" ARTIFACT_DIR="${EMEL_MOSHI_REFERENCE_ARTIFACT_DIR:-$ROOT_DIR/build/moshi_reference}" MLX_ARTIFACT_DIR="${EMEL_PERSONAPLEX_MLX_ARTIFACT_DIR:-$ROOT_DIR/build/personaplex_mlx_reference}" @@ -53,7 +55,7 @@ done # setup path supports) can run the WAV generation and comparison steps too. PYTHON_BIN="${PYTHON_BIN:-}" if [[ -z "$PYTHON_BIN" ]]; then - for candidate in python3 python3.13 python3.12; do + for candidate in python3.12 python3.13 python3; do if command -v "$candidate" >/dev/null 2>&1; then PYTHON_BIN="$(command -v "$candidate")" break @@ -65,7 +67,7 @@ if [[ -z "$PYTHON_BIN" ]]; then exit 1 fi if ! $RUN_ONLY; then - for tool in cmake ninja git; do + for tool in cmake ninja git zig; do if ! command -v "$tool" >/dev/null 2>&1; then echo "error: required build tool missing: $tool" >&2 exit 1 @@ -88,10 +90,20 @@ if ! $RUN_ONLY; then skip_moshi=OFF fi + zig_bin="$(command -v zig)" cmake -S "$ROOT_DIR/tools/bench" -B "$BUILD_DIR" -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DEMEL_BENCH_SKIP_MOSHI_REFERENCE="$skip_moshi" \ - -DEMEL_BENCH_SUITE_FILTER=speech_codec_mimi + -DEMEL_BENCH_SUITE_FILTER=speech_codec_mimi \ + -DCMAKE_C_COMPILER="$zig_bin" \ + -DCMAKE_C_COMPILER_ARG1=cc \ + -DCMAKE_CXX_COMPILER="$zig_bin" \ + -DCMAKE_CXX_COMPILER_ARG1=c++ \ + -DCMAKE_ASM_COMPILER="$zig_bin" \ + -DCMAKE_ASM_COMPILER_ARG1=cc \ + -DCMAKE_C_FLAGS=-fno-sanitize=undefined \ + -DCMAKE_CXX_FLAGS=-fno-sanitize=undefined \ + "${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}" cmake --build "$BUILD_DIR" --parallel "$EMEL_BUILD_JOBS" \ --target "${build_targets[@]}" fi diff --git a/scripts/bench_moshi_lm_compare.sh b/scripts/bench_moshi_lm_compare.sh index 747b7b4a..599951d1 100755 --- a/scripts/bench_moshi_lm_compare.sh +++ b/scripts/bench_moshi_lm_compare.sh @@ -8,6 +8,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/build_jobs.sh source "$ROOT_DIR/scripts/build_jobs.sh" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" BUILD_DIR_OVERRIDE="${EMEL_MOSHI_LM_COMPARE_BUILD_DIR:-}" BUILD_DIR="" @@ -137,6 +139,7 @@ if $USE_ZIG; then "-DCMAKE_ASM_COMPILER_ARG1=cc" "-DCMAKE_C_FLAGS=-fno-sanitize=undefined" "-DCMAKE_CXX_FLAGS=-fno-sanitize=undefined") + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") fi if ! $RUN_ONLY; then diff --git a/scripts/bench_personaplex_compare.sh b/scripts/bench_personaplex_compare.sh index 7e31a570..206bbc55 100755 --- a/scripts/bench_personaplex_compare.sh +++ b/scripts/bench_personaplex_compare.sh @@ -5,6 +5,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/build_jobs.sh source "$ROOT_DIR/scripts/build_jobs.sh" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" BUILD_DIR="${EMEL_PERSONAPLEX_COMPARE_BUILD_DIR:-$ROOT_DIR/build/personaplex_compare_tools_zig}" OUTPUT_DIR="${EMEL_PERSONAPLEX_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/personaplex_compare}" @@ -26,11 +28,12 @@ usage: scripts/bench_personaplex_compare.sh [options] "Hey, I'm Gabe. How are you doing?" --emel-lm PATH enriched EMEL LM artifact; defaults to the unpacked model in the configured artifact directory - --emel-mimi PATH enriched EMEL Mimi artifact; defaults to the float - model in the configured artifact directory + --emel-mimi PATH enriched EMEL Mimi artifact; defaults to the selective + Q8_0 model in the configured artifact directory --frames N generated frames per lane (default: 125 / 10 seconds) --seed N fixed sampling seed (default: 1234) - --threads N CPU threads in moshi.cpp reference lane (default: 1) + --threads N active CPU lanes in EMEL and CPU threads in moshi.cpp + (supported: 1, 2, 4, or 8; default: 1) --output-dir PATH output WAV/log/report directory --build-only fetch/configure/build without running --run-only use an existing build and artifacts @@ -63,8 +66,26 @@ for value in "$FRAMES" "$SEED" "$THREADS"; do exit 1 fi done -if ! command -v python3 >/dev/null 2>&1; then - echo "error: required tool missing: python3" >&2 +if [[ ! "$THREADS" =~ ^(1|2|4|8)$ ]]; then + echo "error: threads must be 1, 2, 4, or 8" >&2 + exit 1 +fi +PYTHON_BIN="${PYTHON_BIN:-}" +if [[ -z "$PYTHON_BIN" ]]; then + for candidate in python3.12 python3.13 /usr/bin/python3 python3; do + if command -v "$candidate" >/dev/null 2>&1; then + candidate_path="$(command -v "$candidate")" + if "$candidate_path" -c 'import hashlib, json, subprocess' \ + >/dev/null 2>&1; then + PYTHON_BIN="$candidate_path" + break + fi + fi + done +fi +if [[ -z "$PYTHON_BIN" ]] || + ! "$PYTHON_BIN" -c 'import hashlib, json, subprocess' >/dev/null 2>&1; then + echo "error: no usable Python 3 interpreter" >&2 exit 1 fi @@ -75,7 +96,7 @@ if ! $RUN_ONLY; then exit 1 fi done - "$ROOT_DIR/scripts/setup_moshi_cpp_reference.sh" >/dev/null + bash "$ROOT_DIR/scripts/setup_moshi_cpp_reference.sh" >/dev/null zig_bin="$(command -v zig)" cmake -S "$ROOT_DIR/tools/bench" -B "$BUILD_DIR" -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ @@ -88,7 +109,8 @@ if ! $RUN_ONLY; then -DCMAKE_ASM_COMPILER="$zig_bin" \ -DCMAKE_ASM_COMPILER_ARG1=cc \ -DCMAKE_C_FLAGS=-fno-sanitize=undefined \ - -DCMAKE_CXX_FLAGS=-fno-sanitize=undefined + -DCMAKE_CXX_FLAGS=-fno-sanitize=undefined \ + "${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}" cmake --build "$BUILD_DIR" --parallel "$EMEL_BUILD_JOBS" \ --target personaplex_emel_runner moshi_reference_driver fi @@ -132,19 +154,23 @@ for path in "$AUDIO" "$EMEL_RUNNER" "$REFERENCE_DRIVER" "$EMEL_MIMI" \ fi done -python3 "$ROOT_DIR/tools/bench/personaplex_compare.py" \ - --emel-runner "$EMEL_RUNNER" \ - --reference-driver "$REFERENCE_DRIVER" \ - --emel-mimi "$EMEL_MIMI" \ - --reference-mimi "$REFERENCE_MIMI" \ - --emel-lm "$EMEL_LM" \ - --reference-lm "$REFERENCE_LM" \ - --emel-voice "$EMEL_VOICE" \ - --reference-voice "$REFERENCE_VOICE" \ - --config "$CONFIG" \ - --inference-config "$INFERENCE_CONFIG" \ - --audio "$AUDIO" \ - --output-dir "$OUTPUT_DIR" \ - --frames "$FRAMES" \ - --seed "$SEED" \ +compare_args=( + --emel-runner "$EMEL_RUNNER" + --reference-driver "$REFERENCE_DRIVER" + --emel-mimi "$EMEL_MIMI" + --reference-mimi "$REFERENCE_MIMI" + --emel-lm "$EMEL_LM" + --reference-lm "$REFERENCE_LM" + --emel-voice "$EMEL_VOICE" + --reference-voice "$REFERENCE_VOICE" + --config "$CONFIG" + --inference-config "$INFERENCE_CONFIG" + --audio "$AUDIO" + --output-dir "$OUTPUT_DIR" + --frames "$FRAMES" + --seed "$SEED" --threads "$THREADS" +) + +"$PYTHON_BIN" "$ROOT_DIR/tools/bench/personaplex_compare.py" \ + "${compare_args[@]}" diff --git a/scripts/bench_whisper_compare.sh b/scripts/bench_whisper_compare.sh index ef93af10..4e8ab88a 100755 --- a/scripts/bench_whisper_compare.sh +++ b/scripts/bench_whisper_compare.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" BUILD_DIR="${EMEL_WHISPER_COMPARE_BUILD_DIR:-$ROOT_DIR/build/whisper_compare_tools}" OUTPUT_DIR="${EMEL_WHISPER_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/whisper_compare}" REFERENCE_BUILD_DIR="${EMEL_WHISPER_CPP_BUILD_DIR:-$ROOT_DIR/build/whisper_cpp_reference/build}" @@ -70,6 +72,7 @@ if ! $SKIP_EMEL_BUILD; then ) if [[ -n "$bench_cc_arg" ]]; then cmake_args+=("-DCMAKE_C_COMPILER_ARG1=$bench_cc_arg"); fi if [[ -n "$bench_cxx_arg" ]]; then cmake_args+=("-DCMAKE_CXX_COMPILER_ARG1=$bench_cxx_arg"); fi + if $USE_ZIG; then cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}"); fi cmake "${cmake_args[@]}" cmake --build "$BUILD_DIR" --parallel --target whisper_emel_parity_runner fi diff --git a/scripts/bench_whisper_single_thread.sh b/scripts/bench_whisper_single_thread.sh index 41c26b71..37aa12c1 100755 --- a/scripts/bench_whisper_single_thread.sh +++ b/scripts/bench_whisper_single_thread.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" BUILD_DIR="${EMEL_WHISPER_COMPARE_BUILD_DIR:-$ROOT_DIR/build/whisper_compare_tools}" OUTPUT_DIR="${EMEL_WHISPER_BENCH_OUTPUT_DIR:-$ROOT_DIR/build/whisper_benchmark}" REFERENCE_BUILD_DIR="${EMEL_WHISPER_CPP_BUILD_DIR:-$ROOT_DIR/build/whisper_cpp_reference/build}" @@ -76,6 +78,7 @@ if ! $SKIP_EMEL_BUILD; then ) if [[ -n "$bench_cc_arg" ]]; then cmake_args+=("-DCMAKE_C_COMPILER_ARG1=$bench_cc_arg"); fi if [[ -n "$bench_cxx_arg" ]]; then cmake_args+=("-DCMAKE_CXX_COMPILER_ARG1=$bench_cxx_arg"); fi + if $USE_ZIG; then cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}"); fi cmake "${cmake_args[@]}" cmake --build "$BUILD_DIR" --parallel --target whisper_emel_parity_runner fi diff --git a/scripts/build_with_zig.sh b/scripts/build_with_zig.sh index d9601c4e..4326120b 100755 --- a/scripts/build_with_zig.sh +++ b/scripts/build_with_zig.sh @@ -3,6 +3,8 @@ set -euo pipefail # shellcheck source=scripts/build_jobs.sh source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/build_jobs.sh" +# shellcheck source=scripts/zig_toolchain.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/zig_toolchain.sh" if ! command -v zig >/dev/null 2>&1; then echo "error: zig not found" >&2 @@ -26,5 +28,17 @@ cmake -S . -B build/zig -G Ninja \ -DCMAKE_C_COMPILER_ARG1=cc \ -DCMAKE_CXX_COMPILER="$zig_bin" \ -DCMAKE_CXX_COMPILER_ARG1=c++ \ + "${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}" \ -DEMEL_TEST_SHARDS="$test_shards" cmake --build build/zig --parallel "$EMEL_BUILD_JOBS" + +if [[ -n "$EMEL_ZIG_MACOS_DEPLOYMENT_TARGET" ]]; then + "$PWD/scripts/check_macos_binary_target.sh" \ + "$PWD/build/zig/emel_tests_bin" \ + "$EMEL_ZIG_MACOS_DEPLOYMENT_TARGET" + if ! "$PWD/build/zig/emel_tests_bin" --list-test-cases >/dev/null; then + echo "error: Zig test binary failed the macOS launch probe" >&2 + exit 1 + fi + echo "macOS launch verified: $PWD/build/zig/emel_tests_bin" +fi diff --git a/scripts/check_determinism.sh b/scripts/check_determinism.sh index e4226f2b..12f01a52 100755 --- a/scripts/check_determinism.sh +++ b/scripts/check_determinism.sh @@ -12,6 +12,8 @@ set -euo pipefail # shellcheck source=scripts/build_jobs.sh source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/build_jobs.sh" +# shellcheck source=scripts/zig_toolchain.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/zig_toolchain.sh" ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BUILD_DIR="${EMEL_DETERMINISM_BUILD_DIR:-$ROOT_DIR/build/determinism_check}" @@ -40,6 +42,7 @@ cmake -S "$ROOT_DIR/tools/determinism_check" -B "$BUILD_DIR" -G Ninja \ -DCMAKE_C_COMPILER_ARG1=cc \ -DCMAKE_CXX_COMPILER="$zig_bin" \ -DCMAKE_CXX_COMPILER_ARG1=c++ \ + "${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}" \ -DEMEL_ROOT_DIR="$ROOT_DIR" cmake --build "$BUILD_DIR" --parallel "$EMEL_BUILD_JOBS" diff --git a/scripts/check_macos_binary_target.sh b/scripts/check_macos_binary_target.sh new file mode 100755 index 00000000..60af1a76 --- /dev/null +++ b/scripts/check_macos_binary_target.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$(uname -s)" != "Darwin" ]]; then + exit 0 +fi + +if [[ $# -ne 2 ]]; then + echo "usage: scripts/check_macos_binary_target.sh BINARY EXPECTED_MINIMUM" >&2 + exit 2 +fi + +binary="$1" +expected_minimum="$2" + +if [[ ! -x "$binary" ]]; then + echo "error: macOS target check requires an executable binary: $binary" >&2 + exit 1 +fi +if ! command -v otool >/dev/null 2>&1; then + echo "error: otool is required to inspect the macOS deployment target" >&2 + exit 1 +fi + +actual_minimum="$( + otool -l "$binary" | + awk ' + $1 == "cmd" && $2 == "LC_BUILD_VERSION" { in_build_version = 1; next } + in_build_version && $1 == "minos" { print $2; exit } + ' +)" + +if [[ -z "$actual_minimum" ]]; then + echo "error: missing LC_BUILD_VERSION minimum OS in $binary" >&2 + exit 1 +fi +if [[ "$actual_minimum" != "$expected_minimum" ]]; then + echo "error: macOS minimum OS mismatch for $binary:" \ + "expected $expected_minimum, found $actual_minimum" >&2 + exit 1 +fi + +echo "macOS minimum OS verified: $binary minos=$actual_minimum" diff --git a/scripts/embedded_size.sh b/scripts/embedded_size.sh index 891a64af..faddbd7f 100755 --- a/scripts/embedded_size.sh +++ b/scripts/embedded_size.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" BUILD_ROOT="${EMEL_EMBEDDED_SIZE_BUILD_ROOT:-$ROOT_DIR/build/embedded_size}" REFERENCE_REPOSITORY="${EMEL_EMBEDDED_SIZE_REFERENCE_REPOSITORY:-https://github.com/ggml-org/llama.cpp.git}" REFERENCE_REF="${EMEL_EMBEDDED_SIZE_REF:-}" @@ -214,6 +216,9 @@ configure_cmake_project() { cmake_args+=("-DCMAKE_ASM_COMPILER=$bench_cc") cmake_args+=("-DCMAKE_ASM_COMPILER_ARG1=$bench_asm_arg") fi + if $USE_ZIG; then + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") + fi cmake_args+=("$@") cmake "${cmake_args[@]}" diff --git a/scripts/generate_docs.sh b/scripts/generate_docs.sh index 9db5e062..302deedb 100755 --- a/scripts/generate_docs.sh +++ b/scripts/generate_docs.sh @@ -2,6 +2,10 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/build_jobs.sh +source "$ROOT_DIR/scripts/build_jobs.sh" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" mode="${1:-}" if [[ -n "$mode" && "$mode" != "--check" ]]; then @@ -47,9 +51,10 @@ run_docsgen() { -DCMAKE_C_COMPILER_ARG1="$bench_cc_arg" \ -DCMAKE_CXX_COMPILER_ARG1="$bench_cxx_arg" \ -DCMAKE_ASM_COMPILER_ARG1="$bench_cc_arg" \ + "${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}" \ -DDOCSGEN_PART_SIZE="${DOCSGEN_PART_SIZE:-1}" - cmake --build "$build_dir" --parallel --target docsgen + cmake --build "$build_dir" --parallel "$EMEL_BUILD_JOBS" --target docsgen if $CHECK_MODE; then "$docsgen_bin" --root "$ROOT_DIR" --check diff --git a/scripts/paritychecker.sh b/scripts/paritychecker.sh index 51ca5f8d..18326985 100755 --- a/scripts/paritychecker.sh +++ b/scripts/paritychecker.sh @@ -13,6 +13,11 @@ BUILD_DIR="$ROOT_DIR/build/paritychecker_zig" zig_bin="$(command -v zig)" selected_runners=() +# shellcheck source=build_jobs.sh +source "$ROOT_DIR/scripts/build_jobs.sh" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" + usage() { cat >&2 <<'USAGE' usage: scripts/paritychecker.sh [--runner=|--mode=]... @@ -83,10 +88,11 @@ cmake -S "$ROOT_DIR/tools/paritychecker" -B "$BUILD_DIR" -G Ninja \ -DCMAKE_C_COMPILER_ARG1=cc \ -DCMAKE_CXX_COMPILER="$zig_bin" \ -DCMAKE_CXX_COMPILER_ARG1=c++ \ + "${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}" \ -DGGML_METAL=OFF \ -DLLAMA_METAL=OFF -cmake --build "$BUILD_DIR" --parallel +cmake --build "$BUILD_DIR" --parallel "$EMEL_BUILD_JOBS" if [[ ${#selected_runners[@]} -eq 0 ]]; then ctest --test-dir "$BUILD_DIR" --output-on-failure -R paritychecker_tests exit $? diff --git a/scripts/setup_moshi_cpp_reference.sh b/scripts/setup_moshi_cpp_reference.sh index 2df4b8f6..98a9e021 100755 --- a/scripts/setup_moshi_cpp_reference.sh +++ b/scripts/setup_moshi_cpp_reference.sh @@ -14,8 +14,8 @@ REFERENCE_DIR="${EMEL_MOSHI_CPP_SOURCE_DIR:-$ROOT_DIR/build/moshi_cpp_reference/ ARTIFACT_DIR="${EMEL_MOSHI_REFERENCE_ARTIFACT_DIR:-$ROOT_DIR/build/moshi_reference}" MOSHI_CPP_REPO="${EMEL_MOSHI_CPP_REPO:-https://github.com/Codes4Fun/moshi.cpp.git}" PIN_FILE="$ROOT_DIR/tools/bench/moshi_reference_ref.txt" -MOSHI_CPP_COMMIT="${EMEL_MOSHI_CPP_COMMIT:-$(tr -d '[:space:]' < "$PIN_FILE")}" -MOSHI_CPP_REF="${EMEL_MOSHI_CPP_REF:-$MOSHI_CPP_COMMIT}" +MOSHI_CPP_COMMIT="${EMEL_MOSHI_CPP_COMMIT:-}" +MOSHI_CPP_REF="${EMEL_MOSHI_CPP_REF:-}" PERSONAPLEX_REVISION="1685c70257e525bc6c72470eee1ab2cacff3f709" PERSONAPLEX_BASE="https://huggingface.co/Codes4Fun/personaplex-7b-v1-q4_k-GGUF/resolve/$PERSONAPLEX_REVISION" @@ -39,13 +39,28 @@ MOSHI_MODEL_EMEL="$ARTIFACT_DIR/model-q4_k-emel.gguf" MIMI_MODEL_EMEL="$ARTIFACT_DIR/mimi-e351c8d8-125-emel.gguf" MIMI_MODEL_PERSONAPLEX_EMEL="$ARTIFACT_DIR/mimi-e351c8d8-125-personaplex-emel.gguf" VOICE_MODEL_EMEL="$ARTIFACT_DIR/voices/NATF0-emel.gguf" +MIMI_PERSONAPLEX_Q8_TENSORS=68 FETCH_ONLY=false BUILD_ONLY=false +VALIDATE_Q8_OUTPUT="" +VALIDATE_Q8_MANIFEST="" +VALIDATE_Q8_TENSORS="" +ENSURE_Q8_OUTPUT="" +ENSURE_Q8_MANIFEST="" +ENSURE_Q8_TENSORS="" +ENSURE_Q8_CONVERTER="" +ENSURE_Q8_SOURCE="" +ENSURE_Q8_CONVERTER_ARGS=() usage() { cat <<'USAGE' usage: scripts/setup_moshi_cpp_reference.sh [--fetch-only] [--build-only] + scripts/setup_moshi_cpp_reference.sh \ + --validate-q8-artifact OUTPUT MANIFEST EXPECTED_TENSORS + scripts/setup_moshi_cpp_reference.sh \ + --ensure-q8-artifact OUTPUT MANIFEST EXPECTED_TENSORS CONVERTER SOURCE \ + [CONVERTER_ARGS...] Fetches pinned moshi.cpp, downloads the sha256-pinned PersonaPlex-7B q4_k GGUF plus Mimi/tokenizer/voice artifacts, generates the deterministic 24 kHz WAV @@ -58,35 +73,143 @@ while [[ $# -gt 0 ]]; do case "$1" in --fetch-only) FETCH_ONLY=true; shift ;; --build-only) BUILD_ONLY=true; shift ;; + --validate-q8-artifact) + if [[ $# -lt 4 ]]; then + echo "error: --validate-q8-artifact requires OUTPUT MANIFEST EXPECTED_TENSORS" >&2 + exit 1 + fi + VALIDATE_Q8_OUTPUT="$2" + VALIDATE_Q8_MANIFEST="$3" + VALIDATE_Q8_TENSORS="$4" + shift 4 + ;; + --ensure-q8-artifact) + if [[ $# -lt 6 ]]; then + echo "error: --ensure-q8-artifact requires OUTPUT MANIFEST EXPECTED_TENSORS CONVERTER SOURCE" >&2 + exit 1 + fi + ENSURE_Q8_OUTPUT="$2" + ENSURE_Q8_MANIFEST="$3" + ENSURE_Q8_TENSORS="$4" + ENSURE_Q8_CONVERTER="$5" + ENSURE_Q8_SOURCE="$6" + shift 6 + ENSURE_Q8_CONVERTER_ARGS=("$@") + set -- + ;; -h|--help) usage; exit 0 ;; *) echo "error: unknown argument '$1'" >&2; usage; exit 1 ;; esac done -for tool in git shasum curl; do - if ! command -v "$tool" >/dev/null 2>&1; then - echo "error: required tool missing: $tool" >&2 - exit 1 - fi -done - # Discover the interpreter the same way the compare wrapper does, so hosts # that ship python3.12/python3.13 without a python3 shim can run the WAV # generation and the converter too. PYTHON_BIN="${PYTHON_BIN:-}" if [[ -z "$PYTHON_BIN" ]]; then - for candidate in python3 python3.13 python3.12; do + for candidate in python3.12 python3.13 /usr/bin/python3 python3; do if command -v "$candidate" >/dev/null 2>&1; then - PYTHON_BIN="$(command -v "$candidate")" - break + candidate_path="$(command -v "$candidate")" + if "$candidate_path" -c 'import hashlib, json, subprocess' \ + >/dev/null 2>&1; then + PYTHON_BIN="$candidate_path" + break + fi fi done fi -if [[ -z "$PYTHON_BIN" ]]; then - echo "error: required tool missing: python3 (or python3.12/python3.13)" >&2 +if [[ -z "$PYTHON_BIN" ]] || + ! "$PYTHON_BIN" -c 'import hashlib, json, subprocess' >/dev/null 2>&1; then + echo "error: no usable Python 3 interpreter" >&2 exit 1 fi +q8_manifest_matches_output() { + local output="$1" + local manifest="$2" + local expected_tensors="$3" + "$PYTHON_BIN" - "$output" "$manifest" "$expected_tensors" <<'PY' +import hashlib +import json +import pathlib +import sys + +output = pathlib.Path(sys.argv[1]) +manifest_path = pathlib.Path(sys.argv[2]) +expected_tensors = int(sys.argv[3]) +if not output.is_file() or not manifest_path.is_file(): + raise SystemExit("Q8 output and manifest must both exist") + +manifest = json.load(open(manifest_path, encoding="utf-8")) +if manifest.get("quantization") != "q8_0": + raise SystemExit("Q8 manifest quantization mismatch") +if manifest.get("quantized_tensors") != expected_tensors: + raise SystemExit("Q8 manifest tensor-count mismatch") + +digest = hashlib.sha256() +with open(output, "rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) +if manifest.get("enriched_sha256") != digest.hexdigest(): + raise SystemExit("Q8 manifest output identity mismatch") +PY +} + +rebuild_q8_artifact() { + local output="$1" + local manifest="$2" + local expected_tensors="$3" + local converter="$4" + local source="$5" + shift 5 + "$PYTHON_BIN" "$converter" \ + --source "$source" --output "$output" --manifest "$manifest" "$@" + q8_manifest_matches_output "$output" "$manifest" "$expected_tensors" +} + +ensure_q8_artifact() { + local output="$1" + local manifest="$2" + local expected_tensors="$3" + local converter="$4" + local source="$5" + shift 5 + if q8_manifest_matches_output \ + "$output" "$manifest" "$expected_tensors" >/dev/null 2>&1; then + return 0 + fi + rebuild_q8_artifact \ + "$output" "$manifest" "$expected_tensors" "$converter" "$source" "$@" +} + +if [[ -n "$VALIDATE_Q8_OUTPUT" ]]; then + q8_manifest_matches_output \ + "$VALIDATE_Q8_OUTPUT" "$VALIDATE_Q8_MANIFEST" "$VALIDATE_Q8_TENSORS" + exit 0 +fi + +if [[ -n "$ENSURE_Q8_OUTPUT" ]]; then + ensure_q8_artifact \ + "$ENSURE_Q8_OUTPUT" "$ENSURE_Q8_MANIFEST" "$ENSURE_Q8_TENSORS" \ + "$ENSURE_Q8_CONVERTER" "$ENSURE_Q8_SOURCE" \ + "${ENSURE_Q8_CONVERTER_ARGS[@]}" + exit 0 +fi + +for tool in git shasum curl; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "error: required tool missing: $tool" >&2 + exit 1 + fi +done + +if [[ -z "$MOSHI_CPP_COMMIT" ]]; then + MOSHI_CPP_COMMIT="$(tr -d '[:space:]' < "$PIN_FILE")" +fi +if [[ -z "$MOSHI_CPP_REF" ]]; then + MOSHI_CPP_REF="$MOSHI_CPP_COMMIT" +fi + if $BUILD_ONLY; then # M2: the reference driver builds through the speech_codec_mimi bench # configure (moshi.cpp + ggml + SentencePiece); delegate to the compare @@ -174,7 +297,9 @@ convert_if_needed() { # The converter embeds its side inputs (--config/--tokenizer files) as # hparams/tokenizer metadata, so any of them being newer than the enriched # output must force a reconvert, not just the raw GGUF. + local manifest="${output%.gguf}.manifest.json" local stale=0 + local quantize_q8=0 if [[ ! -f "$output" || "$source" -nt "$output" ]]; then stale=1 fi @@ -184,15 +309,36 @@ convert_if_needed() { stale=1 fi local arg + local previous_arg="" for arg in "$@"; do if [[ -f "$arg" && "$arg" -nt "$output" ]]; then stale=1 fi + if [[ "$previous_arg" == "--quantize" && "$arg" == "q8_0" ]]; then + quantize_q8=1 + fi + previous_arg="$arg" done + # A pre-existing float artifact is not current merely because its mtime is + # newer. When the maintained call requests Q8_0, bind freshness to the + # converter manifest's operand-class contract as well. + if ((quantize_q8)); then + if ! q8_manifest_matches_output \ + "$output" "$manifest" "$MIMI_PERSONAPLEX_Q8_TENSORS" \ + >/dev/null 2>&1; then + stale=1 + fi + fi if ((stale)); then - "$PYTHON_BIN" "$ROOT_DIR/tools/bench/moshi_gguf_convert.py" \ - --source "$source" --output "$output" \ - --manifest "${output%.gguf}.manifest.json" "$@" + if ((quantize_q8)); then + rebuild_q8_artifact \ + "$output" "$manifest" "$MIMI_PERSONAPLEX_Q8_TENSORS" \ + "$ROOT_DIR/tools/bench/moshi_gguf_convert.py" "$source" "$@" + else + "$PYTHON_BIN" "$ROOT_DIR/tools/bench/moshi_gguf_convert.py" \ + --source "$source" --output "$output" \ + --manifest "$manifest" "$@" + fi fi } @@ -202,7 +348,8 @@ convert_if_needed "$MOSHI_MODEL" "$MOSHI_MODEL_EMEL" \ convert_if_needed "$MIMI_MODEL" "$MIMI_MODEL_EMEL" \ --config "$MOSHI_CONFIG" --mimi-n-q "$MIMI_FULL_N_Q" convert_if_needed "$MIMI_MODEL" "$MIMI_MODEL_PERSONAPLEX_EMEL" \ - --config "$MOSHI_CONFIG" --inference-config "$MOSHI_INFERENCE_CONFIG" + --config "$MOSHI_CONFIG" --inference-config "$MOSHI_INFERENCE_CONFIG" \ + --quantize q8_0 convert_if_needed "$VOICE_MODEL" "$VOICE_MODEL_EMEL" echo "EMEL_MOSHI_CPP_SOURCE=$REFERENCE_DIR" diff --git a/scripts/setup_personaplex_mlx_reference.sh b/scripts/setup_personaplex_mlx_reference.sh index 13c3b86f..b1f09d35 100755 --- a/scripts/setup_personaplex_mlx_reference.sh +++ b/scripts/setup_personaplex_mlx_reference.sh @@ -112,9 +112,35 @@ fi if [[ ! -x "$VENV_DIR/bin/python" ]]; then "$PYTHON_BIN" -m venv "$VENV_DIR" fi -"$VENV_DIR/bin/pip" install --quiet --upgrade pip -"$VENV_DIR/bin/pip" install --quiet -e "$SRC_DIR" -"$VENV_DIR/bin/python" -c "from personaplex_mlx import models; models.mimi.mimi_202407(16)" \ +probe_personaplex_mlx() { + local status=0 + for _ in 1 2 3; do + if "$VENV_DIR/bin/python" -c \ + "from personaplex_mlx import models; models.mimi.mimi_202407(16)" \ + >/dev/null 2>&1; then + return 0 + else + status=$? + fi + if [[ "$status" -ne 137 ]]; then + return "$status" + fi + done + return "$status" +} + +if probe_personaplex_mlx; then + : +else + probe_status=$? + if [[ "$probe_status" -eq 137 ]]; then + echo "error: personaplex_mlx import probe was repeatedly killed" >&2 + exit "$probe_status" + fi + "$VENV_DIR/bin/pip" install --quiet --upgrade pip + "$VENV_DIR/bin/pip" install --quiet -e "$SRC_DIR" +fi +probe_personaplex_mlx \ || { echo "error: personaplex_mlx import check failed" >&2; exit 1; } fetch_pinned "$MIMI_SAFETENSORS_URL" "$MIMI_SAFETENSORS" "$MIMI_SAFETENSORS_SHA256" diff --git a/scripts/setup_whisper_cpp_reference.sh b/scripts/setup_whisper_cpp_reference.sh index ac4dd0e3..33025da5 100755 --- a/scripts/setup_whisper_cpp_reference.sh +++ b/scripts/setup_whisper_cpp_reference.sh @@ -2,6 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/zig_toolchain.sh +source "$ROOT_DIR/scripts/zig_toolchain.sh" REFERENCE_DIR="${EMEL_WHISPER_CPP_SOURCE_DIR:-$ROOT_DIR/build/whisper_cpp_reference/src}" BUILD_DIR="${EMEL_WHISPER_CPP_BUILD_DIR:-$ROOT_DIR/build/whisper_cpp_reference/build}" ARTIFACT_DIR="${EMEL_WHISPER_REFERENCE_ARTIFACT_DIR:-$ROOT_DIR/build/whisper_reference}" @@ -122,6 +124,7 @@ if $USE_ZIG; then cmake_args+=("-DCMAKE_C_COMPILER_ARG1=cc") cmake_args+=("-DCMAKE_CXX_COMPILER_ARG1=c++") cmake_args+=("-DCMAKE_ASM_COMPILER_ARG1=cc") + cmake_args+=("${EMEL_ZIG_CMAKE_PLATFORM_ARGS[@]}") fi cmake "${cmake_args[@]}" cmake --build "$BUILD_DIR" --parallel --target whisper-cli diff --git a/scripts/test_with_coverage.sh b/scripts/test_with_coverage.sh index 6ea51ba5..03940a7e 100755 --- a/scripts/test_with_coverage.sh +++ b/scripts/test_with_coverage.sh @@ -16,6 +16,8 @@ COVERAGE_TEST_REGEX="${EMEL_COVERAGE_TEST_REGEX:-}" COVERAGE_GCOV_JOBS="${EMEL_COVERAGE_GCOV_JOBS:-}" COVERAGE_TEST_EXTRA_ARG="${EMEL_COVERAGE_TEST_EXTRA_ARG:-}" COVERAGE_TEST_SHARDS="${EMEL_COVERAGE_TEST_SHARDS:-}" +COVERAGE_C_COMPILER="${EMEL_COVERAGE_C_COMPILER:-gcc}" +COVERAGE_CXX_COMPILER="${EMEL_COVERAGE_CXX_COMPILER:-g++}" # ctest's built-in default test timeout is 1500s; O0 coverage-instrumented # shards can exceed it on slower hosts, so make it overridable. COVERAGE_CTEST_TIMEOUT="${EMEL_COVERAGE_CTEST_TIMEOUT:-1500}" @@ -39,7 +41,16 @@ if ! command -v llvm-cov >/dev/null 2>&1 || ! command -v llvm-profdata >/dev/nul done fi -required_tools=(cmake ctest gcovr clang-format llvm-cov llvm-profdata gcc g++) +required_tools=( + cmake + ctest + gcovr + clang-format + llvm-cov + llvm-profdata + "$COVERAGE_C_COMPILER" + "$COVERAGE_CXX_COMPILER" +) if [[ "$COVERAGE_CHANGED_ONLY" == "1" && "$COVERAGE_CHANGED_LINE_ONLY" != "0" ]]; then required_tools+=(python3) fi @@ -288,16 +299,17 @@ fi cmake -S . -B "$COVERAGE_BUILD_DIR" -G Ninja \ -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_C_COMPILER=gcc \ - -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_C_COMPILER="$COVERAGE_C_COMPILER" \ + -DCMAKE_CXX_COMPILER="$COVERAGE_CXX_COMPILER" \ -DCMAKE_C_FLAGS="--coverage -O0 -fprofile-update=atomic" \ -DCMAKE_CXX_FLAGS="--coverage -O0 -fprofile-update=atomic" \ -DCMAKE_EXE_LINKER_FLAGS="--coverage" \ -DEMEL_TEST_EXTRA_ARG="$COVERAGE_TEST_EXTRA_ARG" \ -DEMEL_TEST_SHARDS="$COVERAGE_TEST_SHARDS" -# Instrumented g++ peaks ~7-8GB on the template-heavy TUs: recompute the -# bound with an 8GB per-job budget regardless of the inherited job count. +# Instrumented C++ compilation peaks ~7-8GB on the template-heavy TUs: +# recompute the bound with an 8GB per-job budget regardless of the inherited +# job count. COVERAGE_BUILD_JOBS="${EMEL_COVERAGE_BUILD_JOBS:-$(emel_compute_build_jobs 8)}" cmake --build "$COVERAGE_BUILD_DIR" --parallel "$COVERAGE_BUILD_JOBS" diff --git a/scripts/zig_toolchain.sh b/scripts/zig_toolchain.sh new file mode 100644 index 00000000..4248c09c --- /dev/null +++ b/scripts/zig_toolchain.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Shared CMake target configuration for Zig builds. +# +# Zig 0.15 on macOS derives an exact host-version deployment target when no +# explicit compiler target is provided. That can produce binaries whose +# LC_BUILD_VERSION is newer than the active Xcode SDK and which dyld rejects +# before main. CMake's OSX deployment setting alone is insufficient because +# Zig ignores the resulting -mmacosx-version-min flag; CMAKE_*_COMPILER_TARGET +# is required so CMake passes --target to Zig. + +EMEL_ZIG_CMAKE_PLATFORM_ARGS=() +EMEL_ZIG_MACOS_DEPLOYMENT_TARGET="" + +if [[ "$(uname -s)" == "Darwin" ]]; then + if ! command -v xcrun >/dev/null 2>&1; then + echo "error: xcrun is required to configure Zig for macOS" >&2 + return 1 2>/dev/null || exit 1 + fi + + emel_zig_macos_sysroot="${EMEL_MACOS_SYSROOT:-$(xcrun --sdk macosx --show-sdk-path)}" + EMEL_ZIG_MACOS_DEPLOYMENT_TARGET="${EMEL_MACOS_DEPLOYMENT_TARGET:-$( + xcrun --sdk macosx --show-sdk-version + )}" + + if [[ ! -d "$emel_zig_macos_sysroot" ]]; then + echo "error: active macOS SDK does not exist: $emel_zig_macos_sysroot" >&2 + return 1 2>/dev/null || exit 1 + fi + if [[ ! "$EMEL_ZIG_MACOS_DEPLOYMENT_TARGET" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then + echo "error: invalid macOS deployment target: $EMEL_ZIG_MACOS_DEPLOYMENT_TARGET" >&2 + return 1 2>/dev/null || exit 1 + fi + + case "$(uname -m)" in + arm64|aarch64) + emel_zig_macos_arch="aarch64" + ;; + x86_64|amd64) + emel_zig_macos_arch="x86_64" + ;; + *) + echo "error: unsupported macOS architecture for Zig: $(uname -m)" >&2 + return 1 2>/dev/null || exit 1 + ;; + esac + + emel_zig_macos_target="${emel_zig_macos_arch}-macos.${EMEL_ZIG_MACOS_DEPLOYMENT_TARGET}" + emel_zig_macos_frameworks="$emel_zig_macos_sysroot/System/Library/Frameworks" + + if [[ ! -d "$emel_zig_macos_frameworks" ]]; then + echo "error: active macOS SDK framework directory does not exist: $emel_zig_macos_frameworks" >&2 + return 1 2>/dev/null || exit 1 + fi + + # Zig's explicit Darwin target does not add the selected Xcode SDK's + # usr/include directory, and its Clang frontend rejects an enum forward- + # declaration extension used by those SDK headers. Keep the deployment + # target explicit while binding the complete matching SDK header surface. + EMEL_ZIG_CMAKE_PLATFORM_ARGS+=( + "-DCMAKE_OSX_SYSROOT=$emel_zig_macos_sysroot" + "-DCMAKE_OSX_DEPLOYMENT_TARGET=$EMEL_ZIG_MACOS_DEPLOYMENT_TARGET" + "-DCMAKE_C_COMPILER_TARGET=$emel_zig_macos_target" + "-DCMAKE_CXX_COMPILER_TARGET=$emel_zig_macos_target" + "-DCMAKE_C_STANDARD_INCLUDE_DIRECTORIES=$emel_zig_macos_sysroot/usr/include" + "-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=$emel_zig_macos_sysroot/usr/include" + "-DCMAKE_C_FLAGS_RELEASE=-O3 -DNDEBUG -Wno-elaborated-enum-base" + "-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -Wno-elaborated-enum-base" + "-DCMAKE_EXE_LINKER_FLAGS=-F$emel_zig_macos_frameworks -Wl,-dead_strip" + "-DCMAKE_SHARED_LINKER_FLAGS=-F$emel_zig_macos_frameworks" + "-DCMAKE_MODULE_LINKER_FLAGS=-F$emel_zig_macos_frameworks" + ) +fi diff --git a/snapshots/lint/clang_format.txt b/snapshots/lint/clang_format.txt index 5b417438..4f967b74 100644 --- a/snapshots/lint/clang_format.txt +++ b/snapshots/lint/clang_format.txt @@ -350,7 +350,6 @@ src/emel/model/tensor/window/events.hpp src/emel/model/tensor/window/guards.hpp src/emel/model/tensor/window/sm.hpp src/emel/sm.hpp -src/emel/speech/codec/mimi/any.hpp src/emel/speech/decoder/whisper/actions.hpp src/emel/speech/decoder/whisper/any.hpp src/emel/speech/decoder/whisper/context.hpp @@ -360,6 +359,8 @@ src/emel/speech/encoder/whisper/context.hpp src/emel/speech/encoder/whisper/errors.hpp src/emel/speech/encoder/whisper/guards.hpp src/emel/speech/predictor/moshi/events.hpp +src/emel/speech/predictor/moshi/executor/actions.hpp +src/emel/speech/predictor/moshi/executor/guards.hpp src/emel/tensor/view/actions.hpp src/emel/tensor/view/context.hpp src/emel/tensor/view/detail.hpp @@ -449,12 +450,6 @@ src/emel/text/generator/initializer/context.hpp src/emel/text/generator/initializer/detail.hpp src/emel/text/generator/initializer/guards.hpp src/emel/text/generator/initializer/sm.hpp -src/emel/text/generator/matmul/actions.hpp -src/emel/text/generator/matmul/context.hpp -src/emel/text/generator/matmul/detail.hpp -src/emel/text/generator/matmul/events.hpp -src/emel/text/generator/matmul/guards.hpp -src/emel/text/generator/matmul/sm.hpp src/emel/text/generator/prefill/actions.hpp src/emel/text/generator/prefill/context.hpp src/emel/text/generator/prefill/detail.hpp @@ -615,7 +610,6 @@ tests/text/generator/action_guard_tests.cpp tests/text/generator/decode_wavefront/lifecycle_tests.cpp tests/text/generator/determinism_tests.cpp tests/text/generator/initializer/lifecycle_tests.cpp -tests/text/generator/lifecycle_tests.cpp tests/text/generator/prefill/lifecycle_tests.cpp tests/text/generator/stream_window_tests.cpp tests/text/jinja/formatter_tests.cpp diff --git a/snapshots/quality_gates/timing.txt b/snapshots/quality_gates/timing.txt index 4f18f4cb..3c5d14cc 100644 --- a/snapshots/quality_gates/timing.txt +++ b/snapshots/quality_gates/timing.txt @@ -1,12 +1,12 @@ # quality_gates timing (seconds) -domain_boundaries 0 +domain_boundaries 1 legacy_sml_surface 0 -build_with_zig 1 -bench_snapshot 4 -test_with_coverage 186 -paritychecker 8 +build_with_zig 9 +bench_snapshot 0 +test_with_coverage 1160 +paritychecker 0 fuzz_smoke 0 determinism_check 0 lint_snapshot 12 -generate_docs 27 -total 226 +generate_docs 0 +total 1182 diff --git a/src/emel/kernel/aarch64/actions.hpp b/src/emel/kernel/aarch64/actions.hpp index 267d4f74..cdddeb40 100644 --- a/src/emel/kernel/aarch64/actions.hpp +++ b/src/emel/kernel/aarch64/actions.hpp @@ -2904,14 +2904,10 @@ dot_q4_k_q8_k_4rows_neon(const ::emel::kernel::detail::quant::block_q4_k *lhs0, #endif } +#if defined(__aarch64__) && defined(__ARM_NEON) inline void decode_q4_k_x8_6bit_scales_neon(const uint8_t *scales_in, int16x8_t &mins_out, int16x8_t &scales_out) noexcept { -#if !(defined(__aarch64__) && defined(__ARM_NEON)) - (void)scales_in; - (void)mins_out; - (void)scales_out; -#else constexpr uint32_t kmask1 = 0x3f3f3f3fu; constexpr uint32_t kmask2 = 0x0f0f0f0fu; constexpr uint32_t kmask3 = 0x03030303u; @@ -2931,8 +2927,8 @@ inline void decode_q4_k_x8_6bit_scales_neon(const uint8_t *scales_in, std::array scales_bytes = {}; std::memcpy(scales_bytes.data(), scales_u32.data(), scales_bytes.size()); scales_out = vmovl_s8(vld1_s8(scales_bytes.data())); -#endif } +#endif inline void dot_q4_k_x8_q8_k_group_bl4_neon( const ::emel::kernel::detail::quant::block_q4_kx8 *lhs, diff --git a/src/emel/kernel/any.hpp b/src/emel/kernel/any.hpp index a85d0942..4de0470b 100644 --- a/src/emel/kernel/any.hpp +++ b/src/emel/kernel/any.hpp @@ -12,19 +12,6 @@ namespace emel::kernel { -enum class kernel_kind : uint8_t { - x86_64 = 0, - aarch64 = 1, -}; - -constexpr kernel_kind detect_host_kind() noexcept { -#if defined(__aarch64__) || defined(_M_ARM64) - return kernel_kind::aarch64; -#else - return kernel_kind::x86_64; -#endif -} - class any { public: any() : core_(detect_host_kind()) {} @@ -43,6 +30,49 @@ class any { bool process_event(const event::dispatch & ev) { return core_.process_event(ev); } + bool process_event(const event::configure_kind &ev) { + core_.set_kind(ev.kind); + return true; + } + + bool process_event(const event::capture_diagnostics &ev) { + ev.out.optimized_flash_dispatch_calls = optimized_flash_dispatch_count(); + ev.out.shared_flash_dispatch_calls = shared_flash_dispatch_count(); + ev.out.optimized_q2_dispatch_calls = optimized_q2_dispatch_count(); + ev.out.shared_q2_dispatch_calls = shared_q2_dispatch_count(); + ev.out.optimized_q3_dispatch_calls = optimized_q3_dispatch_count(); + ev.out.shared_q3_dispatch_calls = shared_q3_dispatch_count(); + ev.out.optimized_q4_dispatch_calls = optimized_q4_dispatch_count(); + ev.out.optimized_q4_vector_dispatch_calls = + optimized_q4_vector_dispatch_count(); + ev.out.optimized_q4_vector_packed_dispatch_calls = + optimized_q4_vector_packed_dispatch_count(); + ev.out.optimized_q4_vector_packed_q8_rhs_dispatch_calls = + optimized_q4_vector_packed_q8_rhs_dispatch_count(); + ev.out.shared_q4_dispatch_calls = shared_q4_dispatch_count(); + ev.out.optimized_q6_dispatch_calls = optimized_q6_dispatch_count(); + ev.out.optimized_q6_vector_dispatch_calls = + optimized_q6_vector_dispatch_count(); + ev.out.optimized_q6_vector_argmax_dispatch_calls = + optimized_q6_vector_argmax_dispatch_count(); + ev.out.optimized_q6_vector_packed_dispatch_calls = + optimized_q6_vector_packed_dispatch_count(); + ev.out.optimized_q6_vector_packed_q8_rhs_dispatch_calls = + optimized_q6_vector_packed_q8_rhs_dispatch_count(); + ev.out.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls = + optimized_q6_vector_packed_q8_rhs_argmax_dispatch_count(); + ev.out.optimized_q6_vector_prepared_q8_rhs_dispatch_calls = + optimized_q6_vector_prepared_q8_rhs_dispatch_count(); + ev.out.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls = + optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_count(); + ev.out.optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls = + optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_count(); + ev.out.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls = + optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_count(); + ev.out.shared_q6_dispatch_calls = shared_q6_dispatch_count(); + return true; + } + template requires(::emel::kernel::is_op_event_v) bool process_event(const event_type & ev) { diff --git a/src/emel/kernel/attention/actions.hpp b/src/emel/kernel/attention/actions.hpp new file mode 100644 index 00000000..a8024ba3 --- /dev/null +++ b/src/emel/kernel/attention/actions.hpp @@ -0,0 +1,125 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "emel/kernel/attention/context.hpp" +#include "emel/kernel/attention/events.hpp" +#include "emel/kernel/detail.hpp" + +namespace emel::kernel::attention::action { + +struct effect_execute_head_range { + void operator()(const event::execute &ev, context &ctx) const noexcept { + ev.result = {}; + const auto &request = ev.request; + const float scale = 1.0f / std::sqrt(static_cast(request.head_dim)); + + for (int32_t head = request.head_begin; head < request.head_end; ++head) { + const int32_t head_offset = head * request.head_dim; + const float *q_head = + request.query.data() + static_cast(head_offset); + for (int32_t dim = 0; dim < request.head_dim; ++dim) { + ctx.q_bf16[static_cast(dim)] = + emel::kernel::detail::fp32_to_bf16(q_head[dim]); + } + for (int32_t physical = 0; physical < request.position_capacity; + ++physical) { + ctx.scores[static_cast(physical)] = + -std::numeric_limits::infinity(); + } + for (int32_t position = 0; position < request.valid_positions; + ++position) { + const int64_t unwrapped = + static_cast(request.physical_begin) + position; + const int32_t physical = static_cast( + unwrapped - + static_cast(unwrapped >= request.position_capacity) * + request.position_capacity); + const std::size_t cache_begin = + request.layer_offset + + static_cast(physical) * + static_cast(request.hidden_dim) + + static_cast(head_offset); + ctx.scores[static_cast(physical)] = + emel::kernel::detail::vec_dot_bf16_ggml( + request.head_dim, request.key_cache.data() + cache_begin, + ctx.q_bf16.data()) * + scale; + } + + emel::kernel::detail::soft_max_row_ggml(request.position_capacity, + ctx.scores.data()); + for (int32_t physical = 0; physical < request.position_capacity; + ++physical) { + ctx.weights_bf16[static_cast(physical)] = + emel::kernel::detail::fp32_to_bf16( + ctx.scores[static_cast(physical)]); + } + + float *attention_head = + request.output.data() + + static_cast(head - request.head_begin) * + static_cast(request.head_dim); + std::fill_n(ctx.output_accumulators.data(), request.head_dim, 0.0); + for (int32_t position = 0; position < request.valid_positions; + ++position) { + const int64_t unwrapped = + static_cast(request.physical_begin) + position; + const int32_t physical = static_cast( + unwrapped - + static_cast(unwrapped >= request.position_capacity) * + request.position_capacity); + const float weight = emel::kernel::detail::bf16_to_fp32( + ctx.weights_bf16[static_cast(physical)]); + const uint16_t *value_head = + request.value_cache.data() + request.layer_offset + + static_cast(physical) * + static_cast(request.hidden_dim) + + static_cast(head_offset); + for (int32_t dim = 0; dim < request.head_dim; ++dim) { + ctx.output_accumulators[static_cast(dim)] += + static_cast( + emel::kernel::detail::bf16_to_fp32(value_head[dim]) * weight); + } + } + for (int32_t dim = 0; dim < request.head_dim; ++dim) { + attention_head[dim] = static_cast( + ctx.output_accumulators[static_cast(dim)]); + } + } + ev.result.accepted = true; + } +}; + +struct effect_reject_execute { + void operator()(const event::execute &ev, context &) const noexcept { + ev.result.accepted = false; + } +}; + +struct effect_accept_execute { + void operator()(const event::execute &, context &) const noexcept {} +}; + +struct effect_emit_execute_done { + void operator()(const event::execute &ev, context &) const noexcept { + ev.on_done(events::execute_done{.request = &ev}); + } +}; + +struct effect_emit_execute_error { + void operator()(const event::execute &ev, context &) const noexcept { + ev.on_error(events::execute_error{.request = &ev}); + } +}; + +struct effect_on_unexpected { + template + void operator()(const event_type &, context &) const noexcept {} +}; + +} // namespace emel::kernel::attention::action diff --git a/src/emel/kernel/attention/any.hpp b/src/emel/kernel/attention/any.hpp new file mode 100644 index 00000000..0624e327 --- /dev/null +++ b/src/emel/kernel/attention/any.hpp @@ -0,0 +1,3 @@ +#pragma once + +#include "emel/kernel/attention/sm.hpp" diff --git a/src/emel/kernel/attention/context.hpp b/src/emel/kernel/attention/context.hpp new file mode 100644 index 00000000..5db3cdd2 --- /dev/null +++ b/src/emel/kernel/attention/context.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include + +namespace emel::kernel::attention::action { + +inline constexpr std::size_t k_max_head_dim = 8192u; +inline constexpr std::size_t k_max_context = 8192u; + +// This persistent actor-owned workspace is constructed once and reused by +// same-RTC head-range dispatches. It is intentionally not dispatch-local state. +struct context { + alignas(64) std::array q_bf16 = {}; + alignas(64) std::array scores = {}; + alignas(64) std::array weights_bf16 = {}; + alignas(64) std::array output_accumulators = {}; +}; + +} // namespace emel::kernel::attention::action diff --git a/src/emel/kernel/attention/events.hpp b/src/emel/kernel/attention/events.hpp new file mode 100644 index 00000000..63b0f90f --- /dev/null +++ b/src/emel/kernel/attention/events.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include + +#include "emel/callback.hpp" + +namespace emel::kernel::attention::events { + +struct execute_done; +struct execute_error; + +} // namespace emel::kernel::attention::events + +namespace emel::kernel::attention::event { + +struct head_range_request { + std::span query = {}; + std::span key_cache = {}; + std::span value_cache = {}; + std::span output = {}; + std::size_t layer_offset = 0u; + int32_t hidden_dim = 0; + int32_t head_dim = 0; + int32_t head_begin = 0; + int32_t head_end = 0; + int32_t position_capacity = 0; + int32_t physical_begin = 0; + int32_t valid_positions = 0; +}; + +struct dispatch_result { + bool accepted = false; +}; + +struct execute { + execute(const head_range_request &request_ref, + dispatch_result &result_ref) noexcept + : request(request_ref), result(result_ref) {} + + const head_range_request &request; + dispatch_result &result; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +} // namespace emel::kernel::attention::event + +namespace emel::kernel::attention::events { + +struct execute_done { + const event::execute *request = nullptr; +}; + +struct execute_error { + const event::execute *request = nullptr; +}; + +} // namespace emel::kernel::attention::events diff --git a/src/emel/kernel/attention/guards.hpp b/src/emel/kernel/attention/guards.hpp new file mode 100644 index 00000000..7d65b675 --- /dev/null +++ b/src/emel/kernel/attention/guards.hpp @@ -0,0 +1,143 @@ +#pragma once + +#include +#include +#include + +#include "emel/kernel/attention/context.hpp" +#include "emel/kernel/attention/events.hpp" + +namespace emel::kernel::attention::guard { + +inline bool guard_ranges_disjoint(const void *first, + const std::size_t first_bytes, + const void *second, + const std::size_t second_bytes) noexcept { + const auto first_begin = reinterpret_cast(first); + const auto second_begin = reinterpret_cast(second); + const auto max_address = std::numeric_limits::max(); + if (first_begin > max_address - first_bytes || + second_begin > max_address - second_bytes) { + return false; + } + const auto first_end = first_begin + first_bytes; + const auto second_end = second_begin + second_bytes; + return first_end <= second_begin || second_end <= first_begin; +} + +inline bool guard_request_shape_supported( + const event::head_range_request &request) noexcept { + if (request.hidden_dim <= 0 || request.head_dim <= 0 || + request.head_begin < 0 || request.head_end <= request.head_begin || + request.position_capacity <= 0 || request.physical_begin < 0 || + request.physical_begin >= request.position_capacity || + request.valid_positions <= 0 || + request.valid_positions > request.position_capacity) { + return false; + } + const std::size_t hidden_dim = static_cast(request.hidden_dim); + const std::size_t head_dim = static_cast(request.head_dim); + const std::size_t capacity = + static_cast(request.position_capacity); + const std::size_t head_begin = static_cast(request.head_begin); + const std::size_t head_end = static_cast(request.head_end); + const std::size_t head_count = head_end - head_begin; + const std::size_t max_size = std::numeric_limits::max(); + if (capacity > (max_size - request.layer_offset) / hidden_dim || + head_count > max_size / head_dim || head_end > hidden_dim / head_dim) { + return false; + } + const std::size_t cache_required = + request.layer_offset + capacity * hidden_dim; + const std::size_t output_required = head_count * head_dim; + if (head_dim > action::k_max_head_dim || capacity > action::k_max_context || + request.query.data() == nullptr || request.key_cache.data() == nullptr || + request.value_cache.data() == nullptr || + request.output.data() == nullptr || request.query.size() < hidden_dim || + request.key_cache.size() < cache_required || + request.value_cache.size() < cache_required || + request.output.size() < output_required || + reinterpret_cast(request.query.data()) % alignof(float) != + 0u || + reinterpret_cast(request.key_cache.data()) % + alignof(uint16_t) != + 0u || + reinterpret_cast(request.value_cache.data()) % + alignof(uint16_t) != + 0u || + reinterpret_cast(request.output.data()) % + alignof(float) != + 0u || + hidden_dim > max_size / sizeof(float) || + cache_required > max_size / sizeof(uint16_t) || + output_required > max_size / sizeof(float)) { + return false; + } + const std::size_t query_bytes = hidden_dim * sizeof(float); + const std::size_t cache_bytes = cache_required * sizeof(uint16_t); + const std::size_t output_bytes = output_required * sizeof(float); + return guard_ranges_disjoint(request.output.data(), output_bytes, + request.query.data(), query_bytes) && + guard_ranges_disjoint(request.output.data(), output_bytes, + request.key_cache.data(), cache_bytes) && + guard_ranges_disjoint(request.output.data(), output_bytes, + request.value_cache.data(), cache_bytes); +} + +struct guard_execute_supported { + bool operator()(const event::execute &ev, + const action::context &) const noexcept { + return guard_request_shape_supported(ev.request); + } +}; + +struct guard_execute_unsupported { + bool operator()(const event::execute &ev, + const action::context &ctx) const noexcept { + return !guard_execute_supported{}(ev, ctx); + } +}; + +struct guard_execute_succeeded { + bool operator()(const event::execute &ev, + const action::context &) const noexcept { + return ev.result.accepted; + } +}; + +struct guard_execute_failed { + bool operator()(const event::execute &ev, + const action::context &) const noexcept { + return !ev.result.accepted; + } +}; + +struct guard_has_done_callback { + bool operator()(const event::execute &ev, + const action::context &) const noexcept { + return static_cast(ev.on_done); + } +}; + +struct guard_no_done_callback { + bool operator()(const event::execute &ev, + const action::context &ctx) const noexcept { + return !guard_has_done_callback{}(ev, ctx); + } +}; + +struct guard_has_error_callback { + bool operator()(const event::execute &ev, + const action::context &) const noexcept { + return static_cast(ev.on_error); + } +}; + +struct guard_no_error_callback { + bool operator()(const event::execute &ev, + const action::context &ctx) const noexcept { + return !guard_has_error_callback{}(ev, ctx); + } +}; + +} // namespace emel::kernel::attention::guard diff --git a/src/emel/kernel/attention/sm.hpp b/src/emel/kernel/attention/sm.hpp new file mode 100644 index 00000000..47255d0d --- /dev/null +++ b/src/emel/kernel/attention/sm.hpp @@ -0,0 +1,125 @@ +#pragma once +// benchmark: designed + +#include "emel/kernel/attention/actions.hpp" +#include "emel/kernel/attention/guards.hpp" +#include "emel/sm.hpp" + +namespace emel::kernel::attention { + +struct state_ready {}; +struct state_execute_result_decision {}; +struct state_execute_done_callback_decision {}; +struct state_execute_error_callback_decision {}; +struct state_execute_done {}; +struct state_execute_errored {}; + +struct model { + auto operator()() const { + namespace sml = stateforward::sml; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Validate and execute one disjoint contiguous range of attention heads. + sml::state <= *sml::state + + sml::event + [ guard::guard_execute_supported{} ] + / action::effect_execute_head_range{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_execute_unsupported{} ] + / action::effect_reject_execute{} + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_execute_succeeded{} ] + / action::effect_accept_execute{} + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_execute_failed{} ] + / action::effect_reject_execute{} + + //------------------------------------------------------------------------------// + // Publish explicit same-RTC outcomes, then return to ready. + , sml::state <= + sml::state + + sml::completion + [ guard::guard_has_done_callback{} ] + / action::effect_emit_execute_done{} + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_no_done_callback{} ] + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_has_error_callback{} ] + / action::effect_emit_execute_error{} + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_no_error_callback{} ] + + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion + + //------------------------------------------------------------------------------// + // Unexpected events are rejected without retaining their payload. + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + ); + // clang-format on + } +}; + +struct sm : public emel::sm { + using base_type = emel::sm; + using base_type::is; + using base_type::visit_current_states; + + sm() = default; + + sm(const sm &) = delete; + sm &operator=(const sm &) = delete; + sm(sm &&) = delete; + sm &operator=(sm &&) = delete; + + bool process_event(const event::execute &ev) { + const bool handled = base_type::process_event(ev); + return handled && ev.result.accepted; + } +}; + +} // namespace emel::kernel::attention diff --git a/src/emel/kernel/events.hpp b/src/emel/kernel/events.hpp index 4b3f02fa..33aabb14 100644 --- a/src/emel/kernel/events.hpp +++ b/src/emel/kernel/events.hpp @@ -11,10 +11,62 @@ namespace emel::kernel::action { struct context; } +namespace emel::kernel { + +enum class kernel_kind : uint8_t { + x86_64 = 0, + aarch64 = 1, +}; + +constexpr kernel_kind detect_host_kind() noexcept { +#if defined(__aarch64__) || defined(_M_ARM64) + return kernel_kind::aarch64; +#else + return kernel_kind::x86_64; +#endif +} + +} // namespace emel::kernel + namespace emel::kernel::event { struct dispatch {}; +struct configure_kind { + emel::kernel::kernel_kind kind = emel::kernel::kernel_kind::x86_64; +}; + +struct diagnostics { + uint64_t optimized_flash_dispatch_calls = 0u; + uint64_t shared_flash_dispatch_calls = 0u; + uint64_t optimized_q2_dispatch_calls = 0u; + uint64_t shared_q2_dispatch_calls = 0u; + uint64_t optimized_q3_dispatch_calls = 0u; + uint64_t shared_q3_dispatch_calls = 0u; + uint64_t optimized_q4_dispatch_calls = 0u; + uint64_t optimized_q4_vector_dispatch_calls = 0u; + uint64_t optimized_q4_vector_packed_dispatch_calls = 0u; + uint64_t optimized_q4_vector_packed_q8_rhs_dispatch_calls = 0u; + uint64_t shared_q4_dispatch_calls = 0u; + uint64_t optimized_q6_dispatch_calls = 0u; + uint64_t optimized_q6_vector_dispatch_calls = 0u; + uint64_t optimized_q6_vector_argmax_dispatch_calls = 0u; + uint64_t optimized_q6_vector_packed_dispatch_calls = 0u; + uint64_t optimized_q6_vector_packed_q8_rhs_dispatch_calls = 0u; + uint64_t optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls = 0u; + uint64_t optimized_q6_vector_prepared_q8_rhs_dispatch_calls = 0u; + uint64_t optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls = 0u; + uint64_t optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls = 0u; + uint64_t optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls = 0u; + uint64_t shared_q6_dispatch_calls = 0u; +}; + +struct capture_diagnostics { + explicit capture_diagnostics(diagnostics &out_ref) noexcept : out(out_ref) {} + + diagnostics &out; +}; + //------------------------------------------------------------------------------ // GGML/GGUF opcode events. // diff --git a/src/emel/kernel/f32_matvec/actions.hpp b/src/emel/kernel/f32_matvec/actions.hpp new file mode 100644 index 00000000..1f0f79b7 --- /dev/null +++ b/src/emel/kernel/f32_matvec/actions.hpp @@ -0,0 +1,186 @@ +#pragma once + +#include +#include + +#if defined(__aarch64__) || defined(_M_ARM64) +#include +#endif + +#include "emel/kernel/detail.hpp" +#include "emel/kernel/f32_matvec/context.hpp" +#include "emel/kernel/f32_matvec/events.hpp" + +namespace emel::kernel::f32_matvec::action { + +template +inline void compute_pack_x4(const std::span source, + const std::span destination, + const uint64_t inner, const uint64_t rows, + const load_type load) noexcept { + const uint64_t full_rows = rows - rows % 4u; + for (uint64_t row = 0u; row < full_rows; row += 4u) { + float *packed = destination.data() + row * inner; + for (uint64_t column = 0u; column < inner; ++column) { + packed[column * 4u + 0u] = load(source[(row + 0u) * inner + column]); + packed[column * 4u + 1u] = load(source[(row + 1u) * inner + column]); + packed[column * 4u + 2u] = load(source[(row + 2u) * inner + column]); + packed[column * 4u + 3u] = load(source[(row + 3u) * inner + column]); + } + } + for (uint64_t row = full_rows; row < rows; ++row) { + for (uint64_t column = 0u; column < inner; ++column) { + destination[row * inner + column] = load(source[row * inner + column]); + } + } +} + +inline float compute_reference_row(const float *weights, const float *input, + const uint64_t inner) noexcept { + double sum = 0.0; + for (uint64_t column = 0u; column < inner; ++column) { + sum += static_cast(weights[column] * input[column]); + } + return static_cast(sum); +} + +struct effect_prepare_f32 { + void operator()(const event::prepare_f32 &ev, context &ctx) const noexcept { + compute_pack_x4(ev.request.source, ev.request.destination, ev.request.inner, + ev.request.rows, + [](const float value) noexcept { return value; }); + ++ctx.prepare_calls; + ctx.prepared_floats += ev.request.inner * ev.request.rows; + ev.result.accepted = true; + } +}; + +struct effect_prepare_f16 { + void operator()(const event::prepare_f16 &ev, context &ctx) const noexcept { + compute_pack_x4(ev.request.source, ev.request.destination, ev.request.inner, + ev.request.rows, [](const uint16_t value) noexcept { + return emel::kernel::detail::quant::fp16_to_fp32(value); + }); + ++ctx.prepare_calls; + ctx.prepared_floats += ev.request.inner * ev.request.rows; + ev.result.accepted = true; + } +}; + +struct effect_execute_reference { + void operator()(const event::execute_reference &ev, + context &ctx) const noexcept { + for (uint64_t row = 0u; row < ev.request.rows; ++row) { + ev.request.output[row] = compute_reference_row( + ev.request.weights.data() + row * ev.request.inner, + ev.request.input.data(), ev.request.inner); + } + ++ctx.reference_calls; + ev.result.accepted = true; + } +}; + +struct effect_execute_exact_x4 { + void operator()(const event::execute_exact_x4 &ev, + context &ctx) const noexcept { +#if defined(__aarch64__) || defined(_M_ARM64) + const uint64_t full_rows = ev.request.rows - ev.request.rows % 4u; + for (uint64_t row = 0u; row < full_rows; row += 4u) { + const float *weights = ev.request.weights.data() + row * ev.request.inner; + float64x2_t sums_low = vdupq_n_f64(0.0); + float64x2_t sums_high = vdupq_n_f64(0.0); + for (uint64_t column = 0u; column < ev.request.inner; ++column) { + const float32x4_t products = vmulq_n_f32( + vld1q_f32(weights + column * 4u), ev.request.input[column]); + sums_low = vaddq_f64(sums_low, vcvt_f64_f32(vget_low_f32(products))); + sums_high = vaddq_f64(sums_high, vcvt_f64_f32(vget_high_f32(products))); + } + const float32x2_t output_low = vcvt_f32_f64(sums_low); + const float32x2_t output_high = vcvt_f32_f64(sums_high); + vst1_f32(ev.request.output.data() + row, output_low); + vst1_f32(ev.request.output.data() + row + 2u, output_high); + } + for (uint64_t row = full_rows; row < ev.request.rows; ++row) { + ev.request.output[row] = compute_reference_row( + ev.request.weights.data() + row * ev.request.inner, + ev.request.input.data(), ev.request.inner); + } + ++ctx.exact_x4_calls; + ev.result.accepted = true; +#else + (void)ev; + (void)ctx; +#endif + } +}; + +template struct effect_reject { + void operator()(const event_type &ev, context &) const noexcept { + ev.result.accepted = false; + } +}; + +template struct effect_accept { + void operator()(const event_type &, context &) const noexcept {} +}; + +template struct effect_emit_done { + void operator()(const event_type &ev, context &) const noexcept { + ev.on_done(events::dispatch_done{.request = ev}); + } +}; + +template struct effect_emit_error { + void operator()(const event_type &ev, context &) const noexcept { + ev.on_error(events::dispatch_error{.request = ev}); + } +}; + +struct effect_reject_prepare_f32 : effect_reject {}; +struct effect_accept_prepare_f32 : effect_accept {}; +struct effect_emit_prepare_f32_done : effect_emit_done {}; +struct effect_emit_prepare_f32_error : effect_emit_error {}; +struct effect_reject_prepare_f16 : effect_reject {}; +struct effect_accept_prepare_f16 : effect_accept {}; +struct effect_emit_prepare_f16_done : effect_emit_done {}; +struct effect_emit_prepare_f16_error : effect_emit_error {}; +struct effect_reject_execute_reference + : effect_reject {}; +struct effect_accept_execute_reference + : effect_accept {}; +struct effect_emit_execute_reference_done + : effect_emit_done {}; +struct effect_emit_execute_reference_error + : effect_emit_error {}; +struct effect_reject_execute_exact_x4 : effect_reject { +}; +struct effect_accept_execute_exact_x4 : effect_accept { +}; +struct effect_emit_execute_exact_x4_done + : effect_emit_done {}; +struct effect_emit_execute_exact_x4_error + : effect_emit_error {}; + +struct effect_capture_diagnostics { + void operator()(const event::capture_diagnostics &ev, + context &ctx) const noexcept { + ev.out = event::diagnostics{ + .prepare_calls = ctx.prepare_calls, + .prepared_floats = ctx.prepared_floats, + .reference_calls = ctx.reference_calls, + .exact_x4_calls = ctx.exact_x4_calls, + }; + ev.result.accepted = true; + } +}; + +struct effect_on_unexpected { + template + void operator()(const event_type &ev, context &) const noexcept { + if constexpr (requires { ev.result.accepted; }) { + ev.result.accepted = false; + } + } +}; + +} // namespace emel::kernel::f32_matvec::action diff --git a/src/emel/kernel/f32_matvec/any.hpp b/src/emel/kernel/f32_matvec/any.hpp new file mode 100644 index 00000000..400e87e0 --- /dev/null +++ b/src/emel/kernel/f32_matvec/any.hpp @@ -0,0 +1,3 @@ +#pragma once + +#include "emel/kernel/f32_matvec/sm.hpp" diff --git a/src/emel/kernel/f32_matvec/context.hpp b/src/emel/kernel/f32_matvec/context.hpp new file mode 100644 index 00000000..61096c0d --- /dev/null +++ b/src/emel/kernel/f32_matvec/context.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace emel::kernel::f32_matvec::action { + +struct context { + uint64_t prepare_calls = 0u; + uint64_t prepared_floats = 0u; + uint64_t reference_calls = 0u; + uint64_t exact_x4_calls = 0u; +}; + +} // namespace emel::kernel::f32_matvec::action diff --git a/src/emel/kernel/f32_matvec/events.hpp b/src/emel/kernel/f32_matvec/events.hpp new file mode 100644 index 00000000..3496dc4e --- /dev/null +++ b/src/emel/kernel/f32_matvec/events.hpp @@ -0,0 +1,114 @@ +#pragma once + +#include +#include + +#include "emel/callback.hpp" + +namespace emel::kernel::f32_matvec::events { + +template struct dispatch_done { + const request_type &request; +}; + +template struct dispatch_error { + const request_type &request; +}; + +} // namespace emel::kernel::f32_matvec::events + +namespace emel::kernel::f32_matvec::event { + +struct dispatch_result { + bool accepted = false; +}; + +struct diagnostics { + uint64_t prepare_calls = 0u; + uint64_t prepared_floats = 0u; + uint64_t reference_calls = 0u; + uint64_t exact_x4_calls = 0u; +}; + +struct prepare_f32_request { + std::span source = {}; + std::span destination = {}; + uint64_t inner = 0u; + uint64_t rows = 0u; +}; + +struct prepare_f16_request { + std::span source = {}; + std::span destination = {}; + uint64_t inner = 0u; + uint64_t rows = 0u; +}; + +struct execute_request { + std::span weights = {}; + std::span input = {}; + std::span output = {}; + uint64_t inner = 0u; + uint64_t rows = 0u; +}; + +struct prepare_f32 { + prepare_f32(const prepare_f32_request &request_ref, + dispatch_result &result_ref) noexcept + : request(request_ref), result(result_ref) {} + + const prepare_f32_request &request; + dispatch_result &result; + emel::callback &)> on_done = {}; + emel::callback &)> on_error = + {}; +}; + +struct prepare_f16 { + prepare_f16(const prepare_f16_request &request_ref, + dispatch_result &result_ref) noexcept + : request(request_ref), result(result_ref) {} + + const prepare_f16_request &request; + dispatch_result &result; + emel::callback &)> on_done = {}; + emel::callback &)> on_error = + {}; +}; + +struct execute_reference { + execute_reference(const execute_request &request_ref, + dispatch_result &result_ref) noexcept + : request(request_ref), result(result_ref) {} + + const execute_request &request; + dispatch_result &result; + emel::callback &)> + on_done = {}; + emel::callback &)> + on_error = {}; +}; + +struct execute_exact_x4 { + execute_exact_x4(const execute_request &request_ref, + dispatch_result &result_ref) noexcept + : request(request_ref), result(result_ref) {} + + const execute_request &request; + dispatch_result &result; + emel::callback &)> + on_done = {}; + emel::callback &)> + on_error = {}; +}; + +struct capture_diagnostics { + capture_diagnostics(diagnostics &out_ref, + dispatch_result &result_ref) noexcept + : out(out_ref), result(result_ref) {} + + diagnostics &out; + dispatch_result &result; +}; + +} // namespace emel::kernel::f32_matvec::event diff --git a/src/emel/kernel/f32_matvec/guards.hpp b/src/emel/kernel/f32_matvec/guards.hpp new file mode 100644 index 00000000..43ba8e12 --- /dev/null +++ b/src/emel/kernel/f32_matvec/guards.hpp @@ -0,0 +1,246 @@ +#pragma once + +#include +#include +#include +#include + +#include "emel/kernel/f32_matvec/context.hpp" +#include "emel/kernel/f32_matvec/events.hpp" + +namespace emel::kernel::f32_matvec::guard { + +inline bool guard_element_count_valid(const uint64_t inner, const uint64_t rows, + uint64_t &elements) noexcept { + if (inner == 0u || rows == 0u || + inner > std::numeric_limits::max() / rows) { + return false; + } + elements = inner * rows; + return elements <= std::numeric_limits::max() / sizeof(float) && + elements <= std::numeric_limits::max() / sizeof(float); +} + +inline bool guard_ranges_disjoint(const void *first, const uint64_t first_bytes, + const void *second, + const uint64_t second_bytes) noexcept { + const auto first_begin = reinterpret_cast(first); + const auto second_begin = reinterpret_cast(second); + if (first_begin > std::numeric_limits::max() - first_bytes || + second_begin > std::numeric_limits::max() - second_bytes) { + return false; + } + const uintptr_t first_end = first_begin + first_bytes; + const uintptr_t second_end = second_begin + second_bytes; + return first_end <= second_begin || second_end <= first_begin; +} + +template +inline bool guard_spans_disjoint(const std::span source, + const std::span destination, + const uint64_t elements) noexcept { + if (elements > std::numeric_limits::max() / sizeof(source_type) || + elements > std::numeric_limits::max() / sizeof(float)) { + return false; + } + return guard_ranges_disjoint(source.data(), elements * sizeof(source_type), + destination.data(), elements * sizeof(float)); +} + +template +inline bool guard_prepare_request_valid(const request_type &request) noexcept { + uint64_t elements = 0u; + if (!guard_element_count_valid(request.inner, request.rows, elements) || + request.source.data() == nullptr || + request.destination.data() == nullptr || + request.source.size() < elements || + request.destination.size() < elements) { + return false; + } + return reinterpret_cast(request.source.data()) % + alignof(source_type) == + 0u && + reinterpret_cast(request.destination.data()) % + alignof(float) == + 0u && + guard_spans_disjoint(request.source, request.destination, + elements); +} + +inline bool +guard_execute_request_valid(const event::execute_request &request) noexcept { + uint64_t elements = 0u; + if (!guard_element_count_valid(request.inner, request.rows, elements) || + request.weights.data() == nullptr || request.input.data() == nullptr || + request.output.data() == nullptr || request.weights.size() < elements || + request.input.size() < request.inner || + request.output.size() < request.rows || + reinterpret_cast(request.weights.data()) % alignof(float) != + 0u || + reinterpret_cast(request.input.data()) % alignof(float) != + 0u || + reinterpret_cast(request.output.data()) % alignof(float) != + 0u) { + return false; + } + const uint64_t weight_bytes = elements * sizeof(float); + const uint64_t input_bytes = request.inner * sizeof(float); + const uint64_t output_bytes = request.rows * sizeof(float); + return guard_ranges_disjoint(request.weights.data(), weight_bytes, + request.output.data(), output_bytes) && + guard_ranges_disjoint(request.input.data(), input_bytes, + request.output.data(), output_bytes); +} + +struct guard_prepare_f32_supported { + bool operator()(const event::prepare_f32 &ev, + const action::context &) const noexcept { + return guard_prepare_request_valid(ev.request); + } +}; + +struct guard_prepare_f32_unsupported { + bool operator()(const event::prepare_f32 &ev, + const action::context &ctx) const noexcept { + return !guard_prepare_f32_supported{}(ev, ctx); + } +}; + +struct guard_prepare_f16_supported { + bool operator()(const event::prepare_f16 &ev, + const action::context &) const noexcept { + return guard_prepare_request_valid(ev.request); + } +}; + +struct guard_prepare_f16_unsupported { + bool operator()(const event::prepare_f16 &ev, + const action::context &ctx) const noexcept { + return !guard_prepare_f16_supported{}(ev, ctx); + } +}; + +struct guard_execute_reference_supported { + bool operator()(const event::execute_reference &ev, + const action::context &) const noexcept { + return guard_execute_request_valid(ev.request); + } +}; + +struct guard_execute_reference_unsupported { + bool operator()(const event::execute_reference &ev, + const action::context &ctx) const noexcept { + return !guard_execute_reference_supported{}(ev, ctx); + } +}; + +struct guard_execute_exact_x4_supported { + bool operator()(const event::execute_exact_x4 &ev, + const action::context &) const noexcept { +#if defined(__aarch64__) || defined(_M_ARM64) + return guard_execute_request_valid(ev.request); +#else + (void)ev; + return false; +#endif + } +}; + +struct guard_execute_exact_x4_unsupported { + bool operator()(const event::execute_exact_x4 &ev, + const action::context &ctx) const noexcept { + return !guard_execute_exact_x4_supported{}(ev, ctx); + } +}; + +template struct guard_dispatch_succeeded { + bool operator()(const event_type &ev, + const action::context &) const noexcept { + return ev.result.accepted; + } +}; + +template struct guard_dispatch_failed { + bool operator()(const event_type &ev, + const action::context &) const noexcept { + return !ev.result.accepted; + } +}; + +template struct guard_has_done_callback { + bool operator()(const event_type &ev, + const action::context &) const noexcept { + return static_cast(ev.on_done); + } +}; + +template struct guard_no_done_callback { + bool operator()(const event_type &ev, + const action::context &ctx) const noexcept { + return !guard_has_done_callback{}(ev, ctx); + } +}; + +template struct guard_has_error_callback { + bool operator()(const event_type &ev, + const action::context &) const noexcept { + return static_cast(ev.on_error); + } +}; + +template struct guard_no_error_callback { + bool operator()(const event_type &ev, + const action::context &ctx) const noexcept { + return !guard_has_error_callback{}(ev, ctx); + } +}; + +struct guard_prepare_f32_succeeded + : guard_dispatch_succeeded {}; +struct guard_prepare_f32_failed : guard_dispatch_failed {}; +struct guard_prepare_f16_succeeded + : guard_dispatch_succeeded {}; +struct guard_prepare_f16_failed : guard_dispatch_failed {}; +struct guard_execute_reference_succeeded + : guard_dispatch_succeeded {}; +struct guard_execute_reference_failed + : guard_dispatch_failed {}; +struct guard_execute_exact_x4_succeeded + : guard_dispatch_succeeded {}; +struct guard_execute_exact_x4_failed + : guard_dispatch_failed {}; + +struct guard_prepare_f32_has_done_callback + : guard_has_done_callback {}; +struct guard_prepare_f32_no_done_callback + : guard_no_done_callback {}; +struct guard_prepare_f32_has_error_callback + : guard_has_error_callback {}; +struct guard_prepare_f32_no_error_callback + : guard_no_error_callback {}; +struct guard_prepare_f16_has_done_callback + : guard_has_done_callback {}; +struct guard_prepare_f16_no_done_callback + : guard_no_done_callback {}; +struct guard_prepare_f16_has_error_callback + : guard_has_error_callback {}; +struct guard_prepare_f16_no_error_callback + : guard_no_error_callback {}; +struct guard_execute_reference_has_done_callback + : guard_has_done_callback {}; +struct guard_execute_reference_no_done_callback + : guard_no_done_callback {}; +struct guard_execute_reference_has_error_callback + : guard_has_error_callback {}; +struct guard_execute_reference_no_error_callback + : guard_no_error_callback {}; +struct guard_execute_exact_x4_has_done_callback + : guard_has_done_callback {}; +struct guard_execute_exact_x4_no_done_callback + : guard_no_done_callback {}; +struct guard_execute_exact_x4_has_error_callback + : guard_has_error_callback {}; +struct guard_execute_exact_x4_no_error_callback + : guard_no_error_callback {}; + +} // namespace emel::kernel::f32_matvec::guard diff --git a/src/emel/kernel/f32_matvec/sm.hpp b/src/emel/kernel/f32_matvec/sm.hpp new file mode 100644 index 00000000..f1aba934 --- /dev/null +++ b/src/emel/kernel/f32_matvec/sm.hpp @@ -0,0 +1,234 @@ +#pragma once +// benchmark: designed + +#include "emel/kernel/f32_matvec/actions.hpp" +#include "emel/kernel/f32_matvec/guards.hpp" +#include "emel/sm.hpp" + +namespace emel::kernel::f32_matvec { + +struct state_ready {}; +struct state_prepare_f32_result_decision {}; +struct state_prepare_f16_result_decision {}; +struct state_reference_result_decision {}; +struct state_exact_x4_result_decision {}; +struct state_done_callback_decision {}; +struct state_error_callback_decision {}; +struct state_done {}; +struct state_errored {}; + +struct model { + auto operator()() const { + namespace sml = stateforward::sml; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Initialization-only four-row packing. + sml::state <= *sml::state + + sml::event + [ guard::guard_prepare_f32_supported{} ] + / action::effect_prepare_f32{} + , sml::state <= sml::state + + sml::event + [ guard::guard_prepare_f32_unsupported{} ] + / action::effect_reject_prepare_f32{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_prepare_f32_succeeded{} ] + / action::effect_accept_prepare_f32{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_prepare_f32_failed{} ] + / action::effect_reject_prepare_f32{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_prepare_f16_supported{} ] + / action::effect_prepare_f16{} + , sml::state <= sml::state + + sml::event + [ guard::guard_prepare_f16_unsupported{} ] + / action::effect_reject_prepare_f16{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_prepare_f16_succeeded{} ] + / action::effect_accept_prepare_f16{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_prepare_f16_failed{} ] + / action::effect_reject_prepare_f16{} + + //------------------------------------------------------------------------------// + // Same-binary oracle and AArch64 exact x4 inference. + , sml::state <= sml::state + + sml::event + [ guard::guard_execute_reference_supported{} ] + / action::effect_execute_reference{} + , sml::state <= sml::state + + sml::event + [ guard::guard_execute_reference_unsupported{} ] + / action::effect_reject_execute_reference{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_execute_reference_succeeded{} ] + / action::effect_accept_execute_reference{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_execute_reference_failed{} ] + / action::effect_reject_execute_reference{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_execute_exact_x4_supported{} ] + / action::effect_execute_exact_x4{} + , sml::state <= sml::state + + sml::event + [ guard::guard_execute_exact_x4_unsupported{} ] + / action::effect_reject_execute_exact_x4{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_execute_exact_x4_succeeded{} ] + / action::effect_accept_execute_exact_x4{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_execute_exact_x4_failed{} ] + / action::effect_reject_execute_exact_x4{} + + //------------------------------------------------------------------------------// + // Publish explicit same-RTC outcomes. + , sml::state <= sml::state + + sml::completion + [ guard::guard_prepare_f32_has_done_callback{} ] + / action::effect_emit_prepare_f32_done{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_prepare_f32_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion + [ guard::guard_prepare_f32_has_error_callback{} ] + / action::effect_emit_prepare_f32_error{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_prepare_f32_no_error_callback{} ] + + , sml::state <= sml::state + + sml::completion + [ guard::guard_prepare_f16_has_done_callback{} ] + / action::effect_emit_prepare_f16_done{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_prepare_f16_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion + [ guard::guard_prepare_f16_has_error_callback{} ] + / action::effect_emit_prepare_f16_error{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_prepare_f16_no_error_callback{} ] + + , sml::state <= sml::state + + sml::completion + [ guard::guard_execute_reference_has_done_callback{} ] + / action::effect_emit_execute_reference_done{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_execute_reference_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion + [ guard::guard_execute_reference_has_error_callback{} ] + / action::effect_emit_execute_reference_error{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_execute_reference_no_error_callback{} ] + + , sml::state <= sml::state + + sml::completion + [ guard::guard_execute_exact_x4_has_done_callback{} ] + / action::effect_emit_execute_exact_x4_done{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_execute_exact_x4_no_done_callback{} ] + , sml::state <= sml::state + + sml::completion + [ guard::guard_execute_exact_x4_has_error_callback{} ] + / action::effect_emit_execute_exact_x4_error{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_execute_exact_x4_no_error_callback{} ] + + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion + + //------------------------------------------------------------------------------// + // Diagnostics and unexpected events. + , sml::state <= sml::state + + sml::event + / action::effect_capture_diagnostics{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected{} + ); + // clang-format on + } +}; + +struct sm : public emel::sm { + using base_type = emel::sm; + using base_type::is; + using base_type::visit_current_states; + + sm() = default; + sm(const sm &) = delete; + sm &operator=(const sm &) = delete; + sm(sm &&) = delete; + sm &operator=(sm &&) = delete; + + template bool process_event(const event_type &ev) { + if constexpr (requires { ev.result.accepted; }) { + ev.result.accepted = false; + } + const bool handled = base_type::process_event(ev); + if constexpr (requires { ev.result.accepted; }) { + return handled && ev.result.accepted; + } else { + return false; + } + } +}; + +} // namespace emel::kernel::f32_matvec diff --git a/src/emel/kernel/matmul/actions.hpp b/src/emel/kernel/matmul/actions.hpp new file mode 100644 index 00000000..ae3b13d1 --- /dev/null +++ b/src/emel/kernel/matmul/actions.hpp @@ -0,0 +1,305 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "emel/kernel/matmul/context.hpp" +#include "emel/kernel/matmul/events.hpp" + +namespace emel::kernel::matmul::action { + +struct lane_dispatch { + emel::kernel::sm *kernel = nullptr; + const emel::kernel::event::op_mul_mat *request = nullptr; + bool accepted = false; +}; + +template +inline void configure_lane_kinds(context &ctx, + std::index_sequence) noexcept { + const emel::kernel::event::configure_kind configure{ctx.kernel_kind}; + (ctx.lanes->kernels[lanes].process_event(configure), ...); +} + +struct effect_configure_kernel_kind { + void operator()(const event::configure_kernel_kind &ev, + context &ctx) const noexcept { + ctx.kernel_kind = ev.kind; + ctx.kernel.process_event( + emel::kernel::event::configure_kind{ctx.kernel_kind}); + configure_lane_kinds(ctx, std::make_index_sequence{}); + } +}; + +struct effect_execute_serial { + void operator()(const event::execute_serial &ev, + context &ctx) const noexcept { + ev.result = {}; + ev.result.lane_count = 1u; + ev.result.all_submitted = true; + ev.result.all_lanes_accepted = ctx.kernel.process_event(ev.request); + } +}; + +template +inline detail::matmul_row_slice +compute_fixed_row_slice(const uint64_t rows) noexcept { + const uint64_t groups = (rows + group_rows - 1u) / group_rows; + const uint64_t groups_per_lane = groups / lane_count; + const uint64_t extra_groups = groups % lane_count; + const uint64_t begin_group = + lane * groups_per_lane + std::min(lane, extra_groups); + const uint64_t lane_groups = + groups_per_lane + static_cast(lane < extra_groups); + const uint64_t begin_row = begin_group * group_rows; + const uint64_t end_row = + std::min(rows, (begin_group + lane_groups) * group_rows); + return detail::matmul_row_slice{ + .row_begin = static_cast(begin_row), + .row_count = static_cast(end_row - begin_row), + }; +} + +template +inline void prepare_fixed_lanes( + const event::execute_parallel &ev, context &ctx, + std::array &row_slices, + std::array &lane_events, + std::array &lane_dispatches, + std::index_sequence) noexcept { + ((row_slices[lanes] = compute_fixed_row_slice( + ev.request.src0.ne[1]), + lane_events[lanes] = detail::compute_sliced_mul_mat_event( + ev.request, group_rows, row_slices[lanes]), + lane_dispatches[lanes] = + lane_dispatch{ + .kernel = &ctx.lanes->kernels[lanes], + .request = &lane_events[lanes], + }), + ...); +} + +template +inline size_t submit_fixed_worker_lanes( + context &ctx, lane_pool::join_group &group, + std::array &lane_dispatches, + std::index_sequence) noexcept { + ((lane_dispatches[lane_offsets + 1u].accepted = false), ...); + return ctx.parallel_matmul_lanes->try_submit_batch( + group, ([&dispatch = lane_dispatches[lane_offsets + 1u]]() noexcept { + dispatch.accepted = dispatch.kernel->process_event(*dispatch.request); + })...); +} + +template +inline bool fixed_worker_lanes_accepted( + const context &ctx, + const std::array &lane_dispatches, + std::index_sequence) noexcept { + (void)ctx; + return (lane_dispatches[lane_offsets + 1u].accepted && ...); +} + +template +struct effect_execute_parallel { + void operator()(const event::execute_parallel &ev, + context &ctx) const noexcept { + static_assert(lane_count == 2u || lane_count == 4u || lane_count == 8u); + ev.result = {}; + ev.result.lane_count = lane_count; + std::array row_slices = {}; + std::array lane_events = {}; + std::array lane_dispatches = {}; + prepare_fixed_lanes( + ev, ctx, row_slices, lane_events, lane_dispatches, + std::make_index_sequence{}); + + lane_pool::join_group group{}; + ev.result.submitted_worker_lanes = submit_fixed_worker_lanes( + ctx, group, lane_dispatches, + std::make_index_sequence{}); + ev.result.all_submitted = + ev.result.submitted_worker_lanes == lane_count - 1u; + const bool owner_accepted = + ctx.lanes->kernels[0].process_event(lane_events[0]); + (void)group.wait(); + ev.result.drained_worker_lanes = ev.result.submitted_worker_lanes; + ev.result.all_lanes_accepted = + owner_accepted && + fixed_worker_lanes_accepted( + ctx, lane_dispatches, std::make_index_sequence{}); + } +}; + +struct effect_accept_serial_execution { + void operator()(const event::execute_serial &ev, context &) const noexcept { + ev.accepted = true; + } +}; + +struct effect_accept_parallel_execution { + void operator()(const event::execute_parallel &ev, context &) const noexcept { + ev.accepted = true; + } +}; + +struct effect_reject_serial_execution { + void operator()(const event::execute_serial &ev, context &) const noexcept { + ev.accepted = false; + } +}; + +struct effect_reject_parallel_execution { + void operator()(const event::execute_parallel &ev, context &) const noexcept { + ev.accepted = false; + } +}; + +template struct effect_emit_execute_done { + void operator()(const event_type &ev, context &) const noexcept { + ev.on_done(events::execute_done{.request = ev}); + } +}; + +template struct effect_emit_execute_error { + void operator()(const event_type &ev, context &) const noexcept { + ev.on_error(events::execute_error{.request = ev}); + } +}; + +struct effect_emit_serial_done + : effect_emit_execute_done {}; +struct effect_emit_serial_error + : effect_emit_execute_error {}; +struct effect_emit_parallel_done + : effect_emit_execute_done {}; +struct effect_emit_parallel_error + : effect_emit_execute_error {}; + +struct effect_on_unexpected { + template + void operator()(const event_type &ev, context &) const noexcept { + if constexpr (requires { ev.accepted; }) { + ev.accepted = false; + } + } +}; + +template +inline bool capture_lane_diagnostics( + context &ctx, + std::array &out, + std::index_sequence) noexcept { + return (ctx.lanes->kernels[lanes].process_event( + emel::kernel::event::capture_diagnostics{out[lanes]}) && + ...); +} + +template +inline uint64_t +compute_diagnostics_total(const emel::kernel::event::diagnostics &serial, + const std::array ¶llel, + member_type emel::kernel::event::diagnostics::*member, + std::index_sequence) noexcept { + return serial.*member + ((parallel[lanes].*member) + ... + 0u); +} + +struct effect_capture_diagnostics { + void operator()(const event::capture_diagnostics &ev, + context &ctx) const noexcept { + ev.out = {}; + emel::kernel::event::diagnostics serial = {}; + std::array parallel = + {}; + const bool serial_accepted = ctx.kernel.process_event( + emel::kernel::event::capture_diagnostics{serial}); + const bool lanes_accepted = capture_lane_diagnostics( + ctx, parallel, std::make_index_sequence{}); + const auto total = [&serial, ¶llel](auto member) noexcept { + return compute_diagnostics_total( + serial, parallel, member, + std::make_index_sequence{}); + }; + ev.out.optimized_flash_dispatch_calls = total( + &emel::kernel::event::diagnostics::optimized_flash_dispatch_calls); + ev.out.shared_flash_dispatch_calls = + total(&emel::kernel::event::diagnostics::shared_flash_dispatch_calls); + ev.out.optimized_q2_dispatch_calls = + total(&emel::kernel::event::diagnostics::optimized_q2_dispatch_calls); + ev.out.shared_q2_dispatch_calls = + total(&emel::kernel::event::diagnostics::shared_q2_dispatch_calls); + ev.out.optimized_q3_dispatch_calls = + total(&emel::kernel::event::diagnostics::optimized_q3_dispatch_calls); + ev.out.shared_q3_dispatch_calls = + total(&emel::kernel::event::diagnostics::shared_q3_dispatch_calls); + ev.out.optimized_q4_dispatch_calls = + total(&emel::kernel::event::diagnostics::optimized_q4_dispatch_calls); + ev.out.optimized_q4_vector_dispatch_calls = total( + &emel::kernel::event::diagnostics::optimized_q4_vector_dispatch_calls); + ev.out.optimized_q4_vector_packed_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q4_vector_packed_dispatch_calls); + ev.out.optimized_q4_vector_packed_q8_rhs_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q4_vector_packed_q8_rhs_dispatch_calls); + ev.out.shared_q4_dispatch_calls = + total(&emel::kernel::event::diagnostics::shared_q4_dispatch_calls); + ev.out.optimized_q6_dispatch_calls = + total(&emel::kernel::event::diagnostics::optimized_q6_dispatch_calls); + ev.out.optimized_q6_vector_dispatch_calls = total( + &emel::kernel::event::diagnostics::optimized_q6_vector_dispatch_calls); + ev.out.optimized_q6_vector_argmax_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q6_vector_argmax_dispatch_calls); + ev.out.optimized_q6_vector_packed_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q6_vector_packed_dispatch_calls); + ev.out.optimized_q6_vector_packed_q8_rhs_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q6_vector_packed_q8_rhs_dispatch_calls); + ev.out.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls); + ev.out.optimized_q6_vector_prepared_q8_rhs_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q6_vector_prepared_q8_rhs_dispatch_calls); + ev.out.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls); + ev.out.optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls = + total( + &emel::kernel::event::diagnostics:: + optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls); + ev.out.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls = + total(&emel::kernel::event::diagnostics:: + optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls); + ev.out.shared_q6_dispatch_calls = + total(&emel::kernel::event::diagnostics::shared_q6_dispatch_calls); + ev.out.serial_optimized_q4_dispatch_calls = + serial.optimized_q4_dispatch_calls; + ev.out.parallel_optimized_q4_dispatch_calls = + total(&emel::kernel::event::diagnostics::optimized_q4_dispatch_calls) - + serial.optimized_q4_dispatch_calls; + ev.accepted = serial_accepted && lanes_accepted; + } +}; + +inline constexpr effect_configure_kernel_kind effect_configure_kernel_kind{}; +inline constexpr effect_execute_serial effect_execute_serial{}; +inline constexpr effect_accept_serial_execution + effect_accept_serial_execution{}; +inline constexpr effect_accept_parallel_execution + effect_accept_parallel_execution{}; +inline constexpr effect_reject_serial_execution + effect_reject_serial_execution{}; +inline constexpr effect_reject_parallel_execution + effect_reject_parallel_execution{}; +inline constexpr effect_capture_diagnostics effect_capture_diagnostics{}; +inline constexpr effect_on_unexpected effect_on_unexpected{}; + +} // namespace emel::kernel::matmul::action diff --git a/src/emel/kernel/matmul/context.hpp b/src/emel/kernel/matmul/context.hpp new file mode 100644 index 00000000..7d1ba205 --- /dev/null +++ b/src/emel/kernel/matmul/context.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "emel/kernel/matmul/detail.hpp" +#include "emel/kernel/sm.hpp" + +namespace emel::kernel::matmul::action { + +inline constexpr size_t MAX_PARALLEL_LANES = 8u; + +struct lane_storage { + explicit lane_storage(const emel::kernel::kernel_kind kind) noexcept + : kernels{emel::kernel::sm{kind}, emel::kernel::sm{kind}, + emel::kernel::sm{kind}, emel::kernel::sm{kind}, + emel::kernel::sm{kind}, emel::kernel::sm{kind}, + emel::kernel::sm{kind}, emel::kernel::sm{kind}} {} + + std::array kernels; +}; + +struct context { + context() noexcept + : context(execution_policy{ + .parallel_matmul_lanes = nullptr, + .kernel_kind = emel::kernel::detect_host_kind(), + .active_lanes = 1u, + .mode = lane_mode::serial, + }) {} + + explicit context(const execution_policy &policy) noexcept + : parallel_matmul_lanes(policy.parallel_matmul_lanes), + kernel_kind(policy.kernel_kind), active_lanes(policy.active_lanes), + kernel(policy.kernel_kind), + lanes(new (std::nothrow) lane_storage{policy.kernel_kind}) { + // Lane actors are large (~67 KiB each). One construction-time allocation + // keeps the owning actor portable on platforms with small thread stacks; + // dispatch reuses this stable storage and never allocates. + if (lanes == nullptr) { + std::terminate(); + } + } + + lane_pool *parallel_matmul_lanes = nullptr; + emel::kernel::kernel_kind kernel_kind = emel::kernel::kernel_kind::x86_64; + size_t active_lanes = 1u; + emel::kernel::sm kernel = {}; + std::unique_ptr lanes = {}; +}; + +} // namespace emel::kernel::matmul::action diff --git a/src/emel/kernel/matmul/detail.hpp b/src/emel/kernel/matmul/detail.hpp new file mode 100644 index 00000000..a6cf95d6 --- /dev/null +++ b/src/emel/kernel/matmul/detail.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include + +#include "emel/kernel/detail.hpp" +#include "emel/kernel/events.hpp" +#include "emel/sm.hpp" + +namespace emel::kernel::matmul { + +using lane_pool = emel::policy::fork_join_lane_pool<7u, 128u, 1048576u>; + +enum class lane_mode : uint8_t { + serial = 0, + parallel = 1, +}; + +struct execution_policy { + lane_pool *parallel_matmul_lanes; + emel::kernel::kernel_kind kernel_kind; + size_t active_lanes; + lane_mode mode = lane_mode::serial; +}; + +namespace detail { + +// A row slice is a disjoint destination interval. Packed operands require +// row_begin to remain aligned to their physical row group. +struct matmul_row_slice { + int32_t row_begin = 0; + int32_t row_count = 0; +}; + +inline emel::kernel::event::op_mul_mat +compute_sliced_mul_mat_event(const emel::kernel::event::op_mul_mat &ev, + const uint64_t group_rows, + const matmul_row_slice slice) noexcept { + emel::kernel::event::op_mul_mat sliced = ev; + const uint64_t begin = static_cast(slice.row_begin); + const uint64_t count = static_cast(slice.row_count); + const uint64_t slice_groups = (count + group_rows - 1u) / group_rows; + sliced.src0.data = static_cast(ev.src0.data) + + (begin / group_rows) * ev.src0.nb[1]; + sliced.src0.ne[1] = count; + sliced.src0.nb[2] = ev.src0.nb[1] * slice_groups; + sliced.src0.nb[3] = sliced.src0.nb[2]; + sliced.dst.data = static_cast(ev.dst.data) + begin * ev.dst.nb[1]; + sliced.dst.ne[1] = count; + sliced.dst.nb[2] = ev.dst.nb[1] * count; + sliced.dst.nb[3] = sliced.dst.nb[2]; + return sliced; +} + +} // namespace detail + +} // namespace emel::kernel::matmul diff --git a/src/emel/kernel/matmul/events.hpp b/src/emel/kernel/matmul/events.hpp new file mode 100644 index 00000000..839cc659 --- /dev/null +++ b/src/emel/kernel/matmul/events.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include + +#include "emel/callback.hpp" +#include "emel/kernel/events.hpp" +#include "emel/kernel/matmul/detail.hpp" +#include "emel/kernel/sm.hpp" + +namespace emel::kernel::matmul::events { + +template struct execute_done { + const request_type &request; +}; + +template struct execute_error { + const request_type &request; +}; + +} // namespace emel::kernel::matmul::events + +namespace emel::kernel::matmul::event { + +struct configure_kernel_kind { + emel::kernel::kernel_kind kind = emel::kernel::kernel_kind::x86_64; +}; + +struct dispatch_result { + size_t lane_count = 0u; + size_t submitted_worker_lanes = 0u; + size_t drained_worker_lanes = 0u; + bool all_submitted = false; + bool all_lanes_accepted = false; +}; + +struct diagnostics : emel::kernel::event::diagnostics { + uint64_t serial_optimized_q4_dispatch_calls = 0u; + uint64_t parallel_optimized_q4_dispatch_calls = 0u; +}; + +struct capture_diagnostics { + capture_diagnostics(diagnostics &out_ref, bool &accepted_ref) noexcept + : out(out_ref), accepted(accepted_ref) {} + + diagnostics &out; + bool &accepted; +}; + +struct execute_serial { + execute_serial(const emel::kernel::event::op_mul_mat &request_ref, + dispatch_result &result_ref, bool &accepted_ref) noexcept + : request(request_ref), result(result_ref), accepted(accepted_ref) {} + + const emel::kernel::event::op_mul_mat &request; + dispatch_result &result; + bool &accepted; + emel::callback &)> on_done = + {}; + emel::callback &)> on_error = + {}; +}; + +struct execute_parallel { + execute_parallel(const emel::kernel::event::op_mul_mat &request_ref, + dispatch_result &result_ref, bool &accepted_ref) noexcept + : request(request_ref), result(result_ref), accepted(accepted_ref) {} + + const emel::kernel::event::op_mul_mat &request; + dispatch_result &result; + bool &accepted; + emel::callback &)> on_done = + {}; + emel::callback &)> + on_error = {}; +}; + +} // namespace emel::kernel::matmul::event diff --git a/src/emel/kernel/matmul/guards.hpp b/src/emel/kernel/matmul/guards.hpp new file mode 100644 index 00000000..e9e09219 --- /dev/null +++ b/src/emel/kernel/matmul/guards.hpp @@ -0,0 +1,281 @@ +#pragma once + +#include +#include +#include + +#include "emel/kernel/matmul/context.hpp" +#include "emel/kernel/matmul/events.hpp" + +namespace emel::kernel::matmul::guard { + +inline bool guard_supported_lane_count(const size_t active_lanes) noexcept { + return active_lanes == 2u || active_lanes == 4u || active_lanes == 8u; +} + +inline lane_mode guard_policy_lane_mode(const size_t active_lanes) noexcept { + return guard_supported_lane_count(active_lanes) ? lane_mode::parallel + : lane_mode::serial; +} + +} // namespace emel::kernel::matmul::guard + +namespace emel::kernel::matmul { + +inline execution_policy +make_execution_policy(lane_pool ¶llel_matmul_lanes, + const emel::kernel::kernel_kind kernel_kind, + const size_t active_lanes) noexcept { + return execution_policy{ + .parallel_matmul_lanes = ¶llel_matmul_lanes, + .kernel_kind = kernel_kind, + .active_lanes = active_lanes, + .mode = guard::guard_policy_lane_mode(active_lanes), + }; +} + +inline execution_policy +make_auto_execution_policy(lane_pool ¶llel_matmul_lanes) noexcept { + return make_execution_policy(parallel_matmul_lanes, + emel::kernel::detect_host_kind(), 8u); +} + +} // namespace emel::kernel::matmul + +namespace emel::kernel::matmul::guard { + +inline bool +supports_parallel_execution(const execution_policy &policy) noexcept { + const bool supported_lane_count = + guard_supported_lane_count(policy.active_lanes); + return policy.mode == lane_mode::parallel && + policy.parallel_matmul_lanes != nullptr && supported_lane_count; +} + +inline bool guard_lane_storage_ready(const action::context &ctx) noexcept { + const bool supported_lane_count = + guard_supported_lane_count(ctx.active_lanes); + return ctx.parallel_matmul_lanes != nullptr && supported_lane_count && + ctx.lanes != nullptr; +} + +inline uint64_t guard_row_group_count(const event::execute_parallel &ev, + const uint64_t group_rows) noexcept { + const uint64_t rows = ev.request.src0.ne[1]; + return rows / group_rows + static_cast((rows % group_rows) != 0u); +} + +inline bool +guard_uses_x8_row_groups(const emel::kernel::event::dtype type) noexcept { + const uint8_t code = emel::kernel::detail::dtype_code(type); + return code == emel::kernel::detail::dtype_q4_k_x8_bl4 || + code == emel::kernel::detail::dtype_q4_k_x8_bl8 || + code == emel::kernel::detail::dtype_q6_k_x8 || + code == emel::kernel::detail::dtype_q6_k_x8_q8_prepared || + code == emel::kernel::detail::dtype_q6_k_x8_q8_argmax_prepared; +} + +inline bool +guard_uses_x4_row_groups(const emel::kernel::event::dtype type) noexcept { + const uint8_t code = emel::kernel::detail::dtype_code(type); + return code == emel::kernel::detail::dtype_q8_0_x4_bl4 || + code == emel::kernel::detail::dtype_q8_0_x4_bl8; +} + +inline bool guard_pointer_extent_valid(const void *data, const uint64_t stride, + const uint64_t count) noexcept { + if (data == nullptr || stride == 0u || count == 0u || + stride > static_cast(std::numeric_limits::max()) / + count) { + return false; + } + const uint64_t extent = stride * count; + const uintptr_t begin = reinterpret_cast(data); + return extent <= std::numeric_limits::max() - begin; +} + +inline bool +guard_parallel_request_valid(const event::execute_parallel &ev) noexcept { + const auto &request = ev.request; + const uint64_t rows = request.src0.ne[1]; + const uint64_t group_rows = guard_uses_x8_row_groups(request.src0.type) + ? emel::kernel::detail::quant::Q4_K_X8_ROWS + : guard_uses_x4_row_groups(request.src0.type) + ? emel::kernel::detail::quant::Q8_0_X4_ROWS + : 1u; + const uint64_t storage_rows = + rows / group_rows + static_cast((rows % group_rows) != 0u); + return rows > 0u && + rows <= static_cast(std::numeric_limits::max()) && + request.src0.data != nullptr && request.src1.data != nullptr && + request.dst.data != nullptr && + request.src0.type != emel::kernel::event::dtype::unknown && + request.src1.type != emel::kernel::event::dtype::unknown && + request.dst.type != emel::kernel::event::dtype::unknown && + request.src0.ne[0] > 0u && request.src1.ne[0] > 0u && + request.src1.ne[1] == request.src0.ne[0] && + request.dst.ne[0] == request.src1.ne[0] && request.dst.ne[1] == rows && + request.src0.ne[2] == 1u && request.src0.ne[3] == 1u && + request.src1.ne[2] == 1u && request.src1.ne[3] == 1u && + request.dst.ne[2] == 1u && request.dst.ne[3] == 1u && + request.op_params_size <= request.op_params.size() && + guard_pointer_extent_valid(request.src0.data, request.src0.nb[1], + storage_rows) && + guard_pointer_extent_valid(request.src1.data, request.src1.nb[1], + request.src1.ne[1]) && + guard_pointer_extent_valid(request.dst.data, request.dst.nb[1], rows); +} + +inline bool guard_parallel_storage_ready(const event::execute_parallel &ev, + const action::context &ctx) noexcept { + return guard_lane_storage_ready(ctx) && guard_parallel_request_valid(ev); +} + +struct guard_parallel_request_invalid { + bool operator()(const event::execute_parallel &ev, + const action::context &) const noexcept { + return !guard_parallel_request_valid(ev); + } +}; + +struct guard_parallel_unavailable { + bool operator()(const event::execute_parallel &, + const action::context &ctx) const noexcept { + return ctx.parallel_matmul_lanes == nullptr || + !guard_lane_storage_ready(ctx); + } +}; + +template +inline bool guard_lane_count_selected(const event::execute_parallel &ev, + const action::context &ctx, + const uint64_t group_rows) noexcept { + const uint64_t groups = guard_row_group_count(ev, group_rows); + if constexpr (lane_count == 8u) { + return ctx.active_lanes == 8u && groups >= 8u; + } else if constexpr (lane_count == 4u) { + return groups >= 4u && + (ctx.active_lanes == 4u || (ctx.active_lanes == 8u && groups < 8u)); + } else { + return groups >= 2u && + (ctx.active_lanes == 2u || (ctx.active_lanes >= 4u && groups < 4u)); + } +} + +template +inline bool +guard_group_rows_selected(const emel::kernel::event::dtype type) noexcept { + if constexpr (group_rows == emel::kernel::detail::quant::Q4_K_X8_ROWS) { + return guard_uses_x8_row_groups(type); + } else if constexpr (group_rows == + emel::kernel::detail::quant::Q8_0_X4_ROWS) { + return guard_uses_x4_row_groups(type); + } else { + return !guard_uses_x8_row_groups(type) && !guard_uses_x4_row_groups(type); + } +} + +template struct guard_parallel_ready { + bool operator()(const event::execute_parallel &ev, + const action::context &ctx) const noexcept { + return guard_parallel_storage_ready(ev, ctx) && + guard_group_rows_selected(ev.request.src0.type) && + guard_lane_count_selected(ev, ctx, group_rows); + } +}; + +struct guard_parallel_no_lane_count { + bool operator()(const event::execute_parallel &ev, + const action::context &ctx) const noexcept { + const uint64_t group_rows = + guard_uses_x8_row_groups(ev.request.src0.type) + ? emel::kernel::detail::quant::Q4_K_X8_ROWS + : (guard_uses_x4_row_groups(ev.request.src0.type) + ? emel::kernel::detail::quant::Q8_0_X4_ROWS + : 1u); + const uint64_t groups = guard_row_group_count(ev, group_rows); + return guard_parallel_storage_ready(ev, ctx) && groups < 2u; + } +}; + +struct guard_serial_accepted { + bool operator()(const event::execute_serial &ev, + const action::context &) const noexcept { + return ev.result.all_lanes_accepted; + } +}; + +struct guard_serial_rejected { + bool operator()(const event::execute_serial &ev, + const action::context &) const noexcept { + return !ev.result.all_lanes_accepted; + } +}; + +struct guard_parallel_submission_failed { + bool operator()(const event::execute_parallel &ev, + const action::context &) const noexcept { + return !ev.result.all_submitted; + } +}; + +struct guard_parallel_lane_rejected { + bool operator()(const event::execute_parallel &ev, + const action::context &) const noexcept { + return ev.result.all_submitted && !ev.result.all_lanes_accepted; + } +}; + +struct guard_parallel_all_lanes_accepted { + bool operator()(const event::execute_parallel &ev, + const action::context &) const noexcept { + return ev.result.all_submitted && ev.result.all_lanes_accepted; + } +}; + +template struct guard_has_done_callback { + bool operator()(const event_type &ev, + const action::context &) const noexcept { + return static_cast(ev.on_done); + } +}; + +template struct guard_no_done_callback { + bool operator()(const event_type &ev, + const action::context &ctx) const noexcept { + return !guard_has_done_callback{}(ev, ctx); + } +}; + +template struct guard_has_error_callback { + bool operator()(const event_type &ev, + const action::context &) const noexcept { + return static_cast(ev.on_error); + } +}; + +template struct guard_no_error_callback { + bool operator()(const event_type &ev, + const action::context &ctx) const noexcept { + return !guard_has_error_callback{}(ev, ctx); + } +}; + +struct guard_serial_has_done_callback + : guard_has_done_callback {}; +struct guard_serial_no_done_callback + : guard_no_done_callback {}; +struct guard_serial_has_error_callback + : guard_has_error_callback {}; +struct guard_serial_no_error_callback + : guard_no_error_callback {}; +struct guard_parallel_has_done_callback + : guard_has_done_callback {}; +struct guard_parallel_no_done_callback + : guard_no_done_callback {}; +struct guard_parallel_has_error_callback + : guard_has_error_callback {}; +struct guard_parallel_no_error_callback + : guard_no_error_callback {}; + +} // namespace emel::kernel::matmul::guard diff --git a/src/emel/kernel/matmul/sm.hpp b/src/emel/kernel/matmul/sm.hpp new file mode 100644 index 00000000..bd962e52 --- /dev/null +++ b/src/emel/kernel/matmul/sm.hpp @@ -0,0 +1,243 @@ +#pragma once +// benchmark: designed + +#include + +#include "emel/kernel/matmul/actions.hpp" +#include "emel/kernel/matmul/guards.hpp" +#include "emel/sm.hpp" + +namespace emel::kernel::matmul { + +struct state_ready {}; +// These choice states turn kernel and join outcomes into explicit accepted or +// rejected event results without storing per-dispatch status in actor context. +struct state_serial_result_decision {}; +struct state_parallel_result_decision {}; +struct state_done_callback_decision {}; +struct state_error_callback_decision {}; +struct state_done {}; +struct state_errored {}; + +struct model { + auto operator()() const { + namespace sml = stateforward::sml; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Kernel binding. + sml::state <= *sml::state + + sml::event + / action::effect_configure_kernel_kind + + //------------------------------------------------------------------------------// + // Explicit serial versus parallel matmul execution. + , sml::state <= sml::state + + sml::event + / action::effect_execute_serial + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_serial_accepted{} ] + / action::effect_accept_serial_execution + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_serial_rejected{} ] + / action::effect_reject_serial_execution + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<8u, emel::kernel::detail::quant::Q4_K_X8_ROWS>{} ] + / action::effect_execute_parallel<8u, emel::kernel::detail::quant::Q4_K_X8_ROWS>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<8u, emel::kernel::detail::quant::Q8_0_X4_ROWS>{} ] + / action::effect_execute_parallel<8u, emel::kernel::detail::quant::Q8_0_X4_ROWS>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<8u, 1u>{} ] + / action::effect_execute_parallel<8u, 1u>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<4u, emel::kernel::detail::quant::Q4_K_X8_ROWS>{} ] + / action::effect_execute_parallel<4u, emel::kernel::detail::quant::Q4_K_X8_ROWS>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<4u, emel::kernel::detail::quant::Q8_0_X4_ROWS>{} ] + / action::effect_execute_parallel<4u, emel::kernel::detail::quant::Q8_0_X4_ROWS>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<4u, 1u>{} ] + / action::effect_execute_parallel<4u, 1u>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<2u, emel::kernel::detail::quant::Q4_K_X8_ROWS>{} ] + / action::effect_execute_parallel<2u, emel::kernel::detail::quant::Q4_K_X8_ROWS>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<2u, emel::kernel::detail::quant::Q8_0_X4_ROWS>{} ] + / action::effect_execute_parallel<2u, emel::kernel::detail::quant::Q8_0_X4_ROWS>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_ready<2u, 1u>{} ] + / action::effect_execute_parallel<2u, 1u>{} + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_request_invalid{} ] + / action::effect_reject_parallel_execution + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_unavailable{} ] + / action::effect_reject_parallel_execution + + , sml::state <= sml::state + + sml::event + [ guard::guard_parallel_no_lane_count{} ] + / action::effect_reject_parallel_execution + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_parallel_submission_failed{} ] + / action::effect_reject_parallel_execution + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_parallel_lane_rejected{} ] + / action::effect_reject_parallel_execution + + , sml::state <= + sml::state + + sml::completion + [ guard::guard_parallel_all_lanes_accepted{} ] + / action::effect_accept_parallel_execution + + //------------------------------------------------------------------------------// + // Publish explicit same-RTC outcomes. + , sml::state <= sml::state + + sml::completion + [ guard::guard_serial_has_done_callback{} ] + / action::effect_emit_serial_done{} + + , sml::state <= sml::state + + sml::completion + [ guard::guard_serial_no_done_callback{} ] + + , sml::state <= sml::state + + sml::completion + [ guard::guard_serial_has_error_callback{} ] + / action::effect_emit_serial_error{} + + , sml::state <= sml::state + + sml::completion + [ guard::guard_serial_no_error_callback{} ] + + , sml::state <= sml::state + + sml::completion + [ guard::guard_parallel_has_done_callback{} ] + / action::effect_emit_parallel_done{} + + , sml::state <= sml::state + + sml::completion + [ guard::guard_parallel_no_done_callback{} ] + + , sml::state <= sml::state + + sml::completion + [ guard::guard_parallel_has_error_callback{} ] + / action::effect_emit_parallel_error{} + + , sml::state <= sml::state + + sml::completion + [ guard::guard_parallel_no_error_callback{} ] + + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion + + //------------------------------------------------------------------------------// + // Bounded observability through an explicit query event. + , sml::state <= sml::state + + sml::event + / action::effect_capture_diagnostics + + //------------------------------------------------------------------------------// + // Unexpected events. + , sml::state <= sml::state + sml::unexpected_event + / action::effect_on_unexpected + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_on_unexpected + ); + // clang-format on + } +}; + +struct sm : public emel::sm { + using base_type = emel::sm; + using base_type::is; + using base_type::visit_current_states; + + explicit sm(const execution_policy &policy) + : base_type(std::in_place, policy) {} + + sm(const sm &) = delete; + sm &operator=(const sm &) = delete; + sm(sm &&) = delete; + sm &operator=(sm &&) = delete; + + bool process_event(const event::configure_kernel_kind &ev) { + return base_type::process_event(ev); + } + + bool process_event(const event::execute_serial &ev) { + return base_type::process_event(ev); + } + + bool process_event(const event::execute_parallel &ev) { + return base_type::process_event(ev); + } + + bool process_event(const event::capture_diagnostics &ev) { + return base_type::process_event(ev); + } +}; + +} // namespace emel::kernel::matmul diff --git a/src/emel/machines.hpp b/src/emel/machines.hpp index 9ce5cd3c..b6594eed 100644 --- a/src/emel/machines.hpp +++ b/src/emel/machines.hpp @@ -12,6 +12,9 @@ #include "emel/io/mmap/sm.hpp" #include "emel/io/read/sm.hpp" #include "emel/io/staged_read/sm.hpp" +#include "emel/kernel/attention/sm.hpp" +#include "emel/kernel/f32_matvec/sm.hpp" +#include "emel/kernel/matmul/sm.hpp" #include "emel/memory/hybrid/sm.hpp" #include "emel/memory/kv/sm.hpp" #include "emel/memory/recurrent/sm.hpp" @@ -52,6 +55,9 @@ using IoLoader = emel::io::loader::sm; using IoMmap = emel::io::mmap::sm; using IoRead = emel::io::read::sm; using IoStagedRead = emel::io::staged_read::sm; +using KernelAttention = emel::kernel::attention::sm; +using F32Matvec = emel::kernel::f32_matvec::sm; +using KernelMatmul = emel::kernel::matmul::sm; using MemoryHybrid = emel::memory::hybrid::sm; using KvCache = emel::memory::kv::sm; using MemoryRecurrent = emel::memory::recurrent::sm; diff --git a/src/emel/sm.hpp b/src/emel/sm.hpp index eed2c89c..56c35792 100644 --- a/src/emel/sm.hpp +++ b/src/emel/sm.hpp @@ -131,7 +131,16 @@ class fork_join_lane_pool { static constexpr std::size_t static_worker_count = worker_count; - fork_join_lane_pool() { start_workers(); } + fork_join_lane_pool() : fork_join_lane_pool(worker_count) {} + + explicit fork_join_lane_pool(const std::size_t active_worker_count) + : active_worker_count_(active_worker_count) { + if (active_worker_count_ == 0u || active_worker_count_ > worker_count) { + throw std::invalid_argument( + "fork_join_lane_pool active worker count is out of range"); + } + start_workers(); + } ~fork_join_lane_pool() { stop_workers(); } @@ -201,10 +210,39 @@ class fork_join_lane_pool { return true; } + template + std::size_t try_submit_batch(join_group & group, fns &&... fns_in) noexcept { + static_assert(sizeof...(fns) > 0u, + "fork_join_lane_pool batch must contain tasks"); + if (running_on_this_worker()) { + group.reject(); + return 0u; + } + + std::array claimed_workers = {}; + std::size_t claimed_count = 0u; + const std::size_t start = + next_worker_.fetch_add(sizeof...(fns), std::memory_order_relaxed) % + active_worker_count_; + (claim_batch_task(group, claimed_workers, claimed_count, start, + std::forward(fns_in)), + ...); + + std::atomic_thread_fence(std::memory_order_release); + for (std::size_t index = 0u; index < claimed_count; ++index) { + workers_[claimed_workers[index]].ready.release(); + } + return claimed_count; + } + bool is_current_thread_worker() const noexcept { return running_on_this_worker(); } + std::size_t active_worker_count() const noexcept { + return active_worker_count_; + } + private: struct task_slot { using invoke_fn = void (*)(void *) noexcept; @@ -263,7 +301,7 @@ class fork_join_lane_pool { void start_workers() { try { - for (std::size_t index = 0u; index < worker_count; ++index) { + for (std::size_t index = 0u; index < active_worker_count_; ++index) { workers_[index].thread = std::thread([this, index]() noexcept { worker_loop(index); }); } @@ -274,11 +312,13 @@ class fork_join_lane_pool { } void stop_workers() noexcept { - for (auto &worker : workers_) { + for (std::size_t index = 0u; index < active_worker_count_; ++index) { + auto &worker = workers_[index]; worker.stopping.store(true, std::memory_order_release); worker.ready.release(); } - for (auto &worker : workers_) { + for (std::size_t index = 0u; index < active_worker_count_; ++index) { + auto &worker = workers_[index]; if (worker.thread.joinable()) { worker.thread.join(); } @@ -291,9 +331,11 @@ class fork_join_lane_pool { void (*completion_fn)(void *) noexcept) noexcept { const std::size_t start = - next_worker_.fetch_add(1u, std::memory_order_relaxed) % worker_count; - for (std::size_t offset = 0u; offset < worker_count; ++offset) { - worker_slot &worker = workers_[(start + offset) % worker_count]; + next_worker_.fetch_add(1u, std::memory_order_relaxed) % + active_worker_count_; + for (std::size_t offset = 0u; offset < active_worker_count_; ++offset) { + worker_slot &worker = + workers_[(start + offset) % active_worker_count_]; bool expected = true; if (!worker.idle.compare_exchange_strong( expected, false, std::memory_order_acq_rel, @@ -308,6 +350,37 @@ class fork_join_lane_pool { return false; } + template + void claim_batch_task(join_group & group, + std::array & claimed_workers, + std::size_t & claimed_count, const std::size_t start, + fn && fn_in) noexcept { + using fn_type = std::decay_t; + static_assert(std::is_nothrow_constructible_v, + "fork_join_lane_pool batch task construction must not throw"); + static_assert(std::is_nothrow_invocable_v, + "fork_join_lane_pool batch task invocation must not throw"); + + for (std::size_t offset = 0u; offset < active_worker_count_; ++offset) { + const std::size_t worker_index = + (start + offset) % active_worker_count_; + worker_slot & worker = workers_[worker_index]; + bool expected = true; + if (!worker.idle.compare_exchange_strong( + expected, false, std::memory_order_acq_rel, + std::memory_order_acquire)) { + continue; + } + group.start_one(); + worker.task.set(std::forward(fn_in), &group, + join_group::complete_one); + claimed_workers[claimed_count] = worker_index; + ++claimed_count; + return; + } + group.reject(); + } + void worker_loop(const std::size_t index) noexcept { struct worker_scope { const fork_join_lane_pool *previous; @@ -356,6 +429,7 @@ class fork_join_lane_pool { } std::array workers_{}; + const std::size_t active_worker_count_; std::atomic next_worker_ = 0u; inline static thread_local const fork_join_lane_pool *active_worker_pool_ = nullptr; diff --git a/src/emel/speech/codec/mimi/actions.hpp b/src/emel/speech/codec/mimi/actions.hpp index 0bfc9a51..a847152c 100644 --- a/src/emel/speech/codec/mimi/actions.hpp +++ b/src/emel/speech/codec/mimi/actions.hpp @@ -55,16 +55,59 @@ struct effect_mark_code_range_invalid { }; // One-time bind of the caller arenas into the codec runtime. Non-failing by -// contract: the transition rows reach this only after guard_bind_contract_valid -// (pure dry-run of the same walk) and guard_arena_capacity_valid passed. -struct effect_bind { +// contract: the transition rows reach these only after the matching immutable +// pre-dispatch contract and guard_arena_capacity_valid passed. +struct effect_bind_f32 { void operator()(const event::initialize_run &runtime_ev, context &ctx) const noexcept { const auto &request = runtime_ev.request; ctx.workspace = request.workspace; ctx.frame = request.frame; - detail::bind_codec_runtime(request.model, request.prepared, - request.state_arena, ctx.runtime, ctx.streaming); + detail::bind_codec_runtime_f32(request.model, request.prepared, + request.state_arena, ctx.runtime, + ctx.streaming); + } +}; + +struct effect_bind_native { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + const auto &request = runtime_ev.request; + ctx.workspace = request.workspace; + ctx.frame = request.frame; + detail::bind_codec_runtime_native(request.model, request.prepared, + request.state_arena, ctx.runtime, + ctx.streaming); + } +}; + +struct effect_bind_q8 { + void operator()(const event::initialize_run &runtime_ev, + context &ctx) const noexcept { + const auto &request = runtime_ev.request; + ctx.workspace = request.workspace; + ctx.frame = request.frame; + detail::bind_codec_runtime_q8(request.model, request.prepared, + request.state_arena, ctx.runtime, + ctx.streaming); + } +}; + +struct effect_capture_diagnostics { + void operator()(const event::capture_diagnostics &ev, + context &ctx) const noexcept { + emel::kernel::f32_matvec::event::diagnostics projection{}; + emel::kernel::f32_matvec::event::dispatch_result result{}; + (void)ctx.runtime.projection_matvec.process_event( + emel::kernel::f32_matvec::event::capture_diagnostics{projection, + result}); + ev.out = event::diagnostics{ + .projection_prepare_calls = projection.prepare_calls, + .projection_prepared_floats = projection.prepared_floats, + .projection_reference_calls = projection.reference_calls, + .projection_exact_x4_calls = projection.exact_x4_calls, + .legacy_f32_projection_calls = ctx.runtime.legacy_f32_projection_calls, + }; } }; diff --git a/src/emel/speech/codec/mimi/any.hpp b/src/emel/speech/codec/mimi/any.hpp index 046a38af..12b9f72f 100644 --- a/src/emel/speech/codec/mimi/any.hpp +++ b/src/emel/speech/codec/mimi/any.hpp @@ -7,6 +7,14 @@ namespace emel::speech::codec::mimi { +// Compute immutable validation facts before entering the actor's RTC +// boundary. The state machine still owns route selection through guards over +// these facts; no behavior is selected by this preflight. +inline event::bind_contract +make_bind_contract(const emel::model::data &model) noexcept { + return guard::make_bind_contract(model); +} + // Public arena-sizing contract for event::initialize: the caller owns the // four arenas, sizes them here before dispatching initialize, and keeps them // alive for the codec's lifetime (the machine never allocates). A result of @@ -21,7 +29,8 @@ inline uint64_t state_arena_floats(const emel::model::data &model) noexcept { return detail::required_state_floats(model); } -inline uint64_t workspace_arena_floats(const emel::model::data &model) noexcept { +inline uint64_t +workspace_arena_floats(const emel::model::data &model) noexcept { return detail::required_workspace_floats(model); } diff --git a/src/emel/speech/codec/mimi/detail.cpp b/src/emel/speech/codec/mimi/detail.cpp index ad29145b..cac4407f 100644 --- a/src/emel/speech/codec/mimi/detail.cpp +++ b/src/emel/speech/codec/mimi/detail.cpp @@ -112,6 +112,13 @@ template struct arena_cursor_t { using arena_cursor = arena_cursor_t; +enum class projection_bind_kind : uint8_t { + f32, + f16, + native, + q8, +}; + // Reads one element of an f32/f16 source tensor as f32. float source_value(const emel::model::data::tensor_record &tensor, const uint64_t index) noexcept { @@ -208,6 +215,68 @@ const float *prepare_matrix_raw(arena_cursor_t &cursor, return out; } +// Packs one exact F32 projection through the kernel-owned actor. The owning +// facade guard has already selected the F32 route and validated every source, +// arena, alignment, capacity, and overlap before the live specialization runs. +template +const float * +prepare_matrix_exact_x4(arena_cursor_t &cursor, + emel::kernel::f32_matvec::sm &projection_matvec, + const emel::model::data::tensor_record *tensor, + const int64_t in_count, + const int64_t out_count) noexcept { + if constexpr (dry_run) { + (void)projection_matvec; + return prepare_matrix_raw(cursor, tensor, in_count, out_count); + } + const uint64_t elements = + static_cast(in_count) * static_cast(out_count); + float *out = cursor.take(elements); + const auto source = std::span{ + static_cast(tensor->data), static_cast(elements)}; + const auto destination = std::span{out, static_cast(elements)}; + const emel::kernel::f32_matvec::event::prepare_f32_request request{ + .source = source, + .destination = destination, + .inner = static_cast(in_count), + .rows = static_cast(out_count), + }; + emel::kernel::f32_matvec::event::dispatch_result result{}; + (void)projection_matvec.process_event( + emel::kernel::f32_matvec::event::prepare_f32{request, result}); + return out; +} + +template +const float * +prepare_matrix_exact_x4_f16(arena_cursor_t &cursor, + emel::kernel::f32_matvec::sm &projection_matvec, + const emel::model::data::tensor_record *tensor, + const int64_t in_count, + const int64_t out_count) noexcept { + if constexpr (dry_run) { + (void)projection_matvec; + return prepare_matrix_raw(cursor, tensor, in_count, out_count); + } + const uint64_t elements = + static_cast(in_count) * static_cast(out_count); + float *out = cursor.take(elements); + const auto source = + std::span{static_cast(tensor->data), + static_cast(elements)}; + const auto destination = std::span{out, static_cast(elements)}; + const emel::kernel::f32_matvec::event::prepare_f16_request request{ + .source = source, + .destination = destination, + .inner = static_cast(in_count), + .rows = static_cast(out_count), + }; + emel::kernel::f32_matvec::event::dispatch_result result{}; + (void)projection_matvec.process_event( + emel::kernel::f32_matvec::event::prepare_f16{request, result}); + return out; +} + // Copies raw q8_0 row blocks verbatim into the arena. Returns nullptr when // the source is not q8_0 or capacity is exceeded. template @@ -405,6 +474,26 @@ bool dispatch_q8_matvec(codec_runtime &runtime, const uint8_t *weight_q8, return runtime.kernel.process_event(ev); } +#if defined(__aarch64__) || defined(_M_ARM64) +void dispatch_f32_exact_x4(codec_runtime &runtime, const float *weights, + const int64_t inner, const int64_t rows, + const float *input, float *output) noexcept { + const uint64_t weight_count = + static_cast(inner) * static_cast(rows); + const emel::kernel::f32_matvec::event::execute_request request{ + .weights = + std::span{weights, static_cast(weight_count)}, + .input = std::span{input, static_cast(inner)}, + .output = std::span{output, static_cast(rows)}, + .inner = static_cast(inner), + .rows = static_cast(rows), + }; + emel::kernel::f32_matvec::event::dispatch_result result{}; + (void)runtime.projection_matvec.process_event( + emel::kernel::f32_matvec::event::execute_exact_x4{request, result}); +} +#endif + kev::tensor_view_mut make_strided_view_mut(float *data, const uint64_t ne0, const uint64_t ne1, const uint64_t nb0_bytes, @@ -1044,9 +1133,15 @@ void compute_transformer(codec_runtime &runtime, return; } } else { +#if defined(__aarch64__) || defined(_M_ARM64) + dispatch_f32_exact_x4(runtime, layer.in_proj, dim, 3 * dim, normed, + qkv); +#else for (int64_t o = 0; o < 3 * dim; ++o) { qkv[o] = kd::vec_dot_f32_ggml(dim, layer.in_proj + o * dim, normed); } + ++runtime.legacy_f32_projection_calls; +#endif } // reference rope: timestep-embedding angles, adjacent-pair rotation, @@ -1121,9 +1216,14 @@ void compute_transformer(codec_runtime &runtime, return; } } else { +#if defined(__aarch64__) || defined(_M_ARM64) + dispatch_f32_exact_x4(runtime, layer.out_proj, dim, dim, attn, proj); +#else for (int64_t o = 0; o < dim; ++o) { proj[o] = kd::vec_dot_f32_ggml(dim, layer.out_proj + o * dim, attn); } + ++runtime.legacy_f32_projection_calls; +#endif } for (int64_t d = 0; d < dim; ++d) { x[d] = x[d] + proj[d] * layer.layer_scale_1[d]; @@ -1142,9 +1242,15 @@ void compute_transformer(codec_runtime &runtime, return; } } else { +#if defined(__aarch64__) || defined(_M_ARM64) + dispatch_f32_exact_x4(runtime, layer.linear1, dim, mlp_dim, normed, + mlp); +#else for (int64_t o = 0; o < mlp_dim; ++o) { mlp[o] = kd::vec_dot_f32_ggml(dim, layer.linear1 + o * dim, normed); } + ++runtime.legacy_f32_projection_calls; +#endif } if (!dispatch_unary(runtime, kev::unary_subop::gelu, make_view(mlp, static_cast(mlp_dim)), @@ -1157,10 +1263,15 @@ void compute_transformer(codec_runtime &runtime, return; } } else { +#if defined(__aarch64__) || defined(_M_ARM64) + dispatch_f32_exact_x4(runtime, layer.linear2, mlp_dim, dim, mlp, proj); +#else for (int64_t o = 0; o < dim; ++o) { proj[o] = kd::vec_dot_f32_ggml(mlp_dim, layer.linear2 + o * mlp_dim, mlp); } + ++runtime.legacy_f32_projection_calls; +#endif } for (int64_t d = 0; d < dim; ++d) { x[d] = x[d] + proj[d] * layer.layer_scale_2[d]; @@ -1918,11 +2029,11 @@ bool bind_conv(arena_cursor_t &cursor, uint64_t &state_cursor, return true; } -template +template bool bind_transformer(arena_cursor_t &cursor, uint64_t &state_cursor, + emel::kernel::f32_matvec::sm &projection_matvec, const emel::model::data &model_data, const char *family, const transformer_plan &plan, const int32_t frame_tokens, - const bool proj_q8, transformer_weights &weights_out) noexcept { const auto &mimi = model_data.mimi; const int64_t dim = mimi.dim; @@ -1950,7 +2061,7 @@ bool bind_transformer(arena_cursor_t &cursor, uint64_t &state_cursor, layer.norm1_bias = prepare_vector( cursor, find_layer_tensor(model_data, format, index, "norm1.bias"), dim); - if (proj_q8) { + if constexpr (bind_kind == projection_bind_kind::q8) { layer.in_proj_q8 = prepare_raw_q8_0( cursor, find_layer_tensor(model_data, format, index, @@ -1961,6 +2072,28 @@ bool bind_transformer(arena_cursor_t &cursor, uint64_t &state_cursor, find_layer_tensor(model_data, format, index, "self_attn.out_projs.0.weight"), static_cast(dim) * static_cast(dim)); + } else if constexpr (bind_kind == projection_bind_kind::f32) { + layer.in_proj = prepare_matrix_exact_x4( + cursor, projection_matvec, + find_layer_tensor(model_data, format, index, + "self_attn.in_projs.0.weight"), + dim, 3 * dim); + layer.out_proj = prepare_matrix_exact_x4( + cursor, projection_matvec, + find_layer_tensor(model_data, format, index, + "self_attn.out_projs.0.weight"), + dim, dim); + } else if constexpr (bind_kind == projection_bind_kind::f16) { + layer.in_proj = prepare_matrix_exact_x4_f16( + cursor, projection_matvec, + find_layer_tensor(model_data, format, index, + "self_attn.in_projs.0.weight"), + dim, 3 * dim); + layer.out_proj = prepare_matrix_exact_x4_f16( + cursor, projection_matvec, + find_layer_tensor(model_data, format, index, + "self_attn.out_projs.0.weight"), + dim, dim); } else { layer.in_proj = prepare_matrix_raw(cursor, @@ -1983,7 +2116,7 @@ bool bind_transformer(arena_cursor_t &cursor, uint64_t &state_cursor, layer.norm2_bias = prepare_vector( cursor, find_layer_tensor(model_data, format, index, "norm2.bias"), dim); - if (proj_q8) { + if constexpr (bind_kind == projection_bind_kind::q8) { layer.linear1_q8 = prepare_raw_q8_0( cursor, find_layer_tensor(model_data, format, index, "linear1.weight"), @@ -1992,6 +2125,24 @@ bool bind_transformer(arena_cursor_t &cursor, uint64_t &state_cursor, cursor, find_layer_tensor(model_data, format, index, "linear2.weight"), static_cast(plan.mlp_dim) * static_cast(dim)); + } else if constexpr (bind_kind == projection_bind_kind::f32) { + layer.linear1 = prepare_matrix_exact_x4( + cursor, projection_matvec, + find_layer_tensor(model_data, format, index, "linear1.weight"), dim, + plan.mlp_dim); + layer.linear2 = prepare_matrix_exact_x4( + cursor, projection_matvec, + find_layer_tensor(model_data, format, index, "linear2.weight"), + plan.mlp_dim, dim); + } else if constexpr (bind_kind == projection_bind_kind::f16) { + layer.linear1 = prepare_matrix_exact_x4_f16( + cursor, projection_matvec, + find_layer_tensor(model_data, format, index, "linear1.weight"), dim, + plan.mlp_dim); + layer.linear2 = prepare_matrix_exact_x4_f16( + cursor, projection_matvec, + find_layer_tensor(model_data, format, index, "linear2.weight"), + plan.mlp_dim, dim); } else { layer.linear1 = prepare_matrix_raw( cursor, @@ -2006,12 +2157,16 @@ bool bind_transformer(arena_cursor_t &cursor, uint64_t &state_cursor, cursor, find_layer_tensor(model_data, format, index, "layer_scale_2.scale"), dim); - const bool projections_bound = - proj_q8 - ? (layer.in_proj_q8 != nullptr && layer.out_proj_q8 != nullptr && - layer.linear1_q8 != nullptr && layer.linear2_q8 != nullptr) - : (layer.in_proj != nullptr && layer.out_proj != nullptr && - layer.linear1 != nullptr && layer.linear2 != nullptr); + bool projections_bound = false; + if constexpr (bind_kind == projection_bind_kind::q8) { + projections_bound = + layer.in_proj_q8 != nullptr && layer.out_proj_q8 != nullptr && + layer.linear1_q8 != nullptr && layer.linear2_q8 != nullptr; + } else { + projections_bound = layer.in_proj != nullptr && + layer.out_proj != nullptr && + layer.linear1 != nullptr && layer.linear2 != nullptr; + } if (layer.norm1_weight == nullptr || layer.norm1_bias == nullptr || !projections_bound || layer.layer_scale_1 == nullptr || layer.norm2_weight == nullptr || layer.norm2_bias == nullptr || @@ -2191,14 +2346,30 @@ uint64_t required_frame_floats(const emel::model::data &model_data) noexcept { // caller memory - it only fills the caller-local scratch runtime); the live // instantiation canonicalizes into the arenas. One walk, two modes, so the // validator can never drift from the bind it authorizes. -template +template bool run_codec_bind_walk(const emel::model::data &model_data, std::span prepared, std::span state, codec_runtime &runtime_out, codec_streaming_state &state_out) noexcept { codec_plan plan{}; - if (!plan_codec(model_data, plan)) { - return false; + if constexpr (dry_run) { + if (!plan_codec(model_data, plan)) { + return false; + } + } else { + // The caller supplied a pre-dispatch bind contract and the owning guard + // accepted it. Rebuild the immutable geometry for binding only; its + // boolean validation result does not route or select an RTC outcome. + (void)plan_codec(model_data, plan); + } + if constexpr (dry_run) { + if constexpr (bind_kind == projection_bind_kind::q8) { + if (!plan.proj_q8) { + return false; + } + } else if (plan.proj_q8) { + return false; + } } if constexpr (!dry_run) { if (prepared.size() < plan.prepared_floats || @@ -2222,18 +2393,34 @@ bool run_codec_bind_walk(const emel::model::data &model_data, arena_cursor_t cursor{prepared, 0}; uint64_t state_cursor = 0; - if (!bind_seanet(cursor, state_cursor, plan.encoder, plan.conv_f16, - runtime_out.encoder_layers) || - !bind_transformer(cursor, state_cursor, model_data, "encoder_transformer", - plan.encoder_transformer, - static_cast(plan.encoder_tokens), plan.proj_q8, - runtime_out.encoder_transformer)) { - return false; - } - - if (!bind_conv(cursor, state_cursor, plan.downsample, false, plan.conv_f16, - runtime_out.downsample)) { - return false; + if constexpr (dry_run) { + if (!bind_seanet(cursor, state_cursor, plan.encoder, plan.conv_f16, + runtime_out.encoder_layers) || + !bind_transformer( + cursor, state_cursor, runtime_out.projection_matvec, model_data, + "encoder_transformer", plan.encoder_transformer, + static_cast(plan.encoder_tokens), + runtime_out.encoder_transformer)) { + return false; + } + } else { + (void)bind_seanet(cursor, state_cursor, plan.encoder, plan.conv_f16, + runtime_out.encoder_layers); + (void)bind_transformer( + cursor, state_cursor, runtime_out.projection_matvec, model_data, + "encoder_transformer", plan.encoder_transformer, + static_cast(plan.encoder_tokens), + runtime_out.encoder_transformer); + } + + if constexpr (dry_run) { + if (!bind_conv(cursor, state_cursor, plan.downsample, false, plan.conv_f16, + runtime_out.downsample)) { + return false; + } + } else { + (void)bind_conv(cursor, state_cursor, plan.downsample, false, plan.conv_f16, + runtime_out.downsample); } runtime_out.conv_f16 = plan.conv_f16; @@ -2242,13 +2429,22 @@ bool run_codec_bind_walk(const emel::model::data &model_data, if constexpr (!dry_run) { runtime_out.kernel.set_kind(runtime_out.kernel_kind); } - if (!bind_rvq_split(cursor, model_data, "rvq_first", mimi.semantic_n_q, - plan.conv_f16, plan.rvq_q8, - runtime_out.quantizer.semantic) || - !bind_rvq_split(cursor, model_data, "rvq_rest", - mimi.n_q - mimi.semantic_n_q, plan.conv_f16, plan.rvq_q8, - runtime_out.quantizer.acoustic)) { - return false; + if constexpr (dry_run) { + if (!bind_rvq_split(cursor, model_data, "rvq_first", mimi.semantic_n_q, + plan.conv_f16, plan.rvq_q8, + runtime_out.quantizer.semantic) || + !bind_rvq_split(cursor, model_data, "rvq_rest", + mimi.n_q - mimi.semantic_n_q, plan.conv_f16, + plan.rvq_q8, runtime_out.quantizer.acoustic)) { + return false; + } + } else { + (void)bind_rvq_split(cursor, model_data, "rvq_first", mimi.semantic_n_q, + plan.conv_f16, plan.rvq_q8, + runtime_out.quantizer.semantic); + (void)bind_rvq_split(cursor, model_data, "rvq_rest", + mimi.n_q - mimi.semantic_n_q, plan.conv_f16, + plan.rvq_q8, runtime_out.quantizer.acoustic); } { @@ -2276,13 +2472,24 @@ bool run_codec_bind_walk(const emel::model::data &model_data, state_cursor += upsample.state_floats; } - if (!bind_transformer(cursor, state_cursor, model_data, "decoder_transformer", - plan.decoder_transformer, - static_cast(plan.decoder_tokens), plan.proj_q8, - runtime_out.decoder_transformer) || - !bind_seanet(cursor, state_cursor, plan.decoder, plan.conv_f16, - runtime_out.decoder_layers)) { - return false; + if constexpr (dry_run) { + if (!bind_transformer( + cursor, state_cursor, runtime_out.projection_matvec, model_data, + "decoder_transformer", plan.decoder_transformer, + static_cast(plan.decoder_tokens), + runtime_out.decoder_transformer) || + !bind_seanet(cursor, state_cursor, plan.decoder, plan.conv_f16, + runtime_out.decoder_layers)) { + return false; + } + } else { + (void)bind_transformer( + cursor, state_cursor, runtime_out.projection_matvec, model_data, + "decoder_transformer", plan.decoder_transformer, + static_cast(plan.decoder_tokens), + runtime_out.decoder_transformer); + (void)bind_seanet(cursor, state_cursor, plan.decoder, plan.conv_f16, + runtime_out.decoder_layers); } runtime_out.prepared_floats_used = cursor.used; @@ -2296,14 +2503,114 @@ bool run_codec_bind_walk(const emel::model::data &model_data, } } -bool validate_codec_contract(const emel::model::data &model_data) noexcept { +template +bool validate_codec_contract_for(const emel::model::data &model_data) noexcept { // Caller-local scratch: the dry-run walk writes only here, never to any // caller arena, so this is a pure predicate suitable for guards. One-time // initialization path only, never per-frame. codec_runtime scratch_runtime{}; codec_streaming_state scratch_state{}; - return run_codec_bind_walk(model_data, {}, {}, scratch_runtime, - scratch_state); + return run_codec_bind_walk(model_data, {}, {}, + scratch_runtime, scratch_state); +} + +template +void bind_validated_codec_runtime_walk( + const emel::model::data &model_data, std::span prepared, + std::span state, codec_runtime &runtime_out, + codec_streaming_state &state_out) noexcept { + codec_plan plan{}; + (void)plan_codec(model_data, plan); + + const auto &mimi = model_data.mimi; + runtime_out.model = &model_data; + runtime_out.sample_rate = mimi.sample_rate; + runtime_out.frame_samples = plan.frame_samples; + runtime_out.n_q = mimi.n_q; + runtime_out.dim = mimi.dim; + runtime_out.prepared_arena = prepared; + runtime_out.quantizer.codebook_entries = mimi.card; + runtime_out.quantizer.codebook_dim = mimi.codebook_dim; + runtime_out.workspace_floats = workspace_floats_from_plan(model_data, plan); + runtime_out.frame_floats = plan.extents.frame + 64u; + + arena_cursor cursor{prepared, 0}; + uint64_t state_cursor = 0u; + (void)bind_seanet(cursor, state_cursor, plan.encoder, plan.conv_f16, + runtime_out.encoder_layers); + (void)bind_transformer( + cursor, state_cursor, runtime_out.projection_matvec, model_data, + "encoder_transformer", plan.encoder_transformer, + static_cast(plan.encoder_tokens), + runtime_out.encoder_transformer); + (void)bind_conv(cursor, state_cursor, plan.downsample, false, plan.conv_f16, + runtime_out.downsample); + + runtime_out.conv_f16 = plan.conv_f16; + runtime_out.proj_q8 = plan.proj_q8; + runtime_out.rvq_q8 = plan.rvq_q8; + runtime_out.kernel.set_kind(runtime_out.kernel_kind); + (void)bind_rvq_split(cursor, model_data, "rvq_first", mimi.semantic_n_q, + plan.conv_f16, plan.rvq_q8, + runtime_out.quantizer.semantic); + (void)bind_rvq_split(cursor, model_data, "rvq_rest", + mimi.n_q - mimi.semantic_n_q, plan.conv_f16, plan.rvq_q8, + runtime_out.quantizer.acoustic); + + const int64_t taps = plan.upsample.taps; + const int64_t channels = plan.upsample.out_channels; + auto &upsample = runtime_out.upsample; + upsample.weight = prepare_conv_transpose(cursor, plan.upsample.weight, + static_cast(taps) * + static_cast(channels)); + upsample.bias = nullptr; + upsample.taps = static_cast(taps); + upsample.in_channels = static_cast(channels); + upsample.out_channels = static_cast(channels); + upsample.stride = 2; + upsample.frame_length = 1; + upsample.state_offset = state_cursor; + const int64_t lout_full = (1 - 1) * 2 + taps; + upsample.state_floats = + static_cast(lout_full) * static_cast(channels); + state_cursor += upsample.state_floats; + + (void)bind_transformer( + cursor, state_cursor, runtime_out.projection_matvec, model_data, + "decoder_transformer", plan.decoder_transformer, + static_cast(plan.decoder_tokens), + runtime_out.decoder_transformer); + (void)bind_seanet(cursor, state_cursor, plan.decoder, plan.conv_f16, + runtime_out.decoder_layers); + + runtime_out.prepared_floats_used = cursor.used; + state_out.arena = state; + reset_streaming_state(runtime_out, state_out); +} + +bool validate_codec_contract_f32(const emel::model::data &model_data) noexcept { +#if defined(__aarch64__) || defined(_M_ARM64) + return validate_codec_contract_for(model_data); +#else + return validate_codec_contract_for(model_data); +#endif +} + +bool validate_codec_contract_native( + const emel::model::data &model_data) noexcept { +#if defined(__aarch64__) || defined(_M_ARM64) + return validate_codec_contract_for(model_data); +#else + return validate_codec_contract_for(model_data); +#endif +} + +bool validate_codec_contract_q8(const emel::model::data &model_data) noexcept { + return validate_codec_contract_for(model_data); +} + +bool validate_codec_contract(const emel::model::data &model_data) noexcept { + return validate_codec_contract_f32(model_data); } bool validate_codes_in_range(const codec_runtime &runtime, @@ -2322,15 +2629,58 @@ bool validate_codes_in_range(const codec_runtime &runtime, return true; } +template +void bind_codec_runtime_for(const emel::model::data &model_data, + std::span prepared, std::span state, + codec_runtime &runtime_out, + codec_streaming_state &state_out) noexcept { + // Non-failing by contract: immutable validation facts and capacities were + // accepted before this route-specialized action runs. The live walk has no + // failure result and therefore cannot select the next RTC transition. + bind_validated_codec_runtime_walk(model_data, prepared, state, + runtime_out, state_out); +} + +void bind_codec_runtime_f32(const emel::model::data &model_data, + std::span prepared, std::span state, + codec_runtime &runtime_out, + codec_streaming_state &state_out) noexcept { +#if defined(__aarch64__) || defined(_M_ARM64) + bind_codec_runtime_for(model_data, prepared, state, + runtime_out, state_out); +#else + bind_codec_runtime_for( + model_data, prepared, state, runtime_out, state_out); +#endif +} + +void bind_codec_runtime_native(const emel::model::data &model_data, + std::span prepared, + std::span state, + codec_runtime &runtime_out, + codec_streaming_state &state_out) noexcept { +#if defined(__aarch64__) || defined(_M_ARM64) + bind_codec_runtime_for(model_data, prepared, state, + runtime_out, state_out); +#else + bind_codec_runtime_for( + model_data, prepared, state, runtime_out, state_out); +#endif +} + +void bind_codec_runtime_q8(const emel::model::data &model_data, + std::span prepared, std::span state, + codec_runtime &runtime_out, + codec_streaming_state &state_out) noexcept { + bind_codec_runtime_for(model_data, prepared, state, + runtime_out, state_out); +} + void bind_codec_runtime(const emel::model::data &model_data, std::span prepared, std::span state, codec_runtime &runtime_out, codec_streaming_state &state_out) noexcept { - // Non-failing by contract: the owning machine's guards route on - // validate_codec_contract and the required_* capacities before this action - // runs, so the walk's data-plane short-circuit cannot trigger here. - (void)run_codec_bind_walk(model_data, prepared, state, runtime_out, - state_out); + bind_codec_runtime_f32(model_data, prepared, state, runtime_out, state_out); } template void compute_seanet_encoder(codec_runtime &, diff --git a/src/emel/speech/codec/mimi/detail.hpp b/src/emel/speech/codec/mimi/detail.hpp index 99b9d893..2b11c398 100644 --- a/src/emel/speech/codec/mimi/detail.hpp +++ b/src/emel/speech/codec/mimi/detail.hpp @@ -7,6 +7,7 @@ #include "emel/error/error.hpp" #include "emel/kernel/events.hpp" +#include "emel/kernel/f32_matvec/sm.hpp" #include "emel/kernel/sm.hpp" #include "emel/model/data.hpp" @@ -125,6 +126,8 @@ struct seanet_layer_weights { struct transformer_layer_weights { const float *norm1_weight = nullptr; const float *norm1_bias = nullptr; + // Native row-major off AArch64. On AArch64 F32 models, complete groups of + // four output rows are column-interleaved; 1/2/3 tail rows remain native. const float *in_proj = nullptr; // [dim, 3*dim] fused qkv const float *out_proj = nullptr; // [dim, dim] // raw q8_0 row blocks when the model carries quantized projections @@ -185,6 +188,7 @@ struct quantizer_weights { struct codec_runtime { const emel::model::data *model = nullptr; emel::kernel::sm kernel = {}; + emel::kernel::f32_matvec::sm projection_matvec = {}; // Host backend, resolved at compile time and applied once at bind; the // compute helpers never re-select it. emel::kernel::kernel_kind kernel_kind = emel::kernel::detect_host_kind(); @@ -195,6 +199,7 @@ struct codec_runtime { bool conv_f16 = false; bool proj_q8 = false; bool rvq_q8 = false; + uint64_t legacy_f32_projection_calls = 0u; std::array encoder_layers = {}; conv_weights downsample = {}; @@ -253,6 +258,15 @@ uint64_t required_frame_floats(const emel::model::data &model_data) noexcept; // element checks, no arena writes), so the guard-side predicate can never // drift from the bind it authorizes. Capacity is validated separately via // the required_* sizing contract. +bool validate_codec_contract_f32(const emel::model::data &model_data) noexcept; + +bool validate_codec_contract_native( + const emel::model::data &model_data) noexcept; + +bool validate_codec_contract_q8(const emel::model::data &model_data) noexcept; + +// F32 compatibility entrypoint retained for direct codec tests. Runtime +// facade initialization uses the explicit typed validators above. bool validate_codec_contract(const emel::model::data &model_data) noexcept; // Pure per-request validator for decode codes: every code addresses a valid @@ -264,6 +278,24 @@ bool validate_codes_in_range(const codec_runtime &runtime, // Canonicalizes weights into `prepared`, carves `state`, fills `runtime_out`. // Non-failing by contract: the owning machine's guards route on // validate_codec_contract and the required_* capacities before this runs. +void bind_codec_runtime_f32(const emel::model::data &model_data, + std::span prepared, std::span state, + codec_runtime &runtime_out, + codec_streaming_state &state_out) noexcept; + +void bind_codec_runtime_native(const emel::model::data &model_data, + std::span prepared, + std::span state, + codec_runtime &runtime_out, + codec_streaming_state &state_out) noexcept; + +void bind_codec_runtime_q8(const emel::model::data &model_data, + std::span prepared, std::span state, + codec_runtime &runtime_out, + codec_streaming_state &state_out) noexcept; + +// F32 compatibility entrypoint retained for direct codec tests. Runtime +// facade initialization uses the explicit typed bind actions above. void bind_codec_runtime(const emel::model::data &model_data, std::span prepared, std::span state, codec_runtime &runtime_out, diff --git a/src/emel/speech/codec/mimi/events.hpp b/src/emel/speech/codec/mimi/events.hpp index f5aaa66c..bf6d54cb 100644 --- a/src/emel/speech/codec/mimi/events.hpp +++ b/src/emel/speech/codec/mimi/events.hpp @@ -6,6 +6,7 @@ #include "emel/callback.hpp" #include "emel/error/error.hpp" #include "emel/model/data.hpp" +#include "emel/speech/codec/mimi/detail.hpp" #include "emel/speech/codec/mimi/errors.hpp" namespace emel::speech::codec::mimi::events { @@ -21,19 +22,87 @@ struct decode_frame_error; namespace emel::speech::codec::mimi::event { +class bind_contract; + +} // namespace emel::speech::codec::mimi::event + +namespace emel::speech::codec::mimi::guard { + +event::bind_contract +make_bind_contract(const emel::model::data &model) noexcept; + +} // namespace emel::speech::codec::mimi::guard + +namespace emel::speech::codec::mimi::event { + +// Opaque pre-dispatch validation facts. Only the authoritative factory can +// construct this value, so callers cannot forge route validity or arena sizes. +// The model address is retained and rechecked by the facade guards before any +// capacity fact is trusted. +class bind_contract { +public: + bind_contract(const bind_contract &) noexcept = default; + bind_contract(bind_contract &&) noexcept = default; + ~bind_contract() noexcept {} + + const emel::model::data *model() const noexcept { return model_; } + uint64_t model_identity_low() const noexcept { return model_identity_low_; } + uint64_t model_identity_high() const noexcept { return model_identity_high_; } + uint64_t prepared_floats() const noexcept { return prepared_floats_; } + uint64_t state_floats() const noexcept { return state_floats_; } + uint64_t workspace_floats() const noexcept { return workspace_floats_; } + uint64_t frame_floats() const noexcept { return frame_floats_; } + bool f32_valid() const noexcept { return f32_valid_; } + bool native_valid() const noexcept { return native_valid_; } + bool q8_valid() const noexcept { return q8_valid_; } + +private: + friend bind_contract + guard::make_bind_contract(const emel::model::data &model) noexcept; + + bind_contract(const emel::model::data &model_ref, + const uint64_t model_identity_low_ref, + const uint64_t model_identity_high_ref, + const uint64_t prepared_floats_ref, + const uint64_t state_floats_ref, + const uint64_t workspace_floats_ref, + const uint64_t frame_floats_ref, const bool f32_valid_ref, + const bool native_valid_ref, const bool q8_valid_ref) noexcept + : model_(&model_ref), model_identity_low_(model_identity_low_ref), + model_identity_high_(model_identity_high_ref), + prepared_floats_(prepared_floats_ref), state_floats_(state_floats_ref), + workspace_floats_(workspace_floats_ref), + frame_floats_(frame_floats_ref), f32_valid_(f32_valid_ref), + native_valid_(native_valid_ref), q8_valid_(q8_valid_ref) {} + + const emel::model::data *model_; + uint64_t model_identity_low_; + uint64_t model_identity_high_; + uint64_t prepared_floats_; + uint64_t state_floats_; + uint64_t workspace_floats_; + uint64_t frame_floats_; + bool f32_valid_; + bool native_valid_; + bool q8_valid_; +}; + // One-time bind. The caller owns all four arenas (sized via the public // arena-sizing contract in any.hpp) and must keep them alive for the codec's // lifetime; the actor performs no allocation. struct initialize { initialize(const emel::model::data &model_ref, + const bind_contract &contract_ref, const std::span prepared_ref, const std::span state_arena_ref, const std::span workspace_ref, const std::span frame_ref) noexcept - : model(model_ref), prepared(prepared_ref), state_arena(state_arena_ref), - workspace(workspace_ref), frame(frame_ref) {} + : model(model_ref), contract(contract_ref), prepared(prepared_ref), + state_arena(state_arena_ref), workspace(workspace_ref), + frame(frame_ref) {} const emel::model::data &model; + bind_contract contract; std::span prepared = {}; std::span state_arena = {}; std::span workspace = {}; @@ -72,6 +141,20 @@ struct decode_frame { // Rewind both streaming directions to the first frame. struct reset_stream {}; +struct diagnostics { + uint64_t projection_prepare_calls = 0u; + uint64_t projection_prepared_floats = 0u; + uint64_t projection_reference_calls = 0u; + uint64_t projection_exact_x4_calls = 0u; + uint64_t legacy_f32_projection_calls = 0u; +}; + +struct capture_diagnostics { + explicit capture_diagnostics(diagnostics &out_ref) noexcept : out(out_ref) {} + + diagnostics &out; +}; + // Per-dispatch runtime ctx: carries the typed error of one top-level // dispatch. Guards never read it - every success/error route is decided by // pure validation guards over the request and the bound runtime BEFORE the diff --git a/src/emel/speech/codec/mimi/guards.hpp b/src/emel/speech/codec/mimi/guards.hpp index e2a3f5e6..6e569077 100644 --- a/src/emel/speech/codec/mimi/guards.hpp +++ b/src/emel/speech/codec/mimi/guards.hpp @@ -1,26 +1,311 @@ #pragma once +#include +#include +#include +#include +#include +#include + #include "emel/speech/codec/mimi/context.hpp" #include "emel/speech/codec/mimi/detail.hpp" #include "emel/speech/codec/mimi/events.hpp" namespace emel::speech::codec::mimi::guard { -// Bind validation: the contract guard runs the pure dry-run instantiation of -// the bind walk; the capacity guard is pure size arithmetic against the -// required_* sizing contract plus the facade latent staging cap. Both are -// one-time initialization predicates, never per-frame. -struct guard_bind_contract_valid { +struct bind_model_identity { + uint64_t low = 14695981039346656037ull; + uint64_t high = 7809847782465536322ull; +}; + +inline void guard_mix_model_identity_byte(bind_model_identity &identity, + const uint8_t value) noexcept { + identity.low = (identity.low ^ value) * 1099511628211ull; + identity.high = (identity.high ^ value) * 14029467366897019727ull; +} + +inline void guard_mix_model_identity_word(bind_model_identity &identity, + const uint64_t value) noexcept { + for (uint32_t shift = 0u; shift < 64u; shift += 8u) { + guard_mix_model_identity_byte( + identity, static_cast((value >> shift) & 0xffu)); + } +} + +inline bind_model_identity +guard_compute_model_identity(const emel::model::data &model) noexcept { + bind_model_identity identity{}; + const auto mix_signed = [&identity](const int64_t value) noexcept { + guard_mix_model_identity_word(identity, static_cast(value)); + }; + const auto mix_unsigned = [&identity](const uint64_t value) noexcept { + guard_mix_model_identity_word(identity, value); + }; + + mix_signed(model.mimi.sample_rate); + mix_unsigned(std::bit_cast(model.mimi.frame_rate)); + mix_signed(model.mimi.n_q); + mix_signed(model.mimi.card); + mix_signed(model.mimi.dim); + mix_signed(model.mimi.semantic_n_q); + mix_signed(model.mimi.codebook_dim); + mix_signed(model.mimi.transformer_num_layers); + mix_signed(model.mimi.transformer_num_heads); + mix_signed(model.mimi.transformer_context); + mix_signed(model.mimi.transformer_max_period); + + mix_unsigned(model.n_tensors); + mix_unsigned(model.name_bytes_used); + const uint32_t name_bytes = std::min( + model.name_bytes_used, static_cast(model.name_storage.size())); + for (uint32_t index = 0u; index < name_bytes; ++index) { + guard_mix_model_identity_byte( + identity, static_cast(model.name_storage[index])); + } + + const uint32_t tensor_count = std::min( + model.n_tensors, static_cast(model.tensors.size())); + for (uint32_t index = 0u; index < tensor_count; ++index) { + const auto &tensor = model.tensors[index]; + mix_unsigned(tensor.name_offset); + mix_unsigned(tensor.name_length); + mix_signed(tensor.type); + mix_signed(tensor.n_dims); + for (const int64_t extent : tensor.dims) { + mix_signed(extent); + } + mix_unsigned(tensor.data_offset); + mix_unsigned(tensor.file_offset); + mix_unsigned(tensor.data_size); + mix_unsigned(reinterpret_cast(tensor.data)); + mix_unsigned(tensor.file_index); + } + return identity; +} + +// Pre-dispatch contract factory. It runs before process_event, captures all +// dry-run validation and sizing facts immutably, and leaves route choice to +// the explicit facade guards below. +inline event::bind_contract +make_bind_contract(const emel::model::data &model) noexcept { + const bind_model_identity identity = guard_compute_model_identity(model); + return event::bind_contract{model, + identity.low, + identity.high, + detail::required_prepared_floats(model), + detail::required_state_floats(model), + detail::required_workspace_floats(model), + detail::required_frame_floats(model), + detail::validate_codec_contract_f32(model), + detail::validate_codec_contract_native(model), + detail::validate_codec_contract_q8(model)}; +} + +inline bool +guard_bind_contract_matches_model(const event::bind_contract &contract, + const emel::model::data &model) noexcept { + if (contract.model() != &model) { + return false; + } + const bind_model_identity identity = guard_compute_model_identity(model); + return contract.model_identity_low() == identity.low && + contract.model_identity_high() == identity.high; +} + +inline constexpr std::string_view k_first_projection_name = + "mimi.encoder_transformer.transformer.layers.0.self_attn.in_projs.0." + "weight"; + +inline bool guard_first_projection_has_dtype(const emel::model::data &model, + const uint8_t dtype) noexcept { + if (model.n_tensors > model.tensors.size()) { + return false; + } + for (uint32_t index = 0u; index < model.n_tensors; ++index) { + const auto &tensor = model.tensors[index]; + if (emel::model::tensor_name_view(model, tensor) == + k_first_projection_name) { + return static_cast(tensor.type) == dtype; + } + } + return false; +} + +inline bool +guard_is_transformer_projection_name(const std::string_view name) noexcept { + const bool transformer = + name.starts_with("mimi.encoder_transformer.transformer.layers.") || + name.starts_with("mimi.decoder_transformer.transformer.layers."); + const bool projection = name.ends_with(".self_attn.in_projs.0.weight") || + name.ends_with(".self_attn.out_projs.0.weight") || + name.ends_with(".linear1.weight") || + name.ends_with(".linear2.weight"); + return transformer && projection; +} + +inline bool +guard_transformer_projections_have_dtype(const emel::model::data &model, + const uint8_t dtype, + const uint64_t alignment) noexcept { + constexpr uint32_t k_projection_kinds_per_layer = 4u; + constexpr uint32_t k_transformer_directions = 2u; + if (model.n_tensors > model.tensors.size() || + model.mimi.transformer_num_layers <= 0 || + static_cast(model.mimi.transformer_num_layers) > + std::numeric_limits::max() / + (k_projection_kinds_per_layer * k_transformer_directions)) { + return false; + } + const uint32_t expected_projection_count = + static_cast(model.mimi.transformer_num_layers) * + k_projection_kinds_per_layer * k_transformer_directions; + uint32_t projection_count = 0u; + for (uint32_t index = 0u; index < model.n_tensors; ++index) { + const auto &tensor = model.tensors[index]; + if (!guard_is_transformer_projection_name( + emel::model::tensor_name_view(model, tensor))) { + continue; + } + if (static_cast(tensor.type) != dtype || tensor.data == nullptr || + reinterpret_cast(tensor.data) % alignment != 0u) { + return false; + } + ++projection_count; + } + return projection_count == expected_projection_count; +} + +inline bool guard_range_valid(const void *data, const uint64_t bytes, + uintptr_t &begin, uintptr_t &end) noexcept { + begin = reinterpret_cast(data); + if (data == nullptr || bytes == 0u || + begin > std::numeric_limits::max() - bytes) { + return false; + } + end = begin + bytes; + return true; +} + +inline bool guard_ranges_disjoint(const void *first, const uint64_t first_bytes, + const void *second, + const uint64_t second_bytes) noexcept { + uintptr_t first_begin = 0u; + uintptr_t first_end = 0u; + uintptr_t second_begin = 0u; + uintptr_t second_end = 0u; + return guard_range_valid(first, first_bytes, first_begin, first_end) && + guard_range_valid(second, second_bytes, second_begin, second_end) && + (first_end <= second_begin || second_end <= first_begin); +} + +inline bool guard_arena_bytes(const std::span arena, + uint64_t &bytes) noexcept { + if (arena.size() > std::numeric_limits::max() / sizeof(float)) { + return false; + } + bytes = static_cast(arena.size()) * sizeof(float); + return bytes != 0u; +} + +inline bool +guard_arena_disjoint_from_model(const emel::model::data &model, + const std::span arena, + const uint64_t arena_bytes) noexcept { + if (model.n_tensors > model.tensors.size()) { + return false; + } + for (uint32_t index = 0u; index < model.n_tensors; ++index) { + const auto &tensor = model.tensors[index]; + if (tensor.data == nullptr || tensor.data_size == 0u) { + continue; + } + if (!guard_ranges_disjoint(arena.data(), arena_bytes, tensor.data, + tensor.data_size)) { + return false; + } + } + return true; +} + +inline bool guard_arenas_disjoint(const event::initialize &request) noexcept { + uint64_t prepared_bytes = 0u; + uint64_t state_bytes = 0u; + uint64_t workspace_bytes = 0u; + uint64_t frame_bytes = 0u; + if (!guard_arena_bytes(request.prepared, prepared_bytes) || + !guard_arena_bytes(request.state_arena, state_bytes) || + !guard_arena_bytes(request.workspace, workspace_bytes) || + !guard_arena_bytes(request.frame, frame_bytes)) { + return false; + } + return guard_ranges_disjoint(request.prepared.data(), prepared_bytes, + request.state_arena.data(), state_bytes) && + guard_ranges_disjoint(request.prepared.data(), prepared_bytes, + request.workspace.data(), workspace_bytes) && + guard_ranges_disjoint(request.prepared.data(), prepared_bytes, + request.frame.data(), frame_bytes) && + guard_ranges_disjoint(request.state_arena.data(), state_bytes, + request.workspace.data(), workspace_bytes) && + guard_ranges_disjoint(request.state_arena.data(), state_bytes, + request.frame.data(), frame_bytes) && + guard_ranges_disjoint(request.workspace.data(), workspace_bytes, + request.frame.data(), frame_bytes) && + guard_arena_disjoint_from_model(request.model, request.prepared, + prepared_bytes) && + guard_arena_disjoint_from_model(request.model, request.state_arena, + state_bytes) && + guard_arena_disjoint_from_model(request.model, request.workspace, + workspace_bytes) && + guard_arena_disjoint_from_model(request.model, request.frame, + frame_bytes); +} + +// Bind validation facts are computed before process_event and carried as an +// immutable event value. Guards own route selection and capacity acceptance; +// no detail helper result decides an RTC transition. +struct guard_bind_f32_contract_valid { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.contract.model() == &runtime_ev.request.model && + runtime_ev.request.contract.f32_valid() && + guard_transformer_projections_have_dtype( + runtime_ev.request.model, emel::kernel::detail::dtype_f32, + alignof(float)) && + guard_bind_contract_matches_model(runtime_ev.request.contract, + runtime_ev.request.model); + } +}; + +struct guard_bind_native_contract_valid { + bool operator()(const event::initialize_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.request.contract.model() == &runtime_ev.request.model && + runtime_ev.request.contract.native_valid() && + guard_transformer_projections_have_dtype( + runtime_ev.request.model, emel::kernel::detail::dtype_f16, + alignof(uint16_t)) && + guard_bind_contract_matches_model(runtime_ev.request.contract, + runtime_ev.request.model); + } +}; + +struct guard_bind_q8_contract_valid { bool operator()(const event::initialize_run &runtime_ev, const action::context &) const noexcept { - return detail::validate_codec_contract(runtime_ev.request.model); + return runtime_ev.request.contract.model() == &runtime_ev.request.model && + runtime_ev.request.contract.q8_valid() && + guard_first_projection_has_dtype(runtime_ev.request.model, 8u) && + guard_bind_contract_matches_model(runtime_ev.request.contract, + runtime_ev.request.model); } }; struct guard_bind_contract_invalid { bool operator()(const event::initialize_run &runtime_ev, const action::context &ctx) const noexcept { - return !guard_bind_contract_valid{}(runtime_ev, ctx); + return !guard_bind_f32_contract_valid{}(runtime_ev, ctx) && + !guard_bind_native_contract_valid{}(runtime_ev, ctx) && + !guard_bind_q8_contract_valid{}(runtime_ev, ctx); } }; @@ -36,11 +321,22 @@ struct guard_arena_capacity_valid { request.state_arena.data() != nullptr && request.workspace.data() != nullptr && request.frame.data() != nullptr && - request.prepared.size() >= detail::required_prepared_floats(model) && - request.state_arena.size() >= detail::required_state_floats(model) && - request.workspace.size() >= - detail::required_workspace_floats(model) && - request.frame.size() >= detail::required_frame_floats(model); + reinterpret_cast(request.prepared.data()) % + alignof(float) == + 0u && + reinterpret_cast(request.state_arena.data()) % + alignof(float) == + 0u && + reinterpret_cast(request.workspace.data()) % + alignof(float) == + 0u && + reinterpret_cast(request.frame.data()) % alignof(float) == + 0u && + request.prepared.size() >= request.contract.prepared_floats() && + request.state_arena.size() >= request.contract.state_floats() && + request.workspace.size() >= request.contract.workspace_floats() && + request.frame.size() >= request.contract.frame_floats() && + guard_arenas_disjoint(request); } }; diff --git a/src/emel/speech/codec/mimi/sm.hpp b/src/emel/speech/codec/mimi/sm.hpp index 30cc3d3c..88c2f4da 100644 --- a/src/emel/speech/codec/mimi/sm.hpp +++ b/src/emel/speech/codec/mimi/sm.hpp @@ -12,16 +12,18 @@ // Mimi codec facade: owns the bound runtime, the streaming state, and the // frontend/quantizer/backend child actors, and sequences one 80 ms frame per // dispatch through them. Every success/error route is decided by pure -// validation guards BEFORE the corresponding action runs (bind contract via -// the dry-run walk, arena capacity via the sizing contract, frame requests -// against the bound runtime), so the compute actions are non-failing and no -// action-written flag routes behavior. Requests before initialization are -// answered with explicit not_initialized errors rather than dropped. +// validation guards BEFORE the corresponding action runs (immutable bind +// facts prepared before RTC, arena capacity from that same contract, frame +// requests against the bound runtime), so the compute actions are non-failing +// and no action-written flag routes behavior. Requests before initialization +// are answered with explicit not_initialized errors rather than dropped. namespace emel::speech::codec::mimi { struct state_uninitialized {}; struct state_bind_contract_decision {}; -struct state_bind_capacity_decision {}; +struct state_bind_f32_capacity_decision {}; +struct state_bind_native_capacity_decision {}; +struct state_bind_q8_capacity_decision {}; struct state_binding {}; struct state_init_error_out_decision {}; struct state_init_callback_decision {}; @@ -58,18 +60,34 @@ struct model { // clang-format off return sml::make_transition_table( //------------------------------------------------------------------------------// - // Initialization: contract, then capacity, then the non-failing bind. + // Initialization: pre-dispatch contract, capacity, non-failing bind. sml::state <= *sml::state + sml::event - , sml::state <= sml::state - + sml::completion [ guard::guard_bind_contract_valid{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_bind_f32_contract_valid{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_bind_native_contract_valid{} ] + , sml::state <= sml::state + + sml::completion [ guard::guard_bind_q8_contract_valid{} ] , sml::state <= sml::state + sml::completion [ guard::guard_bind_contract_invalid{} ] / action::effect_mark_bind_failed{} - , sml::state <= sml::state + , sml::state <= sml::state + sml::completion [ guard::guard_arena_capacity_valid{} ] - / action::effect_bind{} - , sml::state <= sml::state + / action::effect_bind_f32{} + , sml::state <= sml::state + + sml::completion [ guard::guard_arena_capacity_invalid{} ] + / action::effect_mark_arena_capacity_invalid{} + , sml::state <= sml::state + + sml::completion [ guard::guard_arena_capacity_valid{} ] + / action::effect_bind_native{} + , sml::state <= sml::state + + sml::completion [ guard::guard_arena_capacity_invalid{} ] + / action::effect_mark_arena_capacity_invalid{} + , sml::state <= sml::state + + sml::completion [ guard::guard_arena_capacity_valid{} ] + / action::effect_bind_q8{} + , sml::state <= sml::state + sml::completion [ guard::guard_arena_capacity_invalid{} ] / action::effect_mark_arena_capacity_invalid{} , sml::state <= sml::state @@ -179,6 +197,15 @@ struct model { + sml::event / action::effect_reset_stream{} + //------------------------------------------------------------------------------// + // Public read-only diagnostics before or after initialization. + , sml::state <= sml::state + + sml::event + / action::effect_capture_diagnostics{} + , sml::state <= sml::state + + sml::event + / action::effect_capture_diagnostics{} + //------------------------------------------------------------------------------// // Requests before initialization answer with explicit errors. , sml::state <= sml::state @@ -354,11 +381,27 @@ struct model { + sml::unexpected_event [ guard::guard_unexpected_error_out_absent{} ] / action::effect_mark_unexpected{} - , sml::state <= sml::state + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_present{} ] + / action::effect_mark_unexpected_and_store{} + , sml::state <= sml::state + + sml::unexpected_event + [ guard::guard_unexpected_error_out_absent{} ] + / action::effect_mark_unexpected{} + , sml::state <= sml::state + sml::unexpected_event [ guard::guard_unexpected_error_out_present{} ] / action::effect_mark_unexpected_and_store{} - , sml::state <= sml::state + , sml::state <= sml::state + sml::unexpected_event [ guard::guard_unexpected_error_out_absent{} ] / action::effect_mark_unexpected{} @@ -473,6 +516,10 @@ struct sm : public emel::sm { const bool accepted = base_type::process_event(runtime_ev); return accepted && ctx.err == action::detail_ns::to_error(error::none); } + + bool process_event(const event::capture_diagnostics &ev) { + return base_type::process_event(ev); + } }; } // namespace emel::speech::codec::mimi diff --git a/src/emel/speech/generator/actions.hpp b/src/emel/speech/generator/actions.hpp index 3bf12f21..0b0a3415 100644 --- a/src/emel/speech/generator/actions.hpp +++ b/src/emel/speech/generator/actions.hpp @@ -12,6 +12,7 @@ #include "emel/speech/generator/context.hpp" #include "emel/speech/generator/errors.hpp" #include "emel/speech/generator/events.hpp" +#include "emel/speech/generator/frame/events.hpp" namespace emel::speech::generator::action { @@ -768,4 +769,483 @@ template struct effect_unexpected { } }; +struct lane_zero {}; +struct lane_one {}; + +template +void effect_record_worker_entry(context &ctx) noexcept { + if constexpr (requires { ctx.collaborators.stage_diagnostics; }) { + ctx.collaborators.stage_diagnostics.worker_entries.fetch_add( + 1u, std::memory_order_relaxed); + } +} + +template +void effect_record_worker_exit(context &ctx) noexcept { + if constexpr (requires { ctx.collaborators.stage_diagnostics; }) { + ctx.collaborators.stage_diagnostics.worker_exits.fetch_add( + 1u, std::memory_order_relaxed); + } +} + +template +void effect_record_submission(context &ctx, + const bool submitted) noexcept { + if constexpr (requires { ctx.collaborators.stage_diagnostics; }) { + ctx.collaborators.stage_diagnostics.submissions.fetch_add( + static_cast(submitted), std::memory_order_relaxed); + } +} + +template +void effect_record_joins(context &ctx, + const size_t joined) noexcept { + if constexpr (requires { ctx.collaborators.stage_diagnostics; }) { + ctx.collaborators.stage_diagnostics.joins.fetch_add( + static_cast(joined), std::memory_order_relaxed); + } +} + +template +std::span encoded_lane(context &ctx) noexcept { + if constexpr (std::same_as) { + return ctx.encoded_lane0(); + } else { + return ctx.encoded_lane1(); + } +} + +template +std::span generated_lane(context &ctx) noexcept { + if constexpr (std::same_as) { + return ctx.generated_lane0(); + } else { + return ctx.generated_lane1(); + } +} + +template +event::wavefront_attribution & +encoded_attribution(context &ctx) noexcept { + if constexpr (std::same_as) { + return ctx.encoded_lane0_attribution; + } else { + return ctx.encoded_lane1_attribution; + } +} + +template +event::wavefront_attribution & +generated_attribution(context &ctx) noexcept { + if constexpr (std::same_as) { + return ctx.generated_lane0_attribution; + } else { + return ctx.generated_lane1_attribution; + } +} + +template +int32_t &generated_text_token(context &ctx) noexcept { + if constexpr (std::same_as) { + return ctx.generated_lane0_text_token; + } else { + return ctx.generated_lane1_text_token; + } +} + +template +struct effect_execute_wavefront_phase_parallel { + template + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx = {}; + runtime_ev.request.output_attribution = {}; + runtime_ev.request.text_token_out = -1; + runtime_ev.request.sample_count_out = 0; + runtime_ev.request.produced_out = false; + if constexpr (requires { runtime_ev.request.complete_out; }) { + runtime_ev.request.complete_out = false; + } + + wavefront_stage_pool::join_group group{}; + emel::policy::fork_join_start_gate gate{}; + size_t submitted_tasks = 0u; + bool all_submitted = true; + + if constexpr (encode_active) { + const bool submitted = ctx.collaborators.stage_pool->try_submit( + group, [&runtime_ev, &ctx, &gate]() noexcept { + effect_record_worker_entry(ctx); + gate.arrive_and_wait(); + typename dependencies_type::wavefront_encode_event request{ + runtime_ev.request.pcm_in, encoded_lane(ctx)}; + request.error_out = &runtime_ev.ctx.encode_err; + runtime_ev.ctx.encode_accepted = + ctx.collaborators.wavefront_encoder.process_event(request); + encoded_attribution(ctx) = + runtime_ev.request.input_attribution; + effect_record_worker_exit(ctx); + }); + submitted_tasks += static_cast(submitted); + all_submitted = all_submitted && submitted; + effect_record_submission(ctx, submitted); + } + + if constexpr (decode_active) { + const bool submitted = ctx.collaborators.stage_pool->try_submit( + group, [&runtime_ev, &ctx, &gate]() noexcept { + effect_record_worker_entry(ctx); + gate.arrive_and_wait(); + typename dependencies_type::wavefront_decode_event request{ + std::span{generated_lane(ctx)}, + ctx.decoded_pcm()}; + request.error_out = &runtime_ev.ctx.decode_err; + runtime_ev.ctx.decode_accepted = + ctx.collaborators.wavefront_decoder.process_event(request); + runtime_ev.ctx.decoded_attribution = + generated_attribution(ctx); + runtime_ev.ctx.decoded_text_token = + generated_text_token(ctx); + effect_record_worker_exit(ctx); + }); + submitted_tasks += static_cast(submitted); + all_submitted = all_submitted && submitted; + effect_record_submission(ctx, submitted); + } + + gate.open_after_arrivals(submitted_tasks); + + if constexpr (middle_active) { + frame::event::run request{ + std::span{encoded_lane(ctx)}, + generated_lane(ctx), + generated_text_token(ctx), + runtime_ev.ctx.middle_err}; + runtime_ev.ctx.middle_accepted = + ctx.collaborators.wavefront_middle.process_event(request); + generated_attribution(ctx) = + encoded_attribution(ctx); + } + + (void)group.wait(); + effect_record_joins(ctx, submitted_tasks); + runtime_ev.ctx.all_submitted = all_submitted; + runtime_ev.ctx.joined = true; + } +}; + +template +struct effect_execute_wavefront_phase_serial { + template + void operator()(const runtime_event_type &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx = {}; + runtime_ev.request.output_attribution = {}; + runtime_ev.request.text_token_out = -1; + runtime_ev.request.sample_count_out = 0; + runtime_ev.request.produced_out = false; + if constexpr (requires { runtime_ev.request.complete_out; }) { + runtime_ev.request.complete_out = false; + } + + if constexpr (encode_active) { + typename dependencies_type::wavefront_encode_event request{ + runtime_ev.request.pcm_in, encoded_lane(ctx)}; + request.error_out = &runtime_ev.ctx.encode_err; + runtime_ev.ctx.encode_accepted = + ctx.collaborators.wavefront_encoder.process_event(request); + encoded_attribution(ctx) = + runtime_ev.request.input_attribution; + } + + if constexpr (middle_active) { + frame::event::run request{ + std::span{encoded_lane(ctx)}, + generated_lane(ctx), + generated_text_token(ctx), + runtime_ev.ctx.middle_err}; + runtime_ev.ctx.middle_accepted = + ctx.collaborators.wavefront_middle.process_event(request); + generated_attribution(ctx) = + encoded_attribution(ctx); + } + + if constexpr (decode_active) { + typename dependencies_type::wavefront_decode_event request{ + std::span{generated_lane(ctx)}, + ctx.decoded_pcm()}; + request.error_out = &runtime_ev.ctx.decode_err; + runtime_ev.ctx.decode_accepted = + ctx.collaborators.wavefront_decoder.process_event(request); + runtime_ev.ctx.decoded_attribution = + generated_attribution(ctx); + runtime_ev.ctx.decoded_text_token = + generated_text_token(ctx); + } + + runtime_ev.ctx.all_submitted = true; + runtime_ev.ctx.joined = true; + } +}; + +template struct effect_prepare_wavefront_empty_flush { + void operator()(const detail::wavefront_flush_run &runtime_ev, + const context &) const noexcept { + runtime_ev.ctx = {}; + runtime_ev.request.output_attribution = {}; + runtime_ev.request.text_token_out = -1; + runtime_ev.request.sample_count_out = 0; + runtime_ev.request.produced_out = false; + runtime_ev.request.complete_out = false; + } +}; + +template +void effect_commit_wavefront_input( + const detail::wavefront_frame_run &runtime_ev, + context &ctx) noexcept { + ctx.expected_input.sequence = + runtime_ev.request.input_attribution.sequence + 1u; + ctx.expected_input.source = runtime_ev.request.input_attribution.source; +} + +template +struct effect_publish_wavefront_frame_pending { + void operator()(const detail::wavefront_frame_run &runtime_ev, + context &ctx) const noexcept { + effect_commit_wavefront_input(runtime_ev, ctx); + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + if constexpr (emit_callback) { + runtime_ev.request.on_done( + events::wavefront_frame_done{runtime_ev.request, {}, 0, false}); + } + } +}; + +template +struct effect_publish_wavefront_frame_produced { + void operator()(const detail::wavefront_frame_run &runtime_ev, + context &ctx) const noexcept { + std::copy(ctx.decoded_pcm().begin(), ctx.decoded_pcm().end(), + runtime_ev.request.pcm_out.begin()); + std::copy(generated_lane(ctx).begin(), + generated_lane(ctx).end(), + runtime_ev.request.generated_tokens_out.begin()); + runtime_ev.request.output_attribution = runtime_ev.ctx.decoded_attribution; + runtime_ev.request.text_token_out = runtime_ev.ctx.decoded_text_token; + runtime_ev.request.sample_count_out = ctx.collaborators.frame_samples; + runtime_ev.request.produced_out = true; + effect_commit_wavefront_input(runtime_ev, ctx); + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + if constexpr (emit_callback) { + runtime_ev.request.on_done(events::wavefront_frame_done{ + runtime_ev.request, runtime_ev.ctx.decoded_attribution, + ctx.collaborators.frame_samples, true}); + } + } +}; + +template +struct effect_publish_wavefront_flush { + void operator()(const detail::wavefront_flush_run &runtime_ev, + context &ctx) const noexcept { + if constexpr (produced) { + std::copy(ctx.decoded_pcm().begin(), ctx.decoded_pcm().end(), + runtime_ev.request.pcm_out.begin()); + std::copy(generated_lane(ctx).begin(), + generated_lane(ctx).end(), + runtime_ev.request.generated_tokens_out.begin()); + runtime_ev.request.output_attribution = + runtime_ev.ctx.decoded_attribution; + runtime_ev.request.text_token_out = runtime_ev.ctx.decoded_text_token; + runtime_ev.request.sample_count_out = ctx.collaborators.frame_samples; + runtime_ev.request.produced_out = true; + } + runtime_ev.request.complete_out = complete; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + if constexpr (emit_callback) { + runtime_ev.request.on_done(events::wavefront_flush_done{ + runtime_ev.request, runtime_ev.request.output_attribution, + runtime_ev.request.sample_count_out, runtime_ev.request.produced_out, + complete}); + } + } +}; + +template +struct effect_fail_wavefront { + void operator()(const runtime_event_type &runtime_ev, + const context &) const noexcept { + runtime_ev.request.output_attribution = {}; + runtime_ev.request.text_token_out = -1; + runtime_ev.request.sample_count_out = 0; + runtime_ev.request.produced_out = false; + if constexpr (requires { runtime_ev.request.complete_out; }) { + runtime_ev.request.complete_out = false; + } + runtime_ev.ctx.err = error_code(error_value) | error_code(qualifier) | + error_code(second_qualifier); + runtime_ev.request.error_out = runtime_ev.ctx.err; + if constexpr (emit_callback && std::same_as) { + runtime_ev.request.on_error(events::wavefront_frame_error{ + runtime_ev.request, runtime_ev.ctx.err}); + } + if constexpr (emit_callback && std::same_as) { + runtime_ev.request.on_error(events::wavefront_flush_error{ + runtime_ev.request, runtime_ev.ctx.err}); + } + } +}; + +template +struct effect_emit_wavefront_error { + void operator()(const runtime_event_type &runtime_ev, + const context &) const noexcept { + if constexpr (std::same_as) { + runtime_ev.request.on_error(events::wavefront_frame_error{ + runtime_ev.request, runtime_ev.ctx.err}); + } else { + runtime_ev.request.on_error(events::wavefront_flush_error{ + runtime_ev.request, runtime_ev.ctx.err}); + } + } +}; + +template +struct effect_reset_wavefront_children_parallel { + void operator()(const detail::event_wavefront_reset_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx = {}; + wavefront_stage_pool::join_group group{}; + size_t submitted_tasks = 0u; + bool all_submitted = true; + + const bool encode_submitted = ctx.collaborators.stage_pool->try_submit( + group, [&runtime_ev, &ctx]() noexcept { + effect_record_worker_entry(ctx); + typename dependencies_type::wavefront_encode_reset_event request{ + runtime_ev.ctx.encode_err}; + runtime_ev.ctx.encode_accepted = + ctx.collaborators.wavefront_encoder.process_event(request); + effect_record_worker_exit(ctx); + }); + submitted_tasks += static_cast(encode_submitted); + all_submitted = all_submitted && encode_submitted; + effect_record_submission(ctx, encode_submitted); + + const bool decode_submitted = ctx.collaborators.stage_pool->try_submit( + group, [&runtime_ev, &ctx]() noexcept { + effect_record_worker_entry(ctx); + typename dependencies_type::wavefront_decode_reset_event request{ + runtime_ev.ctx.decode_err}; + runtime_ev.ctx.decode_accepted = + ctx.collaborators.wavefront_decoder.process_event(request); + effect_record_worker_exit(ctx); + }); + submitted_tasks += static_cast(decode_submitted); + all_submitted = all_submitted && decode_submitted; + effect_record_submission(ctx, decode_submitted); + + typename dependencies_type::wavefront_middle_reset_event middle_request{ + runtime_ev.ctx.middle_err}; + runtime_ev.ctx.middle_accepted = + ctx.collaborators.wavefront_middle.process_event(middle_request); + + (void)group.wait(); + effect_record_joins(ctx, submitted_tasks); + runtime_ev.ctx.all_submitted = all_submitted; + runtime_ev.ctx.joined = true; + } +}; + +template +struct effect_reset_wavefront_children_serial { + void operator()(const detail::event_wavefront_reset_run &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx = {}; + + typename dependencies_type::wavefront_encode_reset_event encode_request{ + runtime_ev.ctx.encode_err}; + runtime_ev.ctx.encode_accepted = + ctx.collaborators.wavefront_encoder.process_event(encode_request); + + typename dependencies_type::wavefront_middle_reset_event middle_request{ + runtime_ev.ctx.middle_err}; + runtime_ev.ctx.middle_accepted = + ctx.collaborators.wavefront_middle.process_event(middle_request); + + typename dependencies_type::wavefront_decode_reset_event decode_request{ + runtime_ev.ctx.decode_err}; + runtime_ev.ctx.decode_accepted = + ctx.collaborators.wavefront_decoder.process_event(decode_request); + + runtime_ev.ctx.all_submitted = true; + runtime_ev.ctx.joined = true; + } +}; + +template struct effect_reset_wavefront_parent { + void operator()(const detail::event_wavefront_reset_run &runtime_ev, + context &ctx) const noexcept { + std::fill(ctx.encoded_lane0_storage.begin(), + ctx.encoded_lane0_storage.end(), -1); + std::fill(ctx.encoded_lane1_storage.begin(), + ctx.encoded_lane1_storage.end(), -1); + std::fill(ctx.generated_lane0_storage.begin(), + ctx.generated_lane0_storage.end(), -1); + std::fill(ctx.generated_lane1_storage.begin(), + ctx.generated_lane1_storage.end(), -1); + std::fill(ctx.decoded_pcm_storage.begin(), ctx.decoded_pcm_storage.end(), + 0.0f); + ctx.encoded_lane0_attribution = {}; + ctx.encoded_lane1_attribution = {}; + ctx.generated_lane0_attribution = {}; + ctx.generated_lane1_attribution = {}; + ctx.expected_input = {.sequence = 0u, .source = 0u}; + ctx.generated_lane0_text_token = -1; + ctx.generated_lane1_text_token = -1; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template +struct effect_fail_wavefront_reset { + void operator()(const detail::event_wavefront_reset_run &runtime_ev, + const context &) const noexcept { + runtime_ev.ctx.err = error_code(error_value); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_unexpected_wavefront { + template + void operator()(const unexpected_type &unexpected, + const context &) const noexcept { + const auto &ev = stateforward::sml::back::get_origin_event(unexpected); + if constexpr (requires { ev.error_out; }) { + ev.error_out = error_code(error::internal_error); + } + if constexpr (requires { ev.produced_out; }) { + ev.produced_out = false; + } + if constexpr (requires { ev.complete_out; }) { + ev.complete_out = false; + } + } +}; + } // namespace emel::speech::generator::action diff --git a/src/emel/speech/generator/any.hpp b/src/emel/speech/generator/any.hpp index c2f68a04..24291525 100644 --- a/src/emel/speech/generator/any.hpp +++ b/src/emel/speech/generator/any.hpp @@ -6,5 +6,6 @@ namespace emel::speech::generator { using duplex = action::mode::duplex; using synthesis = action::mode::synthesis; +using wavefront = action::mode::wavefront; } // namespace emel::speech::generator diff --git a/src/emel/speech/generator/context.hpp b/src/emel/speech/generator/context.hpp index 2819c58a..a550d3db 100644 --- a/src/emel/speech/generator/context.hpp +++ b/src/emel/speech/generator/context.hpp @@ -1,8 +1,14 @@ #pragma once +#include +#include #include +#include +#include #include "emel/batch/planner/events.hpp" +#include "emel/sm.hpp" +#include "emel/speech/generator/events.hpp" namespace emel::speech::generator::action { @@ -10,6 +16,7 @@ namespace mode { struct duplex {}; struct synthesis {}; +struct wavefront {}; } // namespace mode @@ -84,15 +91,111 @@ concept synthesis_dependencies = requires(dependencies_type deps) { deps.postprocessor_initialize; }; +using wavefront_stage_pool = + emel::policy::fork_join_lane_pool<2u, 256u, 1048576u>; + +enum class wavefront_stage_mode : uint8_t { serial, parallel }; + +struct wavefront_diagnostics { + std::atomic submissions = 0u; + std::atomic joins = 0u; + std::atomic worker_entries = 0u; + std::atomic worker_exits = 0u; +}; + +template +concept wavefront_dependencies = requires(dependencies_type deps) { + requires std::same_as; + requires dependencies_type::wavefront_frame_capacity > 0u; + requires dependencies_type::wavefront_codebook_capacity > 0u; + typename dependencies_type::wavefront_encode_event; + typename dependencies_type::wavefront_encode_reset_event; + typename dependencies_type::wavefront_middle_reset_event; + typename dependencies_type::wavefront_decode_event; + typename dependencies_type::wavefront_decode_reset_event; + deps.wavefront_encoder; + deps.wavefront_middle; + deps.wavefront_decoder; + { deps.stage_pool } -> std::convertible_to; + { deps.stage_mode } -> std::convertible_to; + deps.frame_samples; + deps.codebook_count; +}; + template concept generator_dependencies = duplex_dependencies || - synthesis_dependencies; + synthesis_dependencies || + wavefront_dependencies; + +template +struct context; + +template +struct context { + explicit context(const dependencies_type &deps) noexcept + : collaborators(deps) {} -template struct context { + const dependencies_type collaborators; +}; + +template +struct context { + explicit context(const dependencies_type &deps) noexcept + : collaborators(deps) {} + + const dependencies_type collaborators; +}; + +template +struct context { explicit context(const dependencies_type &deps) noexcept : collaborators(deps) {} + std::span encoded_lane0() noexcept { + return {encoded_lane0_storage.data(), + static_cast(collaborators.codebook_count)}; + } + + std::span encoded_lane1() noexcept { + return {encoded_lane1_storage.data(), + static_cast(collaborators.codebook_count)}; + } + + std::span generated_lane0() noexcept { + return {generated_lane0_storage.data(), + static_cast(collaborators.codebook_count)}; + } + + std::span generated_lane1() noexcept { + return {generated_lane1_storage.data(), + static_cast(collaborators.codebook_count)}; + } + + std::span decoded_pcm() noexcept { + return {decoded_pcm_storage.data(), + static_cast(collaborators.frame_samples)}; + } + const dependencies_type collaborators; + std::array + encoded_lane0_storage{}; + std::array + encoded_lane1_storage{}; + std::array + generated_lane0_storage{}; + std::array + generated_lane1_storage{}; + std::array + decoded_pcm_storage{}; + event::wavefront_attribution encoded_lane0_attribution = {}; + event::wavefront_attribution encoded_lane1_attribution = {}; + event::wavefront_attribution generated_lane0_attribution = {}; + event::wavefront_attribution generated_lane1_attribution = {}; + event::wavefront_attribution expected_input{.sequence = 0u, .source = 0u}; + int32_t generated_lane0_text_token = -1; + int32_t generated_lane1_text_token = -1; }; } // namespace emel::speech::generator::action diff --git a/src/emel/speech/generator/errors.hpp b/src/emel/speech/generator/errors.hpp index 9328a93f..4e510d20 100644 --- a/src/emel/speech/generator/errors.hpp +++ b/src/emel/speech/generator/errors.hpp @@ -31,6 +31,11 @@ enum class error : emel::error::type { detokenize_failed = (1u << 24), planning_failed = (1u << 25), graph_failed = (1u << 26), + stage_submit_failed = (1u << 27), + attribution_failed = (1u << 28), + middle_failed = (1u << 29), + non_production_failed = (1u << 30), + drain_failed = (1u << 31), }; } // namespace emel::speech::generator diff --git a/src/emel/speech/generator/events.hpp b/src/emel/speech/generator/events.hpp index 3d400efb..2fbd2e65 100644 --- a/src/emel/speech/generator/events.hpp +++ b/src/emel/speech/generator/events.hpp @@ -19,6 +19,10 @@ struct stream_frame_done; struct stream_frame_error; struct flush_done; struct flush_error; +struct wavefront_frame_done; +struct wavefront_frame_error; +struct wavefront_flush_done; +struct wavefront_flush_error; } // namespace emel::speech::generator::events @@ -118,6 +122,72 @@ struct reset { emel::error::type &error_out; }; +inline constexpr uint64_t k_invalid_wavefront_sequence = UINT64_MAX; + +struct wavefront_attribution { + uint64_t sequence = k_invalid_wavefront_sequence; + uint64_t source = 0u; +}; + +struct wavefront_frame { + wavefront_frame(std::span pcm_in_ref, + std::span pcm_out_ref, + std::span generated_tokens_out_ref, + const wavefront_attribution input_attribution_ref, + wavefront_attribution &output_attribution_ref, + int32_t &text_token_out_ref, int32_t &sample_count_out_ref, + bool &produced_out_ref, + emel::error::type &error_out_ref) noexcept + : pcm_in(pcm_in_ref), pcm_out(pcm_out_ref), + generated_tokens_out(generated_tokens_out_ref), + input_attribution(input_attribution_ref), + output_attribution(output_attribution_ref), + text_token_out(text_token_out_ref), + sample_count_out(sample_count_out_ref), produced_out(produced_out_ref), + error_out(error_out_ref) {} + + const std::span pcm_in; + const std::span pcm_out; + const std::span generated_tokens_out; + const wavefront_attribution input_attribution; + wavefront_attribution &output_attribution; + int32_t &text_token_out; + int32_t &sample_count_out; + bool &produced_out; + emel::error::type &error_out; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct wavefront_flush { + wavefront_flush(std::span pcm_out_ref, + std::span generated_tokens_out_ref, + wavefront_attribution &output_attribution_ref, + int32_t &text_token_out_ref, int32_t &sample_count_out_ref, + bool &produced_out_ref, bool &complete_out_ref, + emel::error::type &error_out_ref) noexcept + : pcm_out(pcm_out_ref), generated_tokens_out(generated_tokens_out_ref), + output_attribution(output_attribution_ref), + text_token_out(text_token_out_ref), + sample_count_out(sample_count_out_ref), produced_out(produced_out_ref), + complete_out(complete_out_ref), error_out(error_out_ref) {} + + const std::span pcm_out; + const std::span generated_tokens_out; + wavefront_attribution &output_attribution; + int32_t &text_token_out; + int32_t &sample_count_out; + bool &produced_out; + bool &complete_out; + emel::error::type &error_out; + emel::callback on_done = {}; + emel::callback on_error = {}; +}; + +struct wavefront_reset { + emel::error::type &error_out; +}; + struct initialize_ctx { emel::error::type err = {}; emel::error::type child_err = {}; @@ -185,6 +255,51 @@ struct flush_run { } // namespace emel::speech::generator::event +namespace emel::speech::generator::detail { + +struct wavefront_run_ctx { + emel::error::type err = {}; + emel::error::type encode_err = {}; + emel::error::type middle_err = {}; + emel::error::type decode_err = {}; + bool encode_accepted = true; + bool middle_accepted = true; + bool decode_accepted = true; + bool all_submitted = true; + bool joined = true; + event::wavefront_attribution decoded_attribution = {}; + int32_t decoded_text_token = -1; +}; + +struct wavefront_frame_run { + const event::wavefront_frame &request; + wavefront_run_ctx &ctx; +}; + +struct wavefront_flush_run { + const event::wavefront_flush &request; + wavefront_run_ctx &ctx; +}; + +struct wavefront_reset_ctx { + emel::error::type err = {}; + emel::error::type encode_err = {}; + emel::error::type middle_err = {}; + emel::error::type decode_err = {}; + bool encode_accepted = false; + bool middle_accepted = false; + bool decode_accepted = false; + bool all_submitted = true; + bool joined = true; +}; + +struct event_wavefront_reset_run { + const event::wavefront_reset &request; + wavefront_reset_ctx &ctx; +}; + +} // namespace emel::speech::generator::detail + namespace emel::speech::generator::events { struct initialize_done { @@ -241,4 +356,29 @@ struct flush_error { emel::error::type err = {}; }; +struct wavefront_frame_done { + const event::wavefront_frame &request; + event::wavefront_attribution attribution = {}; + int32_t sample_count = 0; + bool produced = false; +}; + +struct wavefront_frame_error { + const event::wavefront_frame &request; + emel::error::type err = {}; +}; + +struct wavefront_flush_done { + const event::wavefront_flush &request; + event::wavefront_attribution attribution = {}; + int32_t sample_count = 0; + bool produced = false; + bool complete = false; +}; + +struct wavefront_flush_error { + const event::wavefront_flush &request; + emel::error::type err = {}; +}; + } // namespace emel::speech::generator::events diff --git a/src/emel/speech/generator/frame/actions.hpp b/src/emel/speech/generator/frame/actions.hpp new file mode 100644 index 00000000..2745ed3b --- /dev/null +++ b/src/emel/speech/generator/frame/actions.hpp @@ -0,0 +1,205 @@ +#pragma once + +#include +#include + +#include + +#include "emel/batch/planner/events.hpp" +#include "emel/error/error.hpp" +#include "emel/speech/generator/frame/context.hpp" +#include "emel/speech/generator/frame/errors.hpp" +#include "emel/speech/generator/frame/events.hpp" + +namespace emel::speech::generator::frame::action { + +inline emel::error::type error_code(const error value) noexcept { + return emel::error::cast(value); +} + +struct frame_plan_capture { + detail::run_ctx &ctx; + + void on_done(const emel::batch::planner::events::plan_done &ev) noexcept { + ctx.child_err = emel::error::cast(emel::batch::planner::error::none); + ctx.plan_step_size = ev.step_sizes[0]; + ctx.plan_output_count = ev.total_outputs; + } + + void on_error(const emel::batch::planner::events::plan_error &ev) noexcept { + ctx.child_err = ev.err; + } +}; + +template struct effect_tokenize { + void operator()(const detail::run_frame &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.tokenizer_err = 0; + typename dependencies_type::tokenize_event request{ + runtime_ev.request.encoded_tokens, ctx.collaborators.model_codes, + runtime_ev.ctx.tokenizer_err}; + runtime_ev.ctx.child_accepted = + ctx.collaborators.tokenizer.process_event(request); + runtime_ev.ctx.child_err = + static_cast(runtime_ev.ctx.tokenizer_err); + } +}; + +template struct effect_plan { + void operator()(const detail::run_frame &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = + emel::error::cast(emel::batch::planner::error::none); + runtime_ev.ctx.plan_step_size = 0; + runtime_ev.ctx.plan_output_count = 0; + frame_plan_capture capture{runtime_ev.ctx}; + const auto on_done = + emel::callback:: + template from( + &capture); + const auto on_error = + emel::callback:: + template from( + &capture); + const emel::batch::planner::event::plan_request request{ + .token_ids = ctx.collaborators.model_codes.data(), + .n_tokens = ctx.collaborators.frame_plan_token_count, + .n_steps = ctx.collaborators.frame_plan_steps, + .mode = ctx.collaborators.frame_plan_mode, + .output_all = ctx.collaborators.frame_plan_output_all, + .on_done = on_done, + .on_error = on_error, + }; + runtime_ev.ctx.child_accepted = + ctx.collaborators.planner.process_event(request); + } +}; + +template struct effect_predict { + void operator()(const detail::run_frame &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + typename dependencies_type::predict_event request{ + std::span{ctx.collaborators.model_codes}, + ctx.collaborators.prediction_workspace, runtime_ev.ctx.plan_step_size, + runtime_ev.ctx.plan_output_count}; + request.error_out = &runtime_ev.ctx.child_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.predictor.process_event(request); + } +}; + +template struct effect_execute_graph { + void operator()(const detail::run_frame &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + runtime_ev.ctx.graph_err = 0; + typename dependencies_type::graph_event request{ + ctx.collaborators.prediction_workspace, + std::span{ctx.collaborators.model_codes}}; + request.error_out = &runtime_ev.ctx.child_err; + request.graph_error_out = &runtime_ev.ctx.graph_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.graph.process_event(request); + } +}; + +template struct effect_sample { + void operator()(const detail::run_frame &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.child_err = 0; + runtime_ev.ctx.graph_err = 0; + runtime_ev.ctx.predicted_text_token = -1; + std::fill(ctx.collaborators.predicted_codes.begin(), + ctx.collaborators.predicted_codes.end(), -1); + typename dependencies_type::sample_event request{ + ctx.collaborators.prediction_workspace, + std::span{ctx.collaborators.model_codes}, + ctx.collaborators.predicted_codes, runtime_ev.ctx.predicted_text_token}; + request.error_out = &runtime_ev.ctx.child_err; + request.graph_error_out = &runtime_ev.ctx.graph_err; + runtime_ev.ctx.child_accepted = + ctx.collaborators.sampler.process_event(request); + } +}; + +template struct effect_detokenize { + void operator()(const detail::run_frame &runtime_ev, + context &ctx) const noexcept { + runtime_ev.ctx.tokenizer_err = 0; + runtime_ev.ctx.produced = false; + runtime_ev.ctx.text_token = -1; + std::fill(runtime_ev.request.generated_tokens_out.begin(), + runtime_ev.request.generated_tokens_out.end(), -1); + typename dependencies_type::detokenize_event request{ + runtime_ev.ctx.predicted_text_token, + std::span{ctx.collaborators.predicted_codes}, + runtime_ev.ctx.text_token, + runtime_ev.request.generated_tokens_out, + runtime_ev.ctx.produced, + runtime_ev.ctx.tokenizer_err}; + runtime_ev.ctx.child_accepted = + ctx.collaborators.tokenizer.process_event(request); + runtime_ev.ctx.child_err = + static_cast(runtime_ev.ctx.tokenizer_err); + } +}; + +template struct effect_publish_done { + void operator()(const detail::run_frame &runtime_ev, + const context &) const noexcept { + runtime_ev.request.text_token_out = runtime_ev.ctx.text_token; + runtime_ev.ctx.err = error_code(error::none); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_fail { + void operator()(const detail::run_frame &runtime_ev, + const context &) const noexcept { + runtime_ev.request.text_token_out = -1; + runtime_ev.ctx.err = error_code(error_value); + runtime_ev.request.error_out = runtime_ev.ctx.err; + } +}; + +template struct effect_emit_done { + void operator()(const detail::run_frame &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_done( + events::run_done{runtime_ev.request, runtime_ev.ctx.text_token}); + } +}; + +template struct effect_emit_error { + void operator()(const detail::run_frame &runtime_ev, + const context &) const noexcept { + runtime_ev.request.on_error( + events::run_error{runtime_ev.request, runtime_ev.ctx.err}); + } +}; + +template struct effect_reset { + void operator()(const event::reset &ev, + const context &) const noexcept { + ev.error_out = error_code(error::none); + } +}; + +template struct effect_unexpected { + template + void operator()(const unexpected_type &unexpected, + const context &) const noexcept { + effect_reject_origin(stateforward::sml::back::get_origin_event(unexpected)); + } + +private: + template + static void effect_reject_origin(const event_type &ev) noexcept { + if constexpr (requires { ev.error_out; }) { + ev.error_out = error_code(error::unexpected_event); + } + } +}; + +} // namespace emel::speech::generator::frame::action diff --git a/src/emel/speech/generator/frame/any.hpp b/src/emel/speech/generator/frame/any.hpp new file mode 100644 index 00000000..4a9297a6 --- /dev/null +++ b/src/emel/speech/generator/frame/any.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include "emel/speech/generator/frame/sm.hpp" + +namespace emel { + +template +using SpeechGeneratorFrame = speech::generator::frame::sm; + +} // namespace emel diff --git a/src/emel/speech/generator/frame/context.hpp b/src/emel/speech/generator/frame/context.hpp new file mode 100644 index 00000000..11ba887a --- /dev/null +++ b/src/emel/speech/generator/frame/context.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include +#include + +#include "emel/error/error.hpp" +#include "emel/speech/generator/frame/events.hpp" + +namespace emel::speech::generator::frame::action { + +template +concept frame_dependencies = requires(dependencies_type deps) { + typename dependencies_type::tokenize_event; + typename dependencies_type::predict_event; + typename dependencies_type::graph_event; + typename dependencies_type::sample_event; + typename dependencies_type::detokenize_event; + deps.planner; + deps.tokenizer; + deps.predictor; + deps.graph; + deps.sampler; + deps.prediction_workspace; + deps.model_codes; + deps.predicted_codes; + deps.codebook_count; + deps.frame_plan_mode; + deps.frame_plan_steps; + deps.frame_plan_token_count; + deps.frame_plan_output_all; +}; + +template struct context { + explicit context(const dependencies_type &deps) noexcept + : collaborators(deps) {} + + const dependencies_type collaborators; +}; + +} // namespace emel::speech::generator::frame::action + +namespace emel::speech::generator::frame::detail { + +struct run_ctx { + emel::error::type err = {}; + emel::error::type child_err = {}; + emel::error::type graph_err = {}; + bool child_accepted = false; + bool produced = false; + int32_t tokenizer_err = 0; + int32_t predicted_text_token = -1; + int32_t text_token = -1; + int32_t plan_step_size = 0; + int32_t plan_output_count = 0; +}; + +struct run_frame { + const event::run &request; + run_ctx &ctx; +}; + +} // namespace emel::speech::generator::frame::detail diff --git a/src/emel/speech/generator/frame/errors.hpp b/src/emel/speech/generator/frame/errors.hpp new file mode 100644 index 00000000..ba59647d --- /dev/null +++ b/src/emel/speech/generator/frame/errors.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "emel/error/error.hpp" + +namespace emel::speech::generator::frame { + +enum class error : emel::error::type { + none = 0u, + invalid_request = (1u << 0), + tokenize_failed = (1u << 1), + planning_failed = (1u << 2), + predict_failed = (1u << 3), + graph_failed = (1u << 4), + sample_failed = (1u << 5), + detokenize_failed = (1u << 6), + frame_pending = (1u << 7), + unexpected_event = (1u << 8), +}; + +} // namespace emel::speech::generator::frame diff --git a/src/emel/speech/generator/frame/events.hpp b/src/emel/speech/generator/frame/events.hpp new file mode 100644 index 00000000..e67eb3be --- /dev/null +++ b/src/emel/speech/generator/frame/events.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include +#include + +#include "emel/callback.hpp" +#include "emel/error/error.hpp" + +namespace emel::speech::generator::frame::events { + +struct run_done; +struct run_error; + +} // namespace emel::speech::generator::frame::events + +namespace emel::speech::generator::frame::event { + +struct run { + run(std::span encoded_tokens_ref, + std::span generated_tokens_out_ref, int32_t &text_token_out_ref, + emel::error::type &error_out_ref, + const emel::callback on_done_ref = {}, + const emel::callback on_error_ref = + {}) noexcept + : encoded_tokens(encoded_tokens_ref), + generated_tokens_out(generated_tokens_out_ref), + text_token_out(text_token_out_ref), error_out(error_out_ref), + on_done(on_done_ref), on_error(on_error_ref) {} + + const std::span encoded_tokens; + const std::span generated_tokens_out; + int32_t &text_token_out; + emel::error::type &error_out; + const emel::callback on_done; + const emel::callback on_error; +}; + +struct reset { + explicit reset(emel::error::type &error_out_ref) noexcept + : error_out(error_out_ref) {} + + emel::error::type &error_out; +}; + +} // namespace emel::speech::generator::frame::event + +namespace emel::speech::generator::frame::events { + +struct run_done { + const event::run &request; + int32_t text_token = -1; +}; + +struct run_error { + const event::run &request; + emel::error::type err = {}; +}; + +} // namespace emel::speech::generator::frame::events diff --git a/src/emel/speech/generator/frame/guards.hpp b/src/emel/speech/generator/frame/guards.hpp new file mode 100644 index 00000000..93292fee --- /dev/null +++ b/src/emel/speech/generator/frame/guards.hpp @@ -0,0 +1,119 @@ +#pragma once + +#include + +#include "emel/speech/generator/frame/context.hpp" +#include "emel/speech/generator/frame/events.hpp" + +namespace emel::speech::generator::frame::guard { + +template struct guard_request_valid { + bool + operator()(const detail::run_frame &runtime_ev, + const action::context &ctx) const noexcept { + const auto count = static_cast(ctx.collaborators.codebook_count); + return ctx.collaborators.codebook_count > 0 && + runtime_ev.request.encoded_tokens.size() == count && + runtime_ev.request.generated_tokens_out.size() == count && + ctx.collaborators.model_codes.size() >= count && + ctx.collaborators.predicted_codes.size() >= count && + ctx.collaborators.frame_plan_steps > 0 && + ctx.collaborators.frame_plan_token_count > 0 && + static_cast(ctx.collaborators.frame_plan_token_count) <= + ctx.collaborators.model_codes.size(); + } +}; + +template struct guard_request_invalid { + bool + operator()(const detail::run_frame &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_request_valid{}(runtime_ev, ctx); + } +}; + +template struct guard_child_succeeded { + bool operator()(const detail::run_frame &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0; + } +}; + +template struct guard_child_failed { + bool + operator()(const detail::run_frame &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_child_succeeded{}(runtime_ev, ctx); + } +}; + +template struct guard_prediction_succeeded { + bool operator()(const detail::run_frame &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0 && + runtime_ev.ctx.graph_err == 0; + } +}; + +template struct guard_prediction_failed { + bool + operator()(const detail::run_frame &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_prediction_succeeded{}(runtime_ev, ctx); + } +}; + +template struct guard_frame_produced { + bool operator()(const detail::run_frame &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0 && + runtime_ev.ctx.produced; + } +}; + +template struct guard_frame_pending { + bool operator()(const detail::run_frame &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.child_accepted && runtime_ev.ctx.child_err == 0 && + !runtime_ev.ctx.produced; + } +}; + +template struct guard_frame_failed { + bool operator()(const detail::run_frame &runtime_ev, + const action::context &) const noexcept { + return !runtime_ev.ctx.child_accepted || runtime_ev.ctx.child_err != 0; + } +}; + +template struct guard_done_callback_present { + bool operator()(const detail::run_frame &runtime_ev, + const action::context &) const noexcept { + return static_cast(runtime_ev.request.on_done); + } +}; + +template struct guard_done_callback_absent { + bool + operator()(const detail::run_frame &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_done_callback_present{}(runtime_ev, ctx); + } +}; + +template struct guard_error_callback_present { + bool operator()(const detail::run_frame &runtime_ev, + const action::context &) const noexcept { + return static_cast(runtime_ev.request.on_error); + } +}; + +template struct guard_error_callback_absent { + bool + operator()(const detail::run_frame &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_error_callback_present{}(runtime_ev, ctx); + } +}; + +} // namespace emel::speech::generator::frame::guard diff --git a/src/emel/speech/generator/frame/sm.hpp b/src/emel/speech/generator/frame/sm.hpp new file mode 100644 index 00000000..0704a40e --- /dev/null +++ b/src/emel/speech/generator/frame/sm.hpp @@ -0,0 +1,216 @@ +#pragma once + +// benchmark: designed + +#include +#include + +#include "emel/sm.hpp" +#include "emel/speech/generator/frame/actions.hpp" +#include "emel/speech/generator/frame/context.hpp" +#include "emel/speech/generator/frame/events.hpp" +#include "emel/speech/generator/frame/guards.hpp" + +namespace emel::speech::generator::frame { + +struct state_idle {}; +struct state_tokenize_result {}; +struct state_plan_result {}; +struct state_predict_result {}; +struct state_graph_result {}; +struct state_sample_result {}; +struct state_detokenize_result {}; +struct state_done_channel_decision {}; +struct state_error_channel_decision {}; + +/* +synchronous speech generator middle-frame actor + +state purpose +- state_idle accepts one already encoded frame. +- result states synchronously interpret tokenize, plan, predict, graph, sample, + and detokenize child outcomes. +- channel-decision states publish exactly one immediate callback when present. + +control invariants +- every child dispatch remains inside the originating RTC boundary. +- request-local status is carried only by detail::run_frame. +- action::context owns only the injected collaborators and reusable buffers. +- every accepted run returns to state_idle before process_event returns. +*/ +template struct model { + auto operator()() const { + namespace sml = stateforward::sml; + + using event_run = detail::run_frame; + using guard_done_present = + guard::guard_done_callback_present; + using guard_done_absent = + guard::guard_done_callback_absent; + using guard_error_present = + guard::guard_error_callback_present; + using guard_error_absent = + guard::guard_error_callback_absent; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Request validation and tokenization. + sml::state <= *sml::state + + sml::event [ guard::guard_request_valid{} ] + / action::effect_tokenize{} + , sml::state <= sml::state + + sml::event [ guard::guard_request_invalid{} ] + / action::effect_fail{} + + //------------------------------------------------------------------------------// + // Bounded middle-frame phase chain. + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_plan{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_failed{} ] + / action::effect_fail{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_predict{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_failed{} ] + / action::effect_fail{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_succeeded{} ] + / action::effect_execute_graph{} + , sml::state <= sml::state + + sml::completion [ guard::guard_child_failed{} ] + / action::effect_fail{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_succeeded{} ] + / action::effect_sample{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_failed{} ] + / action::effect_fail{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_succeeded{} ] + / action::effect_detokenize{} + , sml::state <= sml::state + + sml::completion [ guard::guard_prediction_failed{} ] + / action::effect_fail{} + , sml::state <= sml::state + + sml::completion [ guard::guard_frame_produced{} ] + / action::effect_publish_done{} + , sml::state <= sml::state + + sml::completion [ guard::guard_frame_pending{} ] + / action::effect_fail{} + , sml::state <= sml::state + + sml::completion [ guard::guard_frame_failed{} ] + / action::effect_fail{} + + //------------------------------------------------------------------------------// + // Immediate outcome channels and idle restoration. + , sml::state <= sml::state + + sml::completion [ guard_done_present{} ] + / action::effect_emit_done{} + , sml::state <= sml::state + + sml::completion [ guard_done_absent{} ] + , sml::state <= sml::state + + sml::completion [ guard_error_present{} ] + / action::effect_emit_error{} + , sml::state <= sml::state + + sml::completion [ guard_error_absent{} ] + + //------------------------------------------------------------------------------// + // The frame actor owns no cross-dispatch phase data; reset still has an + // explicit typed actor boundary so its parent can reset all three stages. + , sml::state <= sml::state + + sml::event / action::effect_reset{} + + //------------------------------------------------------------------------------// + // RTC makes idle the only externally observable event boundary. + , sml::state <= sml::state + + sml::unexpected_event / action::effect_unexpected{} + ); + // clang-format on + } +}; + +template +inline bool process_run_event(base_type &base, const event::run &ev) { + detail::run_ctx ctx{}; + const bool accepted = base.process_event(detail::run_frame{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); +} + +template +inline bool process_reset_event(base_type &base, const event::reset &ev) { + const bool accepted = base.process_event(ev); + return accepted && ev.error_out == action::error_code(error::none); +} + +template +struct sm : private emel::sm, + action::context> { + using base_type = + emel::sm, action::context>; + using base_type::is; + using base_type::visit_current_states; + + explicit sm(const dependencies_type &deps) + : base_type(action::context{deps}) {} + + bool process_event(const event::run &ev) { + return process_run_event(static_cast(*this), ev); + } + + bool process_event(const event::reset &ev) { + return process_reset_event(static_cast(*this), ev); + } + + template + requires( + !std::same_as, + detail::run_frame> && + !std::same_as, event::run> && + !std::same_as, event::reset>) + bool process_event(const external_event_type &ev) { + return base_type::process_event(ev); + } +}; + +// Test-only traced actor. Its policy is closed to sml::logger, so it cannot +// activate queue, defer, locking, or scheduler behavior. +template +struct traced_sm : private emel::sm, + action::context, + stateforward::sml::logger> { + using base_type = + emel::sm, action::context, + stateforward::sml::logger>; + using base_type::is; + using base_type::visit_current_states; + + traced_sm(const dependencies_type &deps, logger_type &logger) + : base_type(action::context{deps}, logger) {} + + bool process_event(const event::run &ev) { + return process_run_event(static_cast(*this), ev); + } + + bool process_event(const event::reset &ev) { + return process_reset_event(static_cast(*this), ev); + } + + template + requires( + !std::same_as, + detail::run_frame> && + !std::same_as, event::run> && + !std::same_as, event::reset>) + bool process_event(const external_event_type &ev) { + return base_type::process_event(ev); + } +}; + +template +sm(const dependencies_type &) -> sm; + +} // namespace emel::speech::generator::frame diff --git a/src/emel/speech/generator/guards.hpp b/src/emel/speech/generator/guards.hpp index add259da..eed7f6be 100644 --- a/src/emel/speech/generator/guards.hpp +++ b/src/emel/speech/generator/guards.hpp @@ -4,6 +4,7 @@ #include "emel/speech/generator/context.hpp" #include "emel/speech/generator/events.hpp" +#include "emel/speech/generator/frame/errors.hpp" namespace emel::speech::generator::guard { @@ -263,4 +264,329 @@ struct guard_condition_complete_done_callback_absent { } }; +template struct guard_wavefront_frame_valid { + bool + operator()(const detail::wavefront_frame_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto samples = static_cast(ctx.collaborators.frame_samples); + const auto codebooks = + static_cast(ctx.collaborators.codebook_count); + return ctx.collaborators.frame_samples > 0 && + ctx.collaborators.codebook_count > 0 && + samples <= dependencies_type::wavefront_frame_capacity && + codebooks <= dependencies_type::wavefront_codebook_capacity && + runtime_ev.request.pcm_in.size() == samples && + runtime_ev.request.pcm_out.size() >= samples && + runtime_ev.request.generated_tokens_out.size() >= codebooks && + runtime_ev.request.input_attribution.sequence != + event::k_invalid_wavefront_sequence && + runtime_ev.request.input_attribution.sequence == + ctx.expected_input.sequence; + } +}; + +template struct guard_wavefront_frame_invalid { + bool + operator()(const detail::wavefront_frame_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_wavefront_frame_valid{}(runtime_ev, ctx); + } +}; + +template struct guard_wavefront_flush_valid { + bool + operator()(const detail::wavefront_flush_run &runtime_ev, + const action::context &ctx) const noexcept { + const auto samples = static_cast(ctx.collaborators.frame_samples); + const auto codebooks = + static_cast(ctx.collaborators.codebook_count); + return ctx.collaborators.frame_samples > 0 && + ctx.collaborators.codebook_count > 0 && + samples <= dependencies_type::wavefront_frame_capacity && + codebooks <= dependencies_type::wavefront_codebook_capacity && + runtime_ev.request.pcm_out.size() >= samples && + runtime_ev.request.generated_tokens_out.size() >= codebooks; + } +}; + +template struct guard_wavefront_flush_invalid { + bool + operator()(const detail::wavefront_flush_run &runtime_ev, + const action::context &ctx) const noexcept { + return !guard_wavefront_flush_valid{}(runtime_ev, ctx); + } +}; + +template +struct guard_wavefront_stage_mode { + template + bool + operator()(const runtime_event_type &, + const action::context &ctx) const noexcept { + return ctx.collaborators.stage_mode == stage_mode; + } +}; + +template +struct guard_wavefront_request_stage_mode { + template + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return request_guard_type{}(runtime_ev, ctx) && + guard_wavefront_stage_mode{}( + runtime_ev, ctx); + } +}; + +template +const event::wavefront_attribution &guard_encoded_attribution( + const action::context &ctx) noexcept { + if constexpr (std::same_as) { + return ctx.encoded_lane0_attribution; + } else { + return ctx.encoded_lane1_attribution; + } +} + +template +const event::wavefront_attribution &guard_generated_attribution( + const action::context &ctx) noexcept { + if constexpr (std::same_as) { + return ctx.generated_lane0_attribution; + } else { + return ctx.generated_lane1_attribution; + } +} + +inline bool +guard_attribution_equal(const event::wavefront_attribution lhs, + const event::wavefront_attribution rhs) noexcept { + return lhs.sequence == rhs.sequence && lhs.source == rhs.source; +} + +template +struct guard_wavefront_phase_succeeded { + static constexpr bool has_encode = encode_active; + static constexpr bool has_middle = middle_active; + static constexpr bool has_decode = decode_active; + + template + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + bool succeeded = runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined; + if constexpr (encode_active) { + succeeded = succeeded && runtime_ev.ctx.encode_accepted && + runtime_ev.ctx.encode_err == 0 && + guard_attribution_equal( + guard_encoded_attribution(ctx), + runtime_ev.request.input_attribution); + } + if constexpr (middle_active) { + succeeded = succeeded && runtime_ev.ctx.middle_accepted && + runtime_ev.ctx.middle_err == 0 && + guard_attribution_equal( + guard_generated_attribution(ctx), + guard_encoded_attribution(ctx)); + } + if constexpr (decode_active) { + succeeded = succeeded && runtime_ev.ctx.decode_accepted && + runtime_ev.ctx.decode_err == 0 && + guard_attribution_equal( + runtime_ev.ctx.decoded_attribution, + guard_generated_attribution(ctx)) && + runtime_ev.ctx.decoded_attribution.sequence != + event::k_invalid_wavefront_sequence; + } + if constexpr (two_frame_latency) { + succeeded = succeeded && + runtime_ev.request.input_attribution.sequence >= 2u && + runtime_ev.ctx.decoded_attribution.sequence == + runtime_ev.request.input_attribution.sequence - 2u; + } + return succeeded; + } +}; + +template +struct guard_wavefront_encode_rejected { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + if constexpr (!phase_guard_type::has_encode) { + return false; + } + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + !runtime_ev.ctx.encode_accepted && runtime_ev.ctx.encode_err == 0u; + } +}; + +template +struct guard_wavefront_encode_failed { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + if constexpr (!phase_guard_type::has_encode) { + return false; + } + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + runtime_ev.ctx.encode_err != 0u; + } +}; + +template +struct guard_wavefront_middle_non_production { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + if constexpr (!phase_guard_type::has_middle) { + return false; + } + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + !runtime_ev.ctx.middle_accepted && + runtime_ev.ctx.middle_err == + static_cast(frame::error::frame_pending); + } +}; + +template +struct guard_wavefront_middle_failed { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + if constexpr (!phase_guard_type::has_middle) { + return false; + } + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + runtime_ev.ctx.middle_err != 0u && + runtime_ev.ctx.middle_err != + static_cast(frame::error::frame_pending); + } +}; + +template +struct guard_wavefront_middle_rejected { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + if constexpr (!phase_guard_type::has_middle) { + return false; + } + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + !runtime_ev.ctx.middle_accepted && runtime_ev.ctx.middle_err == 0u; + } +}; + +template +struct guard_wavefront_decode_rejected { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + if constexpr (!phase_guard_type::has_decode) { + return false; + } + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + !runtime_ev.ctx.decode_accepted && runtime_ev.ctx.decode_err == 0u; + } +}; + +template +struct guard_wavefront_decode_failed { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + if constexpr (!phase_guard_type::has_decode) { + return false; + } + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + runtime_ev.ctx.decode_err != 0u; + } +}; + +template +struct guard_wavefront_attribution_failed { + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + const bool encode_ok = + !phase_guard_type::has_encode || + (runtime_ev.ctx.encode_accepted && runtime_ev.ctx.encode_err == 0u); + const bool middle_ok = + !phase_guard_type::has_middle || + (runtime_ev.ctx.middle_accepted && runtime_ev.ctx.middle_err == 0u); + const bool decode_ok = + !phase_guard_type::has_decode || + (runtime_ev.ctx.decode_accepted && runtime_ev.ctx.decode_err == 0u); + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && encode_ok && + middle_ok && decode_ok && !phase_guard_type{}(runtime_ev, ctx); + } +}; + +template +struct guard_wavefront_phase_success_done_present { + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return phase_guard_type{}(runtime_ev, ctx) && + static_cast(runtime_ev.request.on_done); + } +}; + +template +struct guard_wavefront_phase_success_done_absent { + bool + operator()(const runtime_event_type &runtime_ev, + const action::context &ctx) const noexcept { + return phase_guard_type{}(runtime_ev, ctx) && + !static_cast(runtime_ev.request.on_done); + } +}; + +template +struct guard_wavefront_submission_failed { + bool operator()(const runtime_event_type &runtime_ev, + const action::context &) const noexcept { + return !runtime_ev.ctx.all_submitted || !runtime_ev.ctx.joined; + } +}; + +template +struct guard_wavefront_reset_submission_failed { + bool operator()(const detail::event_wavefront_reset_run &runtime_ev, + const action::context &) const noexcept { + return !runtime_ev.ctx.all_submitted || !runtime_ev.ctx.joined; + } +}; + +template +struct guard_wavefront_reset_children_failed { + bool operator()(const detail::event_wavefront_reset_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + (!runtime_ev.ctx.encode_accepted || + runtime_ev.ctx.encode_err != 0u || + !runtime_ev.ctx.middle_accepted || + runtime_ev.ctx.middle_err != 0u || + !runtime_ev.ctx.decode_accepted || runtime_ev.ctx.decode_err != 0u); + } +}; + +template struct guard_wavefront_reset_succeeded { + bool operator()(const detail::event_wavefront_reset_run &runtime_ev, + const action::context &) const noexcept { + return runtime_ev.ctx.all_submitted && runtime_ev.ctx.joined && + runtime_ev.ctx.encode_accepted && runtime_ev.ctx.encode_err == 0u && + runtime_ev.ctx.middle_accepted && runtime_ev.ctx.middle_err == 0u && + runtime_ev.ctx.decode_accepted && runtime_ev.ctx.decode_err == 0u; + } +}; + } // namespace emel::speech::generator::guard diff --git a/src/emel/speech/generator/sm.hpp b/src/emel/speech/generator/sm.hpp index f872ab57..70c5083d 100644 --- a/src/emel/speech/generator/sm.hpp +++ b/src/emel/speech/generator/sm.hpp @@ -74,6 +74,27 @@ struct state_flush_done_channel_decision {}; struct state_flush_error_channel_decision {}; struct state_errored {}; +struct state_wavefront_fill0 {}; +struct state_wavefront_fill0_result {}; +struct state_wavefront_fill1_model0 {}; +struct state_wavefront_fill1_model0_result {}; +struct state_wavefront_steady_even {}; +struct state_wavefront_steady_even_result {}; +struct state_wavefront_steady_odd {}; +struct state_wavefront_steady_odd_result {}; +struct state_wavefront_empty_flush_result {}; +struct state_wavefront_drain_model_result {}; +struct state_wavefront_drain_model_decode_even_result {}; +struct state_wavefront_drain_model_decode_odd_result {}; +struct state_wavefront_final_decode_lane0 {}; +struct state_wavefront_final_decode_lane0_result {}; +struct state_wavefront_final_decode_lane1 {}; +struct state_wavefront_final_decode_lane1_result {}; +struct state_wavefront_complete {}; +struct state_wavefront_error_channel_decision {}; +struct state_wavefront_errored {}; +struct state_wavefront_reset_result {}; + /* generic speech generator (single source of truth) @@ -774,6 +795,989 @@ template struct synthesis_model { } }; +/* +opt-in speech generator stage wavefront + +state purpose +- fill0 accepts only frame zero and runs encode on the dedicated codec worker. +- fill1_model0, steady parity, drain, and final-decode states are extended by + the additive wavefront transitions below without changing serial modes. +- the error state retains no work: every accepted stage submission is joined + by the action that submitted it before the transition can complete. + +control invariants +- phase and parity are represented only by SML states and compile-time lane + action types; no runtime lane table or wrapped inactive index exists. +- the canonical generator sm owns the context and public process wrappers. +- fixed encoded/generated lanes and attribution live in the wavefront context. +*/ +template +struct wavefront_model { + auto operator()() const { + namespace sml = stateforward::sml; + + using event_frame = detail::wavefront_frame_run; + using event_flush = detail::wavefront_flush_run; + using event_reset = detail::event_wavefront_reset_run; + using phase_fill0 = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, true, false, false, false>; + using guard_fill0_done_present = + guard::guard_wavefront_phase_success_done_present< + event_frame, dependencies_type, phase_fill0>; + using guard_fill0_done_absent = + guard::guard_wavefront_phase_success_done_absent< + event_frame, dependencies_type, phase_fill0>; + using phase_fill1_model0 = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_one, action::lane_zero, + action::lane_zero, true, true, false, false>; + using phase_steady_even = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_zero, action::lane_one, + action::lane_zero, true, true, true, true>; + using phase_steady_odd = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_one, action::lane_zero, + action::lane_one, true, true, true, true>; + using phase_empty_flush = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, false, false, false, false>; + using phase_drain_model = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, false, true, false, false>; + using phase_drain_even = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_zero, action::lane_one, + action::lane_zero, false, true, true, false>; + using phase_drain_odd = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_one, false, true, true, false>; + using phase_final_lane0 = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, false, false, true, false>; + using phase_final_lane1 = guard::guard_wavefront_phase_succeeded< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_one, false, false, true, false>; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Fill frame zero on the dedicated encoder worker. + sml::state <= *sml::state + + sml::event [ guard::guard_wavefront_request_stage_mode< + dependencies_type, guard::guard_wavefront_frame_valid, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, true, false, false>{} + , sml::state <= sml::state + + sml::event [ guard::guard_wavefront_request_stage_mode< + dependencies_type, guard::guard_wavefront_frame_valid, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, true, false, false>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_frame_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::completion + [ guard_fill0_done_present{} ] + / action::effect_publish_wavefront_frame_pending{} + , sml::state <= + sml::state + sml::completion + [ guard_fill0_done_absent{} ] + / action::effect_publish_wavefront_frame_pending{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_wavefront_submission_failed< + event_frame, dependencies_type, phase_fill0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_wavefront_encode_rejected< + event_frame, dependencies_type, phase_fill0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_wavefront_encode_failed< + event_frame, dependencies_type, phase_fill0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_frame, dependencies_type, phase_fill0>{} ] + / action::effect_fail_wavefront{} + + //------------------------------------------------------------------------------// + // Fill frame one while the parent runs middle frame zero. + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_one, action::lane_zero, + action::lane_zero, true, true, false>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_one, action::lane_zero, + action::lane_zero, true, true, false>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_frame_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_publish_wavefront_frame_pending{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_publish_wavefront_frame_pending{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_submission_failed< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_encode_rejected< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_encode_failed< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_non_production< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_rejected< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_failed< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_frame, dependencies_type, phase_fill1_model0>{} ] + / action::effect_fail_wavefront{} + + //------------------------------------------------------------------------------// + // Steady parity: encode N, middle N-1, decode N-2. + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_zero, action::lane_one, + action::lane_zero, true, true, true>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_zero, action::lane_one, + action::lane_zero, true, true, true>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_frame_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_publish_wavefront_frame_produced< + dependencies_type, action::lane_zero, true>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_publish_wavefront_frame_produced< + dependencies_type, action::lane_zero, false>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_submission_failed< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_encode_rejected< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_encode_failed< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_non_production< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_rejected< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_failed< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_rejected< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_failed< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_frame, dependencies_type, phase_steady_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_one, action::lane_zero, + action::lane_one, true, true, true>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_one, action::lane_zero, + action::lane_one, true, true, true>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_frame_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_publish_wavefront_frame_produced< + dependencies_type, action::lane_one, true>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_publish_wavefront_frame_produced< + dependencies_type, action::lane_one, false>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_submission_failed< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_encode_rejected< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_encode_failed< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_non_production< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_rejected< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_failed< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_rejected< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_failed< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_frame, dependencies_type, phase_steady_odd>{} ] + / action::effect_fail_wavefront{} + + //------------------------------------------------------------------------------// + // Flush zero or one input frame without fabricating an output. + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_flush_valid{} ] + / action::effect_prepare_wavefront_empty_flush{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_flush_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_flush, dependencies_type, phase_empty_flush>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_zero, false, true, true>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_flush, dependencies_type, phase_empty_flush>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_zero, false, true, false>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, false, true, false>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, false, true, false>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_flush_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_flush, dependencies_type, phase_drain_model>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_zero, false, false, true>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_flush, dependencies_type, phase_drain_model>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_zero, false, false, false>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_submission_failed< + event_flush, dependencies_type, phase_drain_model>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_non_production< + event_flush, dependencies_type, phase_drain_model>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_rejected< + event_flush, dependencies_type, phase_drain_model>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_failed< + event_flush, dependencies_type, phase_drain_model>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_flush, dependencies_type, phase_drain_model>{} ] + / action::effect_fail_wavefront{} + + //------------------------------------------------------------------------------// + // Drain the last middle frame concurrently with the penultimate decode. + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_zero, action::lane_one, + action::lane_zero, false, true, true>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_zero, action::lane_one, + action::lane_zero, false, true, true>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_flush_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_zero, true, false, true>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_zero, true, false, false>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_submission_failed< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_non_production< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_rejected< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_failed< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_rejected< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_failed< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_flush, dependencies_type, phase_drain_even>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_one, false, true, true>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_one, false, true, true>{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_flush_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_one, true, false, true>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_one, true, false, false>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_submission_failed< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_non_production< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_rejected< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_middle_failed< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_rejected< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_failed< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_flush, dependencies_type, phase_drain_odd>{} ] + / action::effect_fail_wavefront{} + + //------------------------------------------------------------------------------// + // Final decode publishes the last frame and completes the session. + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, false, false, true>{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_zero, false, false, true>{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_flush_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_flush, dependencies_type, phase_final_lane0>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_zero, true, true, true>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_flush, dependencies_type, phase_final_lane0>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_zero, true, true, false>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_submission_failed< + event_flush, dependencies_type, phase_final_lane0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_rejected< + event_flush, dependencies_type, phase_final_lane0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_failed< + event_flush, dependencies_type, phase_final_lane0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_flush, dependencies_type, phase_final_lane0>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::parallel>{} ] + / action::effect_execute_wavefront_phase_parallel< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_one, false, false, true>{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_request_stage_mode, + action::wavefront_stage_mode::serial>{} ] + / action::effect_execute_wavefront_phase_serial< + dependencies_type, action::lane_zero, action::lane_zero, + action::lane_one, false, false, true>{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_flush_invalid{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_present< + event_flush, dependencies_type, phase_final_lane1>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_one, true, true, true>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_phase_success_done_absent< + event_flush, dependencies_type, phase_final_lane1>{} ] + / action::effect_publish_wavefront_flush< + dependencies_type, action::lane_one, true, true, false>{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_submission_failed< + event_flush, dependencies_type, phase_final_lane1>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_rejected< + event_flush, dependencies_type, phase_final_lane1>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_decode_failed< + event_flush, dependencies_type, phase_final_lane1>{} ] + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_wavefront_attribution_failed< + event_flush, dependencies_type, phase_final_lane1>{} ] + / action::effect_fail_wavefront{} + + //------------------------------------------------------------------------------// + // Immediate error channel and explicit reset recovery. + , sml::state <= + sml::state + + sml::completion + [ guard::guard_error_callback_present{} ] + / action::effect_emit_wavefront_error{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_error_callback_absent{} ] + , sml::state <= + sml::state + + sml::completion + [ guard::guard_error_callback_present{} ] + / action::effect_emit_wavefront_error{} + , sml::state <= + sml::state + + sml::completion + [ guard::guard_error_callback_absent{} ] + , sml::state <= + sml::state + sml::event + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::event + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::event + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::event + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::event + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::event + / action::effect_fail_wavefront{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_parallel{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_serial{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_parallel{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_serial{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_parallel{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_serial{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_parallel{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_serial{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_parallel{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_serial{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_parallel{} + , sml::state <= + sml::state + + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_serial{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_parallel{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_serial{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_parallel{} + , sml::state <= + sml::state + sml::event + [ guard::guard_wavefront_stage_mode{} ] + / action::effect_reset_wavefront_children_serial{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_wavefront_reset_submission_failed{} ] + / action::effect_fail_wavefront_reset< + dependencies_type, error::stage_submit_failed>{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_wavefront_reset_children_failed{} ] + / action::effect_fail_wavefront_reset< + dependencies_type, error::internal_error>{} + , sml::state <= + sml::state + sml::completion + [ guard::guard_wavefront_reset_succeeded{} ] + / action::effect_reset_wavefront_parent{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_unexpected_wavefront{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_unexpected_wavefront{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_unexpected_wavefront{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_unexpected_wavefront{} + , sml::state <= + sml::state + + sml::unexpected_event + / action::effect_unexpected_wavefront{} + , sml::state <= + sml::state + + sml::unexpected_event + / action::effect_unexpected_wavefront{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_unexpected_wavefront{} + , sml::state <= sml::state + + sml::unexpected_event + / action::effect_unexpected_wavefront{} + ); + // clang-format on + } +}; + template struct model; @@ -787,54 +1791,96 @@ struct model : synthesis_model {}; template -struct sm : public emel::sm, - action::context> { +struct model + : wavefront_model {}; + +template +struct sm : private emel::sm, + action::context> { using base_type = emel::sm, action::context>; using base_type::is; - using base_type::process_event; using base_type::visit_current_states; explicit sm(const dependencies_type &deps) : base_type(std::in_place, deps) {} - bool process_event(const event::initialize &ev) { + bool process_event(const event::initialize &ev) + requires(!action::wavefront_dependencies) + { event::initialize_ctx ctx{}; const bool accepted = base_type::process_event(event::initialize_run{ev, ctx}); return accepted && ctx.err == action::error_code(error::none); } - bool process_event(const event::condition &ev) { + bool process_event(const event::condition &ev) + requires(!action::wavefront_dependencies) + { event::condition_ctx ctx{}; const bool accepted = base_type::process_event(event::condition_run{ev, ctx}); return accepted && ctx.err == action::error_code(error::none); } - bool process_event(const event::generate &ev) { + bool process_event(const event::generate &ev) + requires(!action::wavefront_dependencies) + { event::generate_ctx ctx{}; const bool accepted = base_type::process_event(event::generate_run{ev, ctx}); return accepted && ctx.err == action::error_code(error::none); } - bool process_event(const event::stream_frame &ev) { + bool process_event(const event::stream_frame &ev) + requires(!action::wavefront_dependencies) + { event::frame_ctx ctx{}; const bool accepted = base_type::process_event(event::stream_frame_run{ev, ctx}); return accepted && ctx.err == action::error_code(error::none); } - bool process_event(const event::flush &ev) { + bool process_event(const event::flush &ev) + requires(!action::wavefront_dependencies) + { event::flush_ctx ctx{}; const bool accepted = base_type::process_event(event::flush_run{ev, ctx}); return accepted && ctx.err == action::error_code(error::none); } - bool process_event(const event::reset &ev) { + bool process_event(const event::reset &ev) + requires(!action::wavefront_dependencies) + { const bool accepted = base_type::process_event(ev); return accepted && ev.error_out == action::error_code(error::none); } + + bool process_event(const event::wavefront_frame &ev) + requires action::wavefront_dependencies + { + detail::wavefront_run_ctx ctx{}; + const bool accepted = + base_type::process_event(detail::wavefront_frame_run{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); + } + + bool process_event(const event::wavefront_flush &ev) + requires action::wavefront_dependencies + { + detail::wavefront_run_ctx ctx{}; + const bool accepted = + base_type::process_event(detail::wavefront_flush_run{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); + } + + bool process_event(const event::wavefront_reset &ev) + requires action::wavefront_dependencies + { + detail::wavefront_reset_ctx ctx{}; + const bool accepted = + base_type::process_event(detail::event_wavefront_reset_run{ev, ctx}); + return accepted && ctx.err == action::error_code(error::none); + } }; template diff --git a/src/emel/speech/predictor/moshi/executor/actions.hpp b/src/emel/speech/predictor/moshi/executor/actions.hpp index 6e59db81..f7a73fbd 100644 --- a/src/emel/speech/predictor/moshi/executor/actions.hpp +++ b/src/emel/speech/predictor/moshi/executor/actions.hpp @@ -6,9 +6,12 @@ #include "emel/speech/predictor/moshi/executor/events.hpp" #include +#include #include +#include #include #include +#include namespace emel::speech::predictor::moshi::executor::action { @@ -20,6 +23,25 @@ inline emel::error::type to_error(const error value) noexcept { } // namespace detail_ns +template +inline bool +effect_dispatch_matmul(const emel::kernel::event::op_mul_mat &request, + context &ctx) noexcept { + if constexpr (route == emel::kernel::matmul::lane_mode::serial) { + emel::kernel::matmul::event::dispatch_result result = {}; + bool accepted = false; + const emel::kernel::matmul::event::execute_serial execute{request, result, + accepted}; + return ctx.matmul.process_event(execute) && accepted; + } else { + emel::kernel::matmul::event::dispatch_result result = {}; + bool accepted = false; + const emel::kernel::matmul::event::execute_parallel execute{request, result, + accepted}; + return ctx.matmul.process_event(execute) && accepted; + } +} + struct effect_bind_contract { void operator()(const event::initialize_run &runtime_ev, context &ctx) const noexcept { @@ -455,6 +477,7 @@ struct effect_bind_temporal_layer_projection { } }; +template struct effect_run_temporal_layer_projection { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -468,7 +491,7 @@ struct effect_run_temporal_layer_projection { static_cast(hidden_dim) * 3u), }; runtime_ev.ctx.temporal_layer_projection_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -579,85 +602,173 @@ struct effect_write_temporal_layer_kv_cache { } }; +struct attention_head_slice { + int32_t head_begin = 0; + int32_t head_end = 0; +}; + +template +inline attention_head_slice +compute_fixed_attention_head_slice(const int32_t heads) noexcept { + const int32_t lanes = static_cast(lane_count); + const int32_t lane_index = static_cast(lane); + const int32_t heads_per_lane = heads / lanes; + const int32_t extra_heads = heads % lanes; + const int32_t head_begin = + lane_index * heads_per_lane + std::min(lane_index, extra_heads); + const int32_t head_count = + heads_per_lane + static_cast(lane_index < extra_heads); + return attention_head_slice{ + .head_begin = head_begin, + .head_end = head_begin + head_count, + }; +} + +struct attention_lane_dispatch { + emel::kernel::attention::sm *actor = nullptr; + const emel::kernel::attention::event::head_range_request *request = nullptr; + emel::kernel::attention::event::dispatch_result *result = nullptr; + bool handled = false; +}; + +template +inline void prepare_attention_lanes( + const event::step_run &runtime_ev, const context &ctx, + std::array &head_slices, + std::array + &requests, + std::array + &results, + std::array &dispatches, + std::index_sequence) noexcept { + const auto &view = runtime_ev.ctx.temporal_kv; + const auto &window = runtime_ev.ctx.temporal_position.window; + const auto &lm = runtime_ev.request.model.moshi_lm; + const int32_t hidden_dim = ctx.session.hidden_dim; + const int32_t head_dim = hidden_dim / lm.num_heads; + const std::size_t layer_offset = view.layer_cache_offsets[ + static_cast(runtime_ev.ctx.temporal_layer_index)]; + ((head_slices[lanes] = + compute_fixed_attention_head_slice(lm.num_heads), + requests[lanes] = + emel::kernel::attention::event::head_range_request{ + .query = std::span{ + runtime_ev.ctx.qkv.data(), static_cast(hidden_dim)}, + .key_cache = std::span{view.key_cache}, + .value_cache = std::span{view.value_cache}, + .output = std::span{ + runtime_ev.ctx.attention.data() + + static_cast(head_slices[lanes].head_begin) * + static_cast(head_dim), + static_cast(head_slices[lanes].head_end - + head_slices[lanes].head_begin) * + static_cast(head_dim)}, + .layer_offset = layer_offset, + .hidden_dim = hidden_dim, + .head_dim = head_dim, + .head_begin = head_slices[lanes].head_begin, + .head_end = head_slices[lanes].head_end, + .position_capacity = view.position_capacity, + .physical_begin = window.physical_begin, + .valid_positions = window.valid_positions, + }, + dispatches[lanes] = attention_lane_dispatch{ + .actor = &ctx.attention_actors->actors[lanes], + .request = &requests[lanes], + .result = &results[lanes], + }), + ...); +} + +template +inline bool submit_attention_lane( + context &ctx, emel::kernel::matmul::lane_pool::join_group &group, + emel::policy::fork_join_start_gate &gate, + attention_lane_dispatch &dispatch) noexcept { + dispatch.handled = false; + return ctx.attention_lanes->try_submit( + group, [&dispatch, &gate]() noexcept { + gate.arrive_and_wait(); + const emel::kernel::attention::event::execute execute{ + *dispatch.request, *dispatch.result}; + dispatch.handled = dispatch.actor->process_event(execute); + }); +} + +template +inline std::size_t submit_attention_worker_lanes( + context &ctx, emel::kernel::matmul::lane_pool::join_group &group, + emel::policy::fork_join_start_gate &gate, + std::array &dispatches, + std::index_sequence) noexcept { + std::size_t submitted = 0u; + ((submitted += static_cast(submit_attention_lane< + lane_offsets + 1u>(ctx, group, gate, dispatches[lane_offsets + 1u]))), + ...); + return submitted; +} + +template +inline bool attention_worker_lanes_accepted( + const std::array &dispatches, + std::index_sequence) noexcept { + return (dispatches[lane_offsets + 1u].handled && ...); +} + +template struct effect_run_temporal_layer_attention { void operator()(const event::step_run &runtime_ev, - const context &ctx) const noexcept { - const auto &view = runtime_ev.ctx.temporal_kv; - const auto &window = runtime_ev.ctx.temporal_position.window; - const auto &lm = runtime_ev.request.model.moshi_lm; + context &ctx) const noexcept { + static_assert(lane_count == 1u || lane_count == 2u || lane_count == 4u || + lane_count == 8u); const int32_t hidden_dim = ctx.session.hidden_dim; - const int32_t head_dim = hidden_dim / lm.num_heads; - const int32_t capacity = view.position_capacity; - const int32_t valid_positions = window.valid_positions; - const float scale = 1.0f / std::sqrt(static_cast(head_dim)); - const size_t layer_offset = view.layer_cache_offsets[static_cast( - runtime_ev.ctx.temporal_layer_index)]; runtime_ev.ctx.temporal_layer_attention_ok = false; std::fill_n(runtime_ev.ctx.attention.data(), static_cast(hidden_dim), 0.0f); - - for (int32_t head = 0; head < lm.num_heads; ++head) { - const int32_t head_offset = head * head_dim; - const float *q_head = - runtime_ev.ctx.qkv.data() + static_cast(head_offset); - for (int32_t dim = 0; dim < head_dim; ++dim) { - runtime_ev.ctx.q_bf16[static_cast(dim)] = - emel::kernel::detail::fp32_to_bf16(q_head[dim]); - } - for (int32_t physical = 0; physical < capacity; ++physical) { - runtime_ev.ctx.attention_scores[static_cast(physical)] = - -std::numeric_limits::infinity(); - } - for (int32_t position = 0; position < valid_positions; ++position) { - const int32_t unwrapped = window.physical_begin + position; - const int32_t physical = - unwrapped - static_cast(unwrapped >= capacity) * capacity; - const size_t cache_begin = - layer_offset + - static_cast(physical) * static_cast(hidden_dim) + - static_cast(head_offset); - runtime_ev.ctx.attention_scores[static_cast(physical)] = - emel::kernel::detail::vec_dot_bf16_ggml( - head_dim, view.key_cache.data() + cache_begin, - runtime_ev.ctx.q_bf16.data()) * - scale; - } - - emel::kernel::detail::soft_max_row_ggml( - capacity, runtime_ev.ctx.attention_scores.data()); - for (int32_t physical = 0; physical < capacity; ++physical) { - runtime_ev.ctx.attention_weights_bf16[static_cast(physical)] = - emel::kernel::detail::fp32_to_bf16( - runtime_ev.ctx.attention_scores[static_cast(physical)]); - } - - float *attention_head = - runtime_ev.ctx.attention.data() + static_cast(head_offset); - for (int32_t dim = 0; dim < head_dim; ++dim) { - double sum = 0.0; - for (int32_t position = 0; position < valid_positions; ++position) { - const int32_t unwrapped = window.physical_begin + position; - const int32_t physical = - unwrapped - - static_cast(unwrapped >= capacity) * capacity; - const size_t value_index = - layer_offset + - static_cast(physical) * static_cast(hidden_dim) + - static_cast(head_offset + dim); - sum += static_cast( - emel::kernel::detail::bf16_to_fp32( - view.value_cache[value_index]) * - emel::kernel::detail::bf16_to_fp32( - runtime_ev.ctx - .attention_weights_bf16[static_cast(physical)])); - } - attention_head[dim] = static_cast(sum); - } + std::array head_slices = {}; + std::array + requests = {}; + std::array + results = {}; + std::array dispatches = {}; + prepare_attention_lanes( + runtime_ev, ctx, head_slices, requests, results, dispatches, + std::make_index_sequence{}); + + if constexpr (lane_count == 1u) { + const emel::kernel::attention::event::execute execute{requests[0], + results[0]}; + runtime_ev.ctx.temporal_layer_attention_ok = + ctx.attention_actors->actors[0].process_event(execute); + } else { + emel::kernel::matmul::lane_pool::join_group group{}; + emel::policy::fork_join_start_gate gate{}; + const std::size_t submitted = submit_attention_worker_lanes( + ctx, group, gate, dispatches, + std::make_index_sequence{}); + gate.open_after_arrivals(submitted); + const emel::kernel::attention::event::execute owner_execute{requests[0], + results[0]}; + const bool owner_accepted = + ctx.attention_actors->actors[0].process_event(owner_execute); + const bool workers_drained = group.wait(); + runtime_ev.ctx.temporal_layer_attention_ok = + submitted == lane_count - 1u && owner_accepted && workers_drained && + attention_worker_lanes_accepted( + dispatches, std::make_index_sequence{}); } - runtime_ev.ctx.temporal_layer_attention_ok = true; } }; +using effect_run_temporal_layer_attention_serial = + effect_run_temporal_layer_attention<1u>; +using effect_run_temporal_layer_attention_two = + effect_run_temporal_layer_attention<2u>; +using effect_run_temporal_layer_attention_four = + effect_run_temporal_layer_attention<4u>; +using effect_run_temporal_layer_attention_eight = + effect_run_temporal_layer_attention<8u>; + struct effect_bind_temporal_layer_out_projection { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -677,6 +788,7 @@ struct effect_bind_temporal_layer_out_projection { } }; +template struct effect_run_temporal_layer_out_projection { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -690,7 +802,7 @@ struct effect_run_temporal_layer_out_projection { static_cast(hidden_dim)), }; runtime_ev.ctx.temporal_layer_out_projection_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -773,6 +885,7 @@ struct effect_bind_temporal_layer_gating_in { } }; +template struct effect_run_temporal_layer_gating_in { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -791,7 +904,7 @@ struct effect_run_temporal_layer_gating_in { projection_dim), }; runtime_ev.ctx.temporal_layer_gating_in_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -868,6 +981,7 @@ struct effect_bind_temporal_layer_gating_out { } }; +template struct effect_run_temporal_layer_gating_out { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -885,7 +999,7 @@ struct effect_run_temporal_layer_gating_out { static_cast(hidden_dim)), }; runtime_ev.ctx.temporal_layer_gating_out_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -1041,6 +1155,7 @@ struct effect_select_text_token { } }; +template struct effect_compute_text_token_logits { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -1056,7 +1171,8 @@ struct effect_compute_text_token_logits { .dst = detail::make_f32_dst(runtime_ev.ctx.logits.data(), 1u, static_cast(text_card)), }; - runtime_ev.ctx.text_logits_ok = ctx.kernel.process_event(logits_ev); + runtime_ev.ctx.text_logits_ok = + effect_dispatch_matmul(logits_ev, ctx); } }; @@ -1224,6 +1340,7 @@ struct effect_bind_depformer_audio_input_projection { } }; +template struct effect_run_depformer_input_projection { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -1239,7 +1356,7 @@ struct effect_run_depformer_input_projection { static_cast(dep_dim)), }; runtime_ev.ctx.depformer_input_projection_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -1364,6 +1481,7 @@ struct effect_bind_depformer_layer_projection { } }; +template struct effect_run_depformer_layer_projection { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -1377,7 +1495,7 @@ struct effect_run_depformer_layer_projection { static_cast(dep_dim) * 3u), }; runtime_ev.ctx.depformer_layer_projection_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -1512,6 +1630,7 @@ struct effect_bind_depformer_layer_out_projection { } }; +template struct effect_run_depformer_layer_out_projection { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -1525,7 +1644,7 @@ struct effect_run_depformer_layer_out_projection { static_cast(dep_dim)), }; runtime_ev.ctx.depformer_layer_out_projection_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -1611,6 +1730,7 @@ struct effect_bind_depformer_layer_gating_in { } }; +template struct effect_run_depformer_layer_gating_in { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -1631,7 +1751,7 @@ struct effect_run_depformer_layer_gating_in { projection_dim), }; runtime_ev.ctx.depformer_layer_gating_in_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -1715,6 +1835,7 @@ struct effect_bind_depformer_layer_gating_out { } }; +template struct effect_run_depformer_layer_gating_out { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -1735,7 +1856,7 @@ struct effect_run_depformer_layer_gating_out { static_cast(dep_dim)), }; runtime_ev.ctx.depformer_layer_gating_out_ok = - ctx.kernel.process_event(projection_ev); + effect_dispatch_matmul(projection_ev, ctx); } }; @@ -1809,6 +1930,7 @@ struct effect_select_depformer_token { } }; +template struct effect_compute_depformer_token_logits { void operator()(const event::step_run &runtime_ev, context &ctx) const noexcept { @@ -1825,7 +1947,8 @@ struct effect_compute_depformer_token_logits { .dst = detail::make_f32_dst(runtime_ev.ctx.logits.data(), 1u, static_cast(audio_card)), }; - runtime_ev.ctx.depformer_logits_ok = ctx.kernel.process_event(logits_ev); + runtime_ev.ctx.depformer_logits_ok = + effect_dispatch_matmul(logits_ev, ctx); } }; @@ -1989,4 +2112,71 @@ struct effect_mark_unexpected_and_store { } }; +struct effect_run_temporal_layer_projection_serial + : effect_run_temporal_layer_projection< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_temporal_layer_projection_parallel + : effect_run_temporal_layer_projection< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_run_temporal_layer_out_projection_serial + : effect_run_temporal_layer_out_projection< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_temporal_layer_out_projection_parallel + : effect_run_temporal_layer_out_projection< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_run_temporal_layer_gating_in_serial + : effect_run_temporal_layer_gating_in< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_temporal_layer_gating_in_parallel + : effect_run_temporal_layer_gating_in< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_run_temporal_layer_gating_out_serial + : effect_run_temporal_layer_gating_out< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_temporal_layer_gating_out_parallel + : effect_run_temporal_layer_gating_out< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_compute_text_token_logits_serial + : effect_compute_text_token_logits< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_compute_text_token_logits_parallel + : effect_compute_text_token_logits< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_run_depformer_input_projection_serial + : effect_run_depformer_input_projection< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_depformer_input_projection_parallel + : effect_run_depformer_input_projection< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_run_depformer_layer_projection_serial + : effect_run_depformer_layer_projection< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_depformer_layer_projection_parallel + : effect_run_depformer_layer_projection< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_run_depformer_layer_out_projection_serial + : effect_run_depformer_layer_out_projection< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_depformer_layer_out_projection_parallel + : effect_run_depformer_layer_out_projection< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_run_depformer_layer_gating_in_serial + : effect_run_depformer_layer_gating_in< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_depformer_layer_gating_in_parallel + : effect_run_depformer_layer_gating_in< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_run_depformer_layer_gating_out_serial + : effect_run_depformer_layer_gating_out< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_run_depformer_layer_gating_out_parallel + : effect_run_depformer_layer_gating_out< + emel::kernel::matmul::lane_mode::parallel> {}; +struct effect_compute_depformer_token_logits_serial + : effect_compute_depformer_token_logits< + emel::kernel::matmul::lane_mode::serial> {}; +struct effect_compute_depformer_token_logits_parallel + : effect_compute_depformer_token_logits< + emel::kernel::matmul::lane_mode::parallel> {}; + } // namespace emel::speech::predictor::moshi::executor::action diff --git a/src/emel/speech/predictor/moshi/executor/context.hpp b/src/emel/speech/predictor/moshi/executor/context.hpp index 197f9fca..8bb877cc 100644 --- a/src/emel/speech/predictor/moshi/executor/context.hpp +++ b/src/emel/speech/predictor/moshi/executor/context.hpp @@ -1,8 +1,14 @@ #pragma once +#include #include #include +#include +#include +#include +#include "emel/kernel/attention/sm.hpp" +#include "emel/kernel/matmul/sm.hpp" #include "emel/kernel/sm.hpp" #include "emel/logits/sampler/sm.hpp" #include "emel/memory/streaming/sm.hpp" @@ -35,9 +41,20 @@ struct capacities { uint64_t sampling_top_k = 0u; }; +inline constexpr std::size_t k_max_attention_lanes = 8u; + +struct attention_lane_storage { + std::array actors = {}; +}; + struct dependencies { kv_views kv = {}; emel::kernel::sm &kernel; + emel::kernel::matmul::sm &matmul; + emel::kernel::matmul::lane_mode matmul_lane_mode = + emel::kernel::matmul::lane_mode::serial; + emel::kernel::matmul::lane_pool *attention_lanes = nullptr; + std::size_t active_attention_lanes = 1u; emel::logits::sampler::sm *sampler = nullptr; policies policy = {}; capacities capacity = {}; @@ -78,7 +95,18 @@ struct context { : temporal_kv(deps.kv.temporal), depformer_kv(deps.kv.depformer), temporal_positions(deps.kv.temporal_positions), depformer_positions(deps.kv.depformer_positions), kernel(deps.kernel), - sampler(deps.sampler), policy(deps.policy), capacity(deps.capacity) {} + matmul(deps.matmul), matmul_lane_mode(deps.matmul_lane_mode), + attention_lanes(deps.attention_lanes), + active_attention_lanes(deps.active_attention_lanes), + attention_actors(new (std::nothrow) attention_lane_storage{}), + sampler(deps.sampler), policy(deps.policy), capacity(deps.capacity) { + // Attention actors own substantial reusable scratch. One construction-time + // allocation avoids oversized parent and thread stacks; same-RTC inference + // dispatches reuse it without allocation. + if (attention_actors == nullptr) { + std::terminate(); + } + } runtime session = {}; sampling_config sampling = {}; @@ -87,6 +115,11 @@ struct context { emel::memory::streaming::sm *temporal_positions = nullptr; emel::memory::streaming::sm *depformer_positions = nullptr; emel::kernel::sm &kernel; + emel::kernel::matmul::sm &matmul; + const emel::kernel::matmul::lane_mode matmul_lane_mode; + emel::kernel::matmul::lane_pool *attention_lanes = nullptr; + const std::size_t active_attention_lanes; + std::unique_ptr attention_actors = {}; emel::logits::sampler::sm *sampler = nullptr; const policies policy; const capacities capacity; diff --git a/src/emel/speech/predictor/moshi/executor/guards.hpp b/src/emel/speech/predictor/moshi/executor/guards.hpp index 29a99c70..66cc2d69 100644 --- a/src/emel/speech/predictor/moshi/executor/guards.hpp +++ b/src/emel/speech/predictor/moshi/executor/guards.hpp @@ -12,6 +12,35 @@ namespace emel::speech::predictor::moshi::executor::guard { +inline bool guard_serial_matmul_mode(const action::context &ctx) noexcept { + return ctx.matmul_lane_mode == emel::kernel::matmul::lane_mode::serial; +} + +inline bool guard_parallel_matmul_mode(const action::context &ctx) noexcept { + return ctx.matmul_lane_mode == emel::kernel::matmul::lane_mode::parallel; +} + +struct guard_serial_matmul_route { + bool operator()(const event::step_run &, + const action::context &ctx) const noexcept { + return guard_serial_matmul_mode(ctx); + } +}; + +struct guard_parallel_matmul_route { + bool operator()(const event::step_run &, + const action::context &ctx) const noexcept { + return guard_parallel_matmul_mode(ctx); + } +}; + +template struct guard_for_matmul_route { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + return base_guard{}(runtime_ev, ctx) && route_guard{}(runtime_ev, ctx); + } +}; + inline bool guard_cache_span_valid(const size_t layer_offset, const size_t first_position, const size_t position_count, @@ -1006,6 +1035,48 @@ struct guard_temporal_layer_attention_unsupported { } }; +template +struct guard_temporal_layer_attention_route { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + static_assert(lane_count == 1u || lane_count == 2u || lane_count == 4u || + lane_count == 8u); + const bool route_selected = ctx.active_attention_lanes == lane_count; + const bool enough_heads = + runtime_ev.request.model.moshi_lm.num_heads >= + static_cast(lane_count); + if constexpr (lane_count == 1u) { + return guard_temporal_layer_attention_supported{}(runtime_ev, ctx) && + route_selected && enough_heads; + } else { + return guard_temporal_layer_attention_supported{}(runtime_ev, ctx) && + route_selected && enough_heads && ctx.attention_lanes != nullptr; + } + } +}; + +using guard_temporal_layer_attention_serial = + guard_temporal_layer_attention_route<1u>; +using guard_temporal_layer_attention_two = + guard_temporal_layer_attention_route<2u>; +using guard_temporal_layer_attention_four = + guard_temporal_layer_attention_route<4u>; +using guard_temporal_layer_attention_eight = + guard_temporal_layer_attention_route<8u>; + +struct guard_temporal_layer_attention_route_unavailable { + bool operator()(const event::step_run &runtime_ev, + const action::context &ctx) const noexcept { + const bool selected = + guard_temporal_layer_attention_serial{}(runtime_ev, ctx) || + guard_temporal_layer_attention_two{}(runtime_ev, ctx) || + guard_temporal_layer_attention_four{}(runtime_ev, ctx) || + guard_temporal_layer_attention_eight{}(runtime_ev, ctx); + return guard_temporal_layer_attention_supported{}(runtime_ev, ctx) && + !selected; + } +}; + struct guard_temporal_layer_attention_succeeded { bool operator()(const event::step_run &runtime_ev, const action::context &) const noexcept { @@ -2439,4 +2510,46 @@ struct guard_unexpected_error_out_absent { } }; +struct guard_projection_view_bound_serial + : guard_for_matmul_route {}; + +struct guard_projection_view_bound_parallel + : guard_for_matmul_route {}; + +struct guard_forced_text_token_sampling_serial + : guard_for_matmul_route {}; + +struct guard_forced_text_token_sampling_parallel + : guard_for_matmul_route {}; + +struct guard_text_logits_sampling_serial + : guard_for_matmul_route< + guard_text_logits_projection_bound_and_no_forced_token_sampling, + guard_serial_matmul_route> {}; + +struct guard_text_logits_sampling_parallel + : guard_for_matmul_route< + guard_text_logits_projection_bound_and_no_forced_token_sampling, + guard_parallel_matmul_route> {}; + +struct guard_depformer_input_projection_serial + : guard_for_matmul_route {}; + +struct guard_depformer_input_projection_parallel + : guard_for_matmul_route {}; + +struct guard_depformer_logits_sampling_serial + : guard_for_matmul_route {}; + +struct guard_depformer_logits_sampling_parallel + : guard_for_matmul_route {}; + } // namespace emel::speech::predictor::moshi::executor::guard diff --git a/src/emel/speech/predictor/moshi/executor/sm.hpp b/src/emel/speech/predictor/moshi/executor/sm.hpp index e136f446..851fbda5 100644 --- a/src/emel/speech/predictor/moshi/executor/sm.hpp +++ b/src/emel/speech/predictor/moshi/executor/sm.hpp @@ -415,8 +415,11 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_projection_view_bound{} ] - / action::effect_run_temporal_layer_projection{} + + sml::completion [ guard::guard_projection_view_bound_serial{} ] + / action::effect_run_temporal_layer_projection_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound_parallel{} ] + / action::effect_run_temporal_layer_projection_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_projection_view_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -458,11 +461,23 @@ struct model { + sml::completion [ guard::guard_temporal_layer_cache_write_failed{} ] / action::effect_mark_graph_execution_unsupported{} , sml::state <= sml::state - + sml::completion [ guard::guard_temporal_layer_attention_supported{} ] - / action::effect_run_temporal_layer_attention{} + + sml::completion [ guard::guard_temporal_layer_attention_serial{} ] + / action::effect_run_temporal_layer_attention_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_attention_two{} ] + / action::effect_run_temporal_layer_attention_two{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_attention_four{} ] + / action::effect_run_temporal_layer_attention_four{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_attention_eight{} ] + / action::effect_run_temporal_layer_attention_eight{} , sml::state <= sml::state + sml::completion [ guard::guard_temporal_layer_attention_unsupported{} ] / action::effect_mark_graph_execution_unsupported{} + , sml::state <= sml::state + + sml::completion [ guard::guard_temporal_layer_attention_route_unavailable{} ] + / action::effect_mark_graph_execution_unsupported{} , sml::state <= sml::state + sml::completion [ guard::guard_temporal_layer_attention_succeeded{} ] , sml::state <= sml::state @@ -477,8 +492,11 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_projection_view_bound{} ] - / action::effect_run_temporal_layer_out_projection{} + + sml::completion [ guard::guard_projection_view_bound_serial{} ] + / action::effect_run_temporal_layer_out_projection_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound_parallel{} ] + / action::effect_run_temporal_layer_out_projection_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_projection_view_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -523,8 +541,11 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_projection_view_bound{} ] - / action::effect_run_temporal_layer_gating_in{} + + sml::completion [ guard::guard_projection_view_bound_serial{} ] + / action::effect_run_temporal_layer_gating_in_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound_parallel{} ] + / action::effect_run_temporal_layer_gating_in_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_projection_view_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -561,8 +582,11 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_projection_view_bound{} ] - / action::effect_run_temporal_layer_gating_out{} + + sml::completion [ guard::guard_projection_view_bound_serial{} ] + / action::effect_run_temporal_layer_gating_out_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound_parallel{} ] + / action::effect_run_temporal_layer_gating_out_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_projection_view_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -630,8 +654,13 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_forced_text_token_valid_and_sampling_consume{} ] - / action::effect_compute_text_token_logits{} + + sml::completion + [ guard::guard_forced_text_token_sampling_serial{} ] + / action::effect_compute_text_token_logits_serial{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_forced_text_token_sampling_parallel{} ] + / action::effect_compute_text_token_logits_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_forced_text_token_valid_without_sampling_consume{} ] / action::effect_publish_forced_text_token{} @@ -645,8 +674,13 @@ struct model { + sml::completion [ guard::guard_text_logits_projection_bound_and_no_forced_token_argmax{} ] / action::effect_select_text_token{} , sml::state <= sml::state - + sml::completion [ guard::guard_text_logits_projection_bound_and_no_forced_token_sampling{} ] - / action::effect_compute_text_token_logits{} + + sml::completion + [ guard::guard_text_logits_sampling_serial{} ] + / action::effect_compute_text_token_logits_serial{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_text_logits_sampling_parallel{} ] + / action::effect_compute_text_token_logits_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid{} ] / action::effect_mark_graph_execution_unsupported{} @@ -733,8 +767,13 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_depformer_input_projection_bound{} ] - / action::effect_run_depformer_input_projection{} + + sml::completion + [ guard::guard_depformer_input_projection_serial{} ] + / action::effect_run_depformer_input_projection_serial{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_depformer_input_projection_parallel{} ] + / action::effect_run_depformer_input_projection_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_depformer_input_projection_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -801,8 +840,11 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_projection_view_bound{} ] - / action::effect_run_depformer_layer_projection{} + + sml::completion [ guard::guard_projection_view_bound_serial{} ] + / action::effect_run_depformer_layer_projection_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound_parallel{} ] + / action::effect_run_depformer_layer_projection_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_projection_view_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -842,8 +884,11 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_projection_view_bound{} ] - / action::effect_run_depformer_layer_out_projection{} + + sml::completion [ guard::guard_projection_view_bound_serial{} ] + / action::effect_run_depformer_layer_out_projection_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound_parallel{} ] + / action::effect_run_depformer_layer_out_projection_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_projection_view_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -888,8 +933,11 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_projection_view_bound{} ] - / action::effect_run_depformer_layer_gating_in{} + + sml::completion [ guard::guard_projection_view_bound_serial{} ] + / action::effect_run_depformer_layer_gating_in_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound_parallel{} ] + / action::effect_run_depformer_layer_gating_in_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_projection_view_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -926,8 +974,11 @@ struct model { , sml::state <= sml::state + sml::completion , sml::state <= sml::state - + sml::completion [ guard::guard_projection_view_bound{} ] - / action::effect_run_depformer_layer_gating_out{} + + sml::completion [ guard::guard_projection_view_bound_serial{} ] + / action::effect_run_depformer_layer_gating_out_serial{} + , sml::state <= sml::state + + sml::completion [ guard::guard_projection_view_bound_parallel{} ] + / action::effect_run_depformer_layer_gating_out_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_projection_view_bind_failed{} ] / action::effect_mark_graph_execution_unsupported{} @@ -959,8 +1010,13 @@ struct model { + sml::completion [ guard::guard_depformer_logits_projection_bound_argmax{} ] / action::effect_select_depformer_token{} , sml::state <= sml::state - + sml::completion [ guard::guard_depformer_logits_projection_bound_sampling{} ] - / action::effect_compute_depformer_token_logits{} + + sml::completion + [ guard::guard_depformer_logits_sampling_serial{} ] + / action::effect_compute_depformer_token_logits_serial{} + , sml::state <= sml::state + + sml::completion + [ guard::guard_depformer_logits_sampling_parallel{} ] + / action::effect_compute_depformer_token_logits_parallel{} , sml::state <= sml::state + sml::completion [ guard::guard_depformer_logits_projection_bound_sampling_invalid{} ] / action::effect_mark_graph_execution_unsupported{} diff --git a/src/emel/text/generator/actions.hpp b/src/emel/text/generator/actions.hpp index 50e4bc4c..69d59051 100644 --- a/src/emel/text/generator/actions.hpp +++ b/src/emel/text/generator/actions.hpp @@ -1161,8 +1161,15 @@ struct on_unexpected { struct capture_diagnostics { void operator()(const event::capture_diagnostics & ev, const context & ctx) const noexcept { const auto & backend = ctx.compute.backend; - const auto total = [&backend](auto counter) noexcept { - return detail::compute_kernel_counter_total(backend, counter); + emel::kernel::matmul::event::diagnostics matmul = {}; + bool matmul_accepted = false; + const emel::kernel::matmul::event::capture_diagnostics capture{ + matmul, matmul_accepted}; + ctx.matmul_actor->process_event(capture); + (void)matmul_accepted; + const auto total = [&backend](auto counter, + const uint64_t matmul_total) noexcept { + return std::invoke(counter, backend.kernel) + matmul_total; }; ev.out.kernel_kind = backend.kernel_kind; ev.out.kernel_dispatch_calls = backend.kernel_dispatch_calls; @@ -1170,40 +1177,71 @@ struct capture_diagnostics { ev.out.packed_q8_0_dispatch_calls = backend.packed_q8_0_dispatch_calls; ev.out.flash_attention_dispatch_calls = backend.flash_attention_dispatch_calls; ev.out.optimized_flash_dispatch_calls = - total(&emel::kernel::sm::optimized_flash_dispatch_count); - ev.out.shared_flash_dispatch_calls = total(&emel::kernel::sm::shared_flash_dispatch_count); - ev.out.optimized_q2_dispatch_calls = total(&emel::kernel::sm::optimized_q2_dispatch_count); - ev.out.shared_q2_dispatch_calls = total(&emel::kernel::sm::shared_q2_dispatch_count); - ev.out.optimized_q3_dispatch_calls = total(&emel::kernel::sm::optimized_q3_dispatch_count); - ev.out.shared_q3_dispatch_calls = total(&emel::kernel::sm::shared_q3_dispatch_count); - ev.out.optimized_q4_dispatch_calls = total(&emel::kernel::sm::optimized_q4_dispatch_count); + total(&emel::kernel::sm::optimized_flash_dispatch_count, + matmul.optimized_flash_dispatch_calls); + ev.out.shared_flash_dispatch_calls = total( + &emel::kernel::sm::shared_flash_dispatch_count, + matmul.shared_flash_dispatch_calls); + ev.out.optimized_q2_dispatch_calls = total( + &emel::kernel::sm::optimized_q2_dispatch_count, + matmul.optimized_q2_dispatch_calls); + ev.out.shared_q2_dispatch_calls = total( + &emel::kernel::sm::shared_q2_dispatch_count, + matmul.shared_q2_dispatch_calls); + ev.out.optimized_q3_dispatch_calls = total( + &emel::kernel::sm::optimized_q3_dispatch_count, + matmul.optimized_q3_dispatch_calls); + ev.out.shared_q3_dispatch_calls = total( + &emel::kernel::sm::shared_q3_dispatch_count, + matmul.shared_q3_dispatch_calls); + ev.out.optimized_q4_dispatch_calls = total( + &emel::kernel::sm::optimized_q4_dispatch_count, + matmul.optimized_q4_dispatch_calls); ev.out.optimized_q4_vector_dispatch_calls = - total(&emel::kernel::sm::optimized_q4_vector_dispatch_count); + total(&emel::kernel::sm::optimized_q4_vector_dispatch_count, + matmul.optimized_q4_vector_dispatch_calls); ev.out.optimized_q4_vector_packed_dispatch_calls = - total(&emel::kernel::sm::optimized_q4_vector_packed_dispatch_count); + total(&emel::kernel::sm::optimized_q4_vector_packed_dispatch_count, + matmul.optimized_q4_vector_packed_dispatch_calls); ev.out.optimized_q4_vector_packed_q8_rhs_dispatch_calls = - total(&emel::kernel::sm::optimized_q4_vector_packed_q8_rhs_dispatch_count); - ev.out.shared_q4_dispatch_calls = total(&emel::kernel::sm::shared_q4_dispatch_count); - ev.out.optimized_q6_dispatch_calls = total(&emel::kernel::sm::optimized_q6_dispatch_count); + total(&emel::kernel::sm::optimized_q4_vector_packed_q8_rhs_dispatch_count, + matmul.optimized_q4_vector_packed_q8_rhs_dispatch_calls); + ev.out.shared_q4_dispatch_calls = total( + &emel::kernel::sm::shared_q4_dispatch_count, + matmul.shared_q4_dispatch_calls); + ev.out.optimized_q6_dispatch_calls = total( + &emel::kernel::sm::optimized_q6_dispatch_count, + matmul.optimized_q6_dispatch_calls); ev.out.optimized_q6_vector_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_dispatch_count, + matmul.optimized_q6_vector_dispatch_calls); ev.out.optimized_q6_vector_argmax_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_argmax_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_argmax_dispatch_count, + matmul.optimized_q6_vector_argmax_dispatch_calls); ev.out.optimized_q6_vector_packed_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_packed_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_packed_dispatch_count, + matmul.optimized_q6_vector_packed_dispatch_calls); ev.out.optimized_q6_vector_packed_q8_rhs_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_packed_q8_rhs_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_packed_q8_rhs_dispatch_count, + matmul.optimized_q6_vector_packed_q8_rhs_dispatch_calls); ev.out.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_packed_q8_rhs_argmax_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_packed_q8_rhs_argmax_dispatch_count, + matmul.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls); ev.out.optimized_q6_vector_prepared_q8_rhs_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_prepared_q8_rhs_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_prepared_q8_rhs_dispatch_count, + matmul.optimized_q6_vector_prepared_q8_rhs_dispatch_calls); ev.out.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_count, + matmul.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls); ev.out.optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_count, + matmul.optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls); ev.out.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls = - total(&emel::kernel::sm::optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_count); - ev.out.shared_q6_dispatch_calls = total(&emel::kernel::sm::shared_q6_dispatch_count); + total(&emel::kernel::sm::optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_count, + matmul.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls); + ev.out.shared_q6_dispatch_calls = total( + &emel::kernel::sm::shared_q6_dispatch_count, + matmul.shared_q6_dispatch_calls); ev.out.native_quantized_stage_count = detail::quantized_contract_stage_count( ctx.compute.backend, emel::model::generation::quantized_contract_kind::native_quantized); diff --git a/src/emel/text/generator/context.hpp b/src/emel/text/generator/context.hpp index 82a43e13..4ad1c551 100644 --- a/src/emel/text/generator/context.hpp +++ b/src/emel/text/generator/context.hpp @@ -136,7 +136,9 @@ struct context { emel::text::generator::action::initializer_dispatch_fn * dispatch_initializer = nullptr; void * prefill_actor = nullptr; emel::text::generator::action::prefill_dispatch_fn * dispatch_prefill = nullptr; - emel::text::generator::matmul::sm * matmul_actor = nullptr; + emel::kernel::matmul::sm * matmul_actor = nullptr; + emel::kernel::matmul::lane_mode matmul_lane_mode = + emel::kernel::matmul::lane_mode::serial; tokenizer_binding conditioning = {}; graph_binding compute = {}; diff --git a/src/emel/text/generator/detail.hpp b/src/emel/text/generator/detail.hpp index dfc8546e..1eebbc7e 100644 --- a/src/emel/text/generator/detail.hpp +++ b/src/emel/text/generator/detail.hpp @@ -19,6 +19,7 @@ #include "emel/graph/processor/errors.hpp" #include "emel/kernel/events.hpp" +#include "emel/kernel/matmul/sm.hpp" #include "emel/kernel/sm.hpp" #include "emel/memory/view.hpp" #include "emel/model/data.hpp" @@ -26,7 +27,6 @@ #include "emel/model/loader/errors.hpp" #include "emel/model/tensor/window/sm.hpp" #include "emel/text/generator/events.hpp" -#include "emel/text/generator/matmul/sm.hpp" namespace emel::text::generator::detail { @@ -134,11 +134,7 @@ struct kv_addressing_view { int32_t recurrent_slot = 0; }; -using matmul_lane_mode = emel::text::generator::matmul::lane_mode; -using matmul_row_slice = - emel::text::generator::matmul::detail::matmul_row_slice; -using emel::text::generator::matmul::detail::compute_matmul_row_slices; -using emel::text::generator::matmul::detail::compute_sliced_mul_mat_event; +using matmul_lane_mode = emel::kernel::matmul::lane_mode; // Weight residency mode for the layer loop: resident consumes blocks[] views // as prepared; streamed acquires each layer's slot from the tensor window @@ -201,8 +197,10 @@ struct native_backend { emel::model::generation::step_plan prefill_plan = {}; emel::model::generation::step_plan decode_plan = {}; emel::kernel::sm kernel = {}; - emel::text::generator::matmul::sm *matmul_actor = nullptr; + emel::kernel::matmul::sm *matmul_actor = nullptr; route_policy routes = {}; + emel::kernel::matmul::lane_mode matmul_lane_mode = + emel::kernel::matmul::lane_mode::serial; bool parallel_lanes_enabled = true; emel::kernel::kernel_kind kernel_kind = emel::kernel::kernel_kind::x86_64; uint64_t kernel_dispatch_calls = 0; @@ -2217,9 +2215,8 @@ inline bool compute_mul_mat_sliced_parallel( native_backend &backend, const emel::kernel::event::op_mul_mat &ev) noexcept { bool accepted = false; - emel::text::generator::matmul::event::dispatch_result result = {}; - const emel::text::generator::matmul::event::execute_parallel run{ev, result, - accepted}; + emel::kernel::matmul::event::dispatch_result result = {}; + const emel::kernel::matmul::event::execute_parallel run{ev, result, accepted}; const bool dispatched = backend.matmul_actor->process_event(run); return dispatched && accepted; } @@ -2234,26 +2231,13 @@ compute_mul_mat(native_backend &backend, return compute_mul_mat_sliced_parallel(backend, ev); } else { bool accepted = false; - emel::text::generator::matmul::event::dispatch_result result = {}; - const emel::text::generator::matmul::event::execute_serial run{ev, result, - accepted}; + emel::kernel::matmul::event::dispatch_result result = {}; + const emel::kernel::matmul::event::execute_serial run{ev, result, accepted}; const bool dispatched = backend.matmul_actor->process_event(run); return dispatched && accepted; } } -// Evidence counters live per kernel actor; parallel slices accrue on lane -// actors, so audit reads must sum the primary kernel and every lane. -template -inline uint64_t compute_kernel_counter_total(const native_backend &backend, - counter_fn &&counter) noexcept { - uint64_t total = std::invoke(counter, backend.kernel); - if (backend.matmul_actor != nullptr) { - total += backend.matmul_actor->kernel_counter_total(counter); - } - return total; -} - template inline bool matmul_vector_packed_q8_0(native_backend &backend, const tensor_matrix &matrix, @@ -3947,8 +3931,8 @@ namespace emel::text::generator::layer { template bool run_layer(emel::text::generator::detail::native_backend &backend, @@ -3958,8 +3942,8 @@ bool run_layer(emel::text::generator::detail::native_backend &backend, template bool run_layer(emel::text::generator::detail::native_backend &backend, @@ -3969,8 +3953,8 @@ bool run_layer(emel::text::generator::detail::native_backend &backend, template bool run_layer(emel::text::generator::detail::native_backend &backend, @@ -3978,8 +3962,8 @@ bool run_layer(emel::text::generator::detail::native_backend &backend, template bool run_layer(emel::text::generator::detail::native_backend &backend, @@ -3993,14 +3977,14 @@ bool run_layer_nonflash(emel::text::generator::detail::native_backend &backend, template + emel::kernel::matmul::lane_mode lanes> bool run_layer_chunk4( emel::text::generator::detail::native_backend &backend, const emel::text::generator::detail::kv_addressing_view &kv, int32_t layer_index, size_t token_base) noexcept; template + emel::kernel::matmul::lane_mode lanes> bool run_layer_chunk8_q8_k( emel::text::generator::detail::native_backend &backend, const emel::text::generator::detail::kv_addressing_view &kv, @@ -5015,13 +4999,13 @@ inline bool run_decode_preselected_argmax( } // namespace -inline emel::error::type -prepare(native_backend &backend, - const emel::model::generation::contract &generation_contract, - emel::text::generator::matmul::sm &matmul_actor, - const runtime_policy &policy, - const int32_t kv_block_tokens = - emel::memory::view::DEFAULT_BLOCK_TOKENS) noexcept { +inline emel::error::type prepare( + native_backend &backend, + const emel::model::generation::contract &generation_contract, + emel::kernel::matmul::sm &matmul_actor, const runtime_policy &policy, + const int32_t kv_block_tokens = emel::memory::view::DEFAULT_BLOCK_TOKENS, + const emel::kernel::matmul::lane_mode matmul_lane_mode = + emel::kernel::matmul::lane_mode::parallel) noexcept { if (emel::model::generation::validate_contract(generation_contract) != emel::error::cast(emel::model::loader::error::none) || kv_block_tokens <= 0) { @@ -5031,12 +5015,12 @@ prepare(native_backend &backend, std::destroy_at(std::addressof(backend)); std::construct_at(std::addressof(backend)); backend.matmul_actor = &matmul_actor; + backend.matmul_lane_mode = matmul_lane_mode; backend.routes = policy.routes; backend.kernel_kind = policy.kernel_kind; backend.kernel.set_kind(backend.kernel_kind); backend.matmul_actor->process_event( - emel::text::generator::matmul::event::configure_kernel_kind{ - backend.kernel_kind}); + emel::kernel::matmul::event::configure_kernel_kind{backend.kernel_kind}); const auto &model_data = *generation_contract.execution.model; backend.execution = generation_contract.execution; diff --git a/src/emel/text/generator/guards.hpp b/src/emel/text/generator/guards.hpp index b8a20912..649cd005 100644 --- a/src/emel/text/generator/guards.hpp +++ b/src/emel/text/generator/guards.hpp @@ -1179,7 +1179,8 @@ struct guard_decode_parallel_lanes_ready { bool operator()(const event::generate_run &, const action::context & ctx) const noexcept { return ctx.compute.backend.parallel_lanes_enabled && ctx.compute.backend.matmul_actor != nullptr && - ctx.compute.backend.matmul_actor->parallel_lanes_available() && + ctx.compute.backend.matmul_lane_mode == + emel::kernel::matmul::lane_mode::parallel && ctx.compute.backend.n_embd >= ctx.compute.backend.routes.parallel_min_gemv_dim; } diff --git a/src/emel/text/generator/initializer/actions.hpp b/src/emel/text/generator/initializer/actions.hpp index 7ed0c1f3..42832448 100644 --- a/src/emel/text/generator/initializer/actions.hpp +++ b/src/emel/text/generator/initializer/actions.hpp @@ -51,7 +51,8 @@ struct request_backend_prepare { *generator.generation_contract, *generator.matmul_actor, generator.runtime_policy, - generator.limits.block_tokens)); + generator.limits.block_tokens, + generator.matmul_lane_mode)); ev.ctx.phase_accepted = ev.ctx.phase_code == static_cast(emel::error::cast(emel::model::loader::error::none)); diff --git a/src/emel/text/generator/layer/actions.hpp b/src/emel/text/generator/layer/actions.hpp index 4d1ad439..a7ccb00a 100644 --- a/src/emel/text/generator/layer/actions.hpp +++ b/src/emel/text/generator/layer/actions.hpp @@ -3,10 +3,10 @@ #include #include +#include "emel/kernel/matmul/detail.hpp" #include "emel/text/generator/detail.hpp" #include "emel/text/generator/events.hpp" #include "emel/text/generator/layer/events.hpp" -#include "emel/text/generator/matmul/detail.hpp" namespace emel::text::generator::layer::action { @@ -14,7 +14,7 @@ template + emel::kernel::matmul::lane_mode lanes> bool run_layer_attention_residual( emel::text::generator::detail::native_backend &backend, const int32_t layer_index, @@ -26,7 +26,7 @@ bool run_layer_attention_residual( } template + emel::kernel::matmul::lane_mode lanes> bool run_layer_shortconv_residual( emel::text::generator::detail::native_backend &backend, const int32_t layer_index, @@ -37,7 +37,7 @@ bool run_layer_shortconv_residual( } template + emel::kernel::matmul::lane_mode lanes> bool run_layer_feed_forward( emel::text::generator::detail::native_backend &backend, const int32_t layer_index) noexcept { @@ -50,7 +50,7 @@ template + emel::kernel::matmul::lane_mode lanes> bool run_layer_chunk4_attention_residual( emel::text::generator::detail::native_backend &backend, const emel::text::generator::detail::kv_addressing_view &kv, @@ -61,7 +61,7 @@ bool run_layer_chunk4_attention_residual( } template + emel::kernel::matmul::lane_mode lanes> bool run_layer_chunk4_shortconv_residual( emel::text::generator::detail::native_backend &backend, const emel::text::generator::detail::kv_addressing_view &kv, @@ -71,7 +71,7 @@ bool run_layer_chunk4_shortconv_residual( } template + emel::kernel::matmul::lane_mode lanes> bool run_layer_chunk4_feed_forward( emel::text::generator::detail::native_backend &backend, const int32_t layer_index) noexcept { @@ -82,7 +82,7 @@ bool run_layer_chunk4_feed_forward( template + emel::kernel::matmul::lane_mode lanes> bool run_layer_chunk8_q8_k_attention_residual( emel::text::generator::detail::native_backend &backend, const emel::text::generator::detail::kv_addressing_view &kv, @@ -93,7 +93,7 @@ bool run_layer_chunk8_q8_k_attention_residual( backend, kv, layer_index, token_base); } -template +template bool run_layer_chunk8_q8_k_shortconv_residual( emel::text::generator::detail::native_backend &backend, const emel::text::generator::detail::kv_addressing_view &kv, @@ -103,7 +103,7 @@ bool run_layer_chunk8_q8_k_shortconv_residual( layer_index); } -template +template bool run_layer_chunk8_q8_k_feed_forward( emel::text::generator::detail::native_backend &backend, const int32_t layer_index) noexcept { @@ -169,7 +169,7 @@ template + emel::kernel::matmul::lane_mode lanes> struct effect_run_scalar_attention { void operator()(const event::scalar_run &ev) const noexcept { ev.residual_ok = @@ -179,7 +179,7 @@ struct effect_run_scalar_attention { }; template + emel::kernel::matmul::lane_mode lanes> struct effect_run_scalar_shortconv { void operator()(const event::scalar_run &ev) const noexcept { ev.residual_ok = run_layer_shortconv_residual( @@ -188,7 +188,7 @@ struct effect_run_scalar_shortconv { }; template + emel::kernel::matmul::lane_mode lanes> struct effect_run_scalar_feed_forward { void operator()(const event::scalar_run &ev) const noexcept { ev.feed_forward_ok = @@ -200,7 +200,7 @@ template + emel::kernel::matmul::lane_mode lanes> struct effect_run_chunk4_attention { void operator()(const event::chunk4_run &ev) const noexcept { ev.residual_ok = run_layer_chunk4_attention_residual + emel::kernel::matmul::lane_mode lanes> struct effect_run_chunk4_shortconv { void operator()(const event::chunk4_run &ev) const noexcept { ev.residual_ok = run_layer_chunk4_shortconv_residual( @@ -219,7 +219,7 @@ struct effect_run_chunk4_shortconv { }; template + emel::kernel::matmul::lane_mode lanes> struct effect_run_chunk4_feed_forward { void operator()(const event::chunk4_run &ev) const noexcept { ev.feed_forward_ok = @@ -230,7 +230,7 @@ struct effect_run_chunk4_feed_forward { template + emel::kernel::matmul::lane_mode lanes> struct effect_run_chunk8_attention { void operator()(const event::chunk8_run &ev) const noexcept { ev.residual_ok = run_layer_chunk8_q8_k_attention_residual +template struct effect_run_chunk8_shortconv { void operator()(const event::chunk8_run &ev) const noexcept { ev.residual_ok = run_layer_chunk8_q8_k_shortconv_residual( @@ -247,7 +247,7 @@ struct effect_run_chunk8_shortconv { } }; -template +template struct effect_run_chunk8_feed_forward { void operator()(const event::chunk8_run &ev) const noexcept { ev.feed_forward_ok = diff --git a/src/emel/text/generator/layer/sm.hpp b/src/emel/text/generator/layer/sm.hpp index e51e433a..8bb0e077 100644 --- a/src/emel/text/generator/layer/sm.hpp +++ b/src/emel/text/generator/layer/sm.hpp @@ -15,7 +15,7 @@ struct state_feed_forward_done {}; template struct scalar_model { auto operator()() const { @@ -137,7 +137,7 @@ struct scalar_model { template + emel::kernel::matmul::lane_mode lanes> struct chunk4_model { auto operator()() const { namespace sml = stateforward::sml; @@ -247,7 +247,7 @@ struct chunk4_model { }; template + emel::kernel::matmul::lane_mode lanes> struct chunk8_model { auto operator()() const { namespace sml = stateforward::sml; @@ -354,22 +354,22 @@ struct chunk8_model { template using scalar_sm = emel::sm>; template + emel::kernel::matmul::lane_mode lanes> using chunk4_sm = emel::sm>; template + emel::kernel::matmul::lane_mode lanes> using chunk8_sm = emel::sm>; template struct scalar_actor { scalar_sm machine{}; @@ -381,7 +381,7 @@ struct scalar_actor { template + emel::kernel::matmul::lane_mode lanes> struct chunk4_actor { chunk4_sm machine{}; @@ -391,7 +391,7 @@ struct chunk4_actor { }; template + emel::kernel::matmul::lane_mode lanes> struct chunk8_actor { chunk8_sm machine{}; @@ -402,7 +402,7 @@ struct chunk8_actor { template inline void process_scalar(const event::scalar_run &ev) noexcept { scalar_actor actor{}; @@ -411,14 +411,14 @@ inline void process_scalar(const event::scalar_run &ev) noexcept { template + emel::kernel::matmul::lane_mode lanes> inline void process_chunk4(const event::chunk4_run &ev) noexcept { chunk4_actor actor{}; (void)actor.process_event(ev); } template + emel::kernel::matmul::lane_mode lanes> inline void process_chunk8(const event::chunk8_run &ev) noexcept { chunk8_actor actor{}; (void)actor.process_event(ev); @@ -426,7 +426,7 @@ inline void process_chunk8(const event::chunk8_run &ev) noexcept { template inline bool run_layer(emel::text::generator::detail::native_backend &backend, @@ -448,7 +448,7 @@ run_layer(emel::text::generator::detail::native_backend &backend, template inline bool run_layer(emel::text::generator::detail::native_backend &backend, const int32_t layer_index, const int32_t position, @@ -460,7 +460,7 @@ inline bool run_layer(emel::text::generator::detail::native_backend &backend, template inline bool run_layer(emel::text::generator::detail::native_backend &backend, @@ -474,7 +474,7 @@ run_layer(emel::text::generator::detail::native_backend &backend, template inline bool run_layer(emel::text::generator::detail::native_backend &backend, const int32_t layer_index, @@ -506,7 +506,7 @@ run_layer_nonflash(emel::text::generator::detail::native_backend &backend, template + emel::kernel::matmul::lane_mode lanes> inline bool run_layer_chunk4(emel::text::generator::detail::native_backend &backend, const emel::text::generator::detail::kv_addressing_view &kv, @@ -524,7 +524,7 @@ run_layer_chunk4(emel::text::generator::detail::native_backend &backend, } template + emel::kernel::matmul::lane_mode lanes> inline bool run_layer_chunk8_q8_k( emel::text::generator::detail::native_backend &backend, const emel::text::generator::detail::kv_addressing_view &kv, diff --git a/src/emel/text/generator/matmul/actions.hpp b/src/emel/text/generator/matmul/actions.hpp deleted file mode 100644 index 77e3de72..00000000 --- a/src/emel/text/generator/matmul/actions.hpp +++ /dev/null @@ -1,153 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "emel/text/generator/matmul/context.hpp" -#include "emel/text/generator/matmul/events.hpp" - -namespace emel::text::generator::matmul::action { - -struct effect_configure_kernel_kind { - void operator()(const event::configure_kernel_kind & ev, context & ctx) const noexcept { - ctx.kernel_kind = ev.kind; - ctx.kernel.set_kind(ctx.kernel_kind); - for (size_t lane = 0u; - lane < ctx.lane_capacity && ctx.lane_kernels != nullptr; ++lane) { - ctx.lane_kernels[lane].set_kind(ctx.kernel_kind); - } - } -}; - -struct effect_execute_serial { - void operator()(const event::execute_serial & ev, context & ctx) const noexcept { - ev.result = {}; - ev.result.lane_count = 1u; - ev.result.all_submitted = true; - ev.result.joined = true; - ctx.kernel.set_kind(ctx.kernel_kind); - ev.result.all_lanes_accepted = ctx.kernel.process_event(ev.request); - } -}; - -inline void execute_parallel_with_group_rows( - const event::execute_parallel & ev, - context & ctx, - const uint64_t group_rows) noexcept { - ev.result = {}; - const size_t lane_count = detail::compute_matmul_row_slices( - ev.request.src0.ne[1], group_rows, ctx.active_lanes, - std::span{ctx.row_slices.get(), ctx.lane_capacity}); - ev.result.lane_count = lane_count; - - lane_join_group group{}; - bool all_submitted = true; - for (size_t lane = 1u; lane < lane_count; ++lane) { - ctx.lane_events[lane] = - detail::compute_sliced_mul_mat_event(ev.request, group_rows, ctx.row_slices[lane]); - auto & lane_kernel = ctx.lane_kernels[lane]; - lane_kernel.set_kind(ctx.kernel_kind); - ctx.lane_dispatches[lane] = lane_dispatch{ - .kernel = &lane_kernel, - .request = &ctx.lane_events[lane], - }; - const bool submitted = ctx.parallel_matmul_lanes.try_submit( - group, lane_task{ - .run = run_lane_dispatch, - .ctx = &ctx.lane_dispatches[lane], - }); - all_submitted = all_submitted && submitted; - } - - ctx.lane_events[0] = - detail::compute_sliced_mul_mat_event(ev.request, group_rows, ctx.row_slices[0]); - ctx.lane_kernels[0].set_kind(ctx.kernel_kind); - bool all_accepted = ctx.lane_kernels[0].process_event(ctx.lane_events[0]); - ev.result.all_submitted = all_submitted; - ev.result.joined = group.wait(); - for (size_t lane = 1u; lane < lane_count; ++lane) { - all_accepted = all_accepted && ctx.lane_dispatches[lane].accepted; - } - ev.result.all_lanes_accepted = all_accepted; -} - -struct effect_execute_parallel_x8 { - void operator()(const event::execute_parallel & ev, - context & ctx) const noexcept { - execute_parallel_with_group_rows( - ev, ctx, emel::kernel::detail::quant::Q4_K_X8_ROWS); - } -}; - -struct effect_execute_parallel_x4 { - void operator()(const event::execute_parallel & ev, - context & ctx) const noexcept { - execute_parallel_with_group_rows( - ev, ctx, emel::kernel::detail::quant::Q8_0_X4_ROWS); - } -}; - -struct effect_execute_parallel_unit { - void operator()(const event::execute_parallel & ev, - context & ctx) const noexcept { - execute_parallel_with_group_rows(ev, ctx, 1u); - } -}; - -struct effect_accept_serial_execution { - void operator()(const event::execute_serial & ev, context &) const noexcept { - ev.accepted = true; - } -}; - -struct effect_accept_parallel_execution { - void operator()(const event::execute_parallel & ev, context &) const noexcept { - ev.accepted = true; - } -}; - -struct effect_reject_serial_execution { - void operator()(const event::execute_serial & ev, context &) const noexcept { - ev.accepted = false; - } -}; - -struct effect_reject_parallel_execution { - void operator()(const event::execute_parallel & ev, context &) const noexcept { - ev.accepted = false; - } -}; - -struct effect_on_unexpected { - template - void operator()(const event_type & ev, context &) const noexcept { - if constexpr (requires { ev.accepted; }) { - ev.accepted = false; - } - } -}; - -template -uint64_t compute_kernel_counter_total(const context & ctx, counter_fn && counter) noexcept { - uint64_t total = std::invoke(counter, ctx.kernel); - for (size_t lane = 0u; - lane < ctx.lane_capacity && ctx.lane_kernels != nullptr; ++lane) { - total += std::invoke(counter, ctx.lane_kernels[lane]); - } - return total; -} - -inline constexpr effect_configure_kernel_kind effect_configure_kernel_kind{}; -inline constexpr effect_execute_serial effect_execute_serial{}; -inline constexpr effect_execute_parallel_x8 effect_execute_parallel_x8{}; -inline constexpr effect_execute_parallel_x4 effect_execute_parallel_x4{}; -inline constexpr effect_execute_parallel_unit effect_execute_parallel_unit{}; -inline constexpr effect_accept_serial_execution effect_accept_serial_execution{}; -inline constexpr effect_accept_parallel_execution effect_accept_parallel_execution{}; -inline constexpr effect_reject_serial_execution effect_reject_serial_execution{}; -inline constexpr effect_reject_parallel_execution effect_reject_parallel_execution{}; -inline constexpr effect_on_unexpected effect_on_unexpected{}; - -} // namespace emel::text::generator::matmul::action diff --git a/src/emel/text/generator/matmul/context.hpp b/src/emel/text/generator/matmul/context.hpp deleted file mode 100644 index d012b7f3..00000000 --- a/src/emel/text/generator/matmul/context.hpp +++ /dev/null @@ -1,83 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -#include "emel/kernel/sm.hpp" -#include "emel/text/generator/matmul/detail.hpp" - -namespace emel::text::generator::matmul::action { - -using lane_pool_ref = emel::text::generator::matmul::lane_pool_ref; - -struct lane_dispatch { - emel::kernel::sm * kernel = nullptr; - const emel::kernel::event::op_mul_mat * request = nullptr; - bool accepted = false; -}; - -inline void run_lane_dispatch(void * ctx) noexcept { - auto & dispatch = *static_cast(ctx); - dispatch.accepted = - dispatch.kernel != nullptr && dispatch.request != nullptr && - dispatch.kernel->process_event(*dispatch.request); -} - -struct context { - lane_pool_ref parallel_matmul_lanes = {}; - emel::kernel::kernel_kind kernel_kind = emel::kernel::kernel_kind::x86_64; - size_t active_lanes = 1u; - size_t lane_capacity = 0u; - emel::kernel::sm kernel = {}; - std::unique_ptr lane_kernels = {}; - std::unique_ptr - row_slices = {}; - std::unique_ptr lane_events = {}; - std::unique_ptr lane_dispatches = {}; -}; - -inline bool reserve_lane_storage(context & ctx, - const size_t lane_capacity) noexcept { - if (lane_capacity == 0u) { - ctx.lane_capacity = 0u; - ctx.lane_kernels.reset(); - ctx.row_slices.reset(); - ctx.lane_events.reset(); - ctx.lane_dispatches.reset(); - return true; - } - - auto lane_kernels = std::unique_ptr( - new (std::nothrow) emel::kernel::sm[lane_capacity]); - auto row_slices = std::unique_ptr< - emel::text::generator::matmul::detail::matmul_row_slice[]>( - new (std::nothrow) - emel::text::generator::matmul::detail::matmul_row_slice[lane_capacity]); - auto lane_events = std::unique_ptr( - new (std::nothrow) emel::kernel::event::op_mul_mat[lane_capacity]); - auto lane_dispatches = std::unique_ptr( - new (std::nothrow) lane_dispatch[lane_capacity]); - if (lane_kernels == nullptr || row_slices == nullptr || - lane_events == nullptr || lane_dispatches == nullptr) { - return false; - } - - ctx.lane_capacity = lane_capacity; - ctx.lane_kernels = std::move(lane_kernels); - ctx.row_slices = std::move(row_slices); - ctx.lane_events = std::move(lane_events); - ctx.lane_dispatches = std::move(lane_dispatches); - return true; -} - -inline bool lane_storage_ready(const context & ctx) noexcept { - return ctx.lane_capacity == ctx.parallel_matmul_lanes.lane_capacity && - ctx.lane_capacity > 0u && ctx.lane_kernels != nullptr && - ctx.row_slices != nullptr && ctx.lane_events != nullptr && - ctx.lane_dispatches != nullptr && ctx.active_lanes > 1u && - ctx.active_lanes <= ctx.lane_capacity; -} - -} // namespace emel::text::generator::matmul::action diff --git a/src/emel/text/generator/matmul/detail.hpp b/src/emel/text/generator/matmul/detail.hpp deleted file mode 100644 index 7e4367ff..00000000 --- a/src/emel/text/generator/matmul/detail.hpp +++ /dev/null @@ -1,201 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "emel/kernel/detail.hpp" -#include "emel/kernel/events.hpp" -#include "emel/sm.hpp" - -namespace emel::text::generator::matmul { - -template -using lane_pool = - emel::policy::fork_join_lane_pool; - -struct lane_task { - void (*run)(void *) noexcept = nullptr; - void * ctx = nullptr; - - void operator()() const noexcept { - if (run != nullptr) { - run(ctx); - } - } -}; - -class lane_join_group { - public: - lane_join_group() = default; - - lane_join_group(const lane_join_group &) = delete; - lane_join_group & operator=(const lane_join_group &) = delete; - - ~lane_join_group() { reset(); } - - template - group_type & get() noexcept { - static_assert(sizeof(group_type) <= storage_bytes, - "matmul lane join group storage is too small"); - static_assert(alignof(group_type) <= alignof(std::max_align_t), - "matmul lane join group alignment is too large"); - if (self_ == nullptr) { - self_ = new (storage.data()) group_type(); - wait_ = [](void * ptr) noexcept { - return static_cast(ptr)->wait(); - }; - destroy_ = [](void * ptr) noexcept { - static_cast(ptr)->~group_type(); - }; - } - return *static_cast(self_); - } - - bool wait() noexcept { - return wait_ == nullptr || wait_(self_); - } - - void reset() noexcept { - if (destroy_ != nullptr) { - destroy_(self_); - } - self_ = nullptr; - wait_ = nullptr; - destroy_ = nullptr; - } - - private: - static constexpr size_t storage_bytes = 64u; - - alignas(std::max_align_t) std::array storage = {}; - void * self_ = nullptr; - bool (*wait_)(void *) noexcept = nullptr; - void (*destroy_)(void *) noexcept = nullptr; -}; - -struct lane_pool_ref { - void * pool = nullptr; - size_t worker_lanes = 0u; - size_t lane_capacity = 0u; - bool (*submit)(void *, lane_join_group &, lane_task) noexcept = nullptr; - bool (*is_current_thread_worker)(void *) noexcept = nullptr; - - bool valid() const noexcept { - return pool != nullptr && submit != nullptr && worker_lanes > 0u && - lane_capacity == worker_lanes + 1u; - } - - bool try_submit(lane_join_group & group, lane_task task) const noexcept { - return submit != nullptr && submit(pool, group, task); - } -}; - -template -lane_pool_ref make_lane_pool_ref(pool_type & pool) noexcept { - using group_type = typename pool_type::join_group; - return lane_pool_ref{ - .pool = &pool, - .worker_lanes = pool_type::static_worker_count, - .lane_capacity = pool_type::static_worker_count + 1u, - .submit = - [](void * pool_ptr, lane_join_group & group, lane_task task) noexcept { - auto & typed_group = group.get(); - return static_cast(pool_ptr)->try_submit(typed_group, task); - }, - .is_current_thread_worker = - [](void * pool_ptr) noexcept { - return static_cast(pool_ptr)->is_current_thread_worker(); - }, - }; -} - -struct execution_policy { - lane_pool_ref parallel_matmul_lanes; - emel::kernel::kernel_kind kernel_kind; - size_t active_lanes; -}; - -template -inline execution_policy make_execution_policy(pool_type & parallel_matmul_lanes, - const emel::kernel::kernel_kind kernel_kind, - const size_t active_lanes) noexcept { - const lane_pool_ref pool_ref = make_lane_pool_ref(parallel_matmul_lanes); - return execution_policy{ - .parallel_matmul_lanes = pool_ref, - .kernel_kind = kernel_kind, - .active_lanes = active_lanes, - }; -} - -template -inline execution_policy make_auto_execution_policy(pool_type & parallel_matmul_lanes) noexcept { - const lane_pool_ref pool_ref = make_lane_pool_ref(parallel_matmul_lanes); - return execution_policy{ - .parallel_matmul_lanes = pool_ref, - .kernel_kind = emel::kernel::detect_host_kind(), - .active_lanes = pool_ref.lane_capacity, - }; -} - -enum class lane_mode : uint8_t { - serial = 0, - parallel = 1, -}; - -namespace detail { - -struct matmul_row_slice { - int32_t row_begin = 0; - int32_t row_count = 0; -}; - -inline size_t compute_matmul_row_slices( - const uint64_t rows, - const uint64_t group_rows, - const size_t active_lanes, - std::span slices) noexcept { - const uint64_t groups = (rows + group_rows - 1u) / group_rows; - const uint64_t lane_capacity = - std::min(slices.size(), std::max(active_lanes, 1u)); - const uint64_t lane_count = std::min(lane_capacity, std::max(groups, 1u)); - const uint64_t groups_per_lane = groups / lane_count; - const uint64_t extra_groups = groups % lane_count; - uint64_t begin_group = 0u; - for (uint64_t lane = 0u; lane < lane_count; ++lane) { - const uint64_t lane_groups = groups_per_lane + static_cast(lane < extra_groups); - const uint64_t begin_row = begin_group * group_rows; - const uint64_t end_row = std::min(rows, (begin_group + lane_groups) * group_rows); - slices[lane].row_begin = static_cast(begin_row); - slices[lane].row_count = static_cast(end_row - begin_row); - begin_group += lane_groups; - } - return static_cast(lane_count); -} - -inline emel::kernel::event::op_mul_mat compute_sliced_mul_mat_event( - const emel::kernel::event::op_mul_mat & ev, - const uint64_t group_rows, - const matmul_row_slice slice) noexcept { - emel::kernel::event::op_mul_mat sliced = ev; - const uint64_t begin = static_cast(slice.row_begin); - const uint64_t count = static_cast(slice.row_count); - const uint64_t slice_groups = (count + group_rows - 1u) / group_rows; - sliced.src0.data = - static_cast(ev.src0.data) + (begin / group_rows) * ev.src0.nb[1]; - sliced.src0.ne[1] = count; - sliced.src0.nb[2] = ev.src0.nb[1] * slice_groups; - sliced.src0.nb[3] = sliced.src0.nb[2]; - sliced.dst.data = static_cast(ev.dst.data) + begin * ev.dst.nb[1]; - sliced.dst.ne[1] = count; - sliced.dst.nb[2] = ev.dst.nb[1] * count; - sliced.dst.nb[3] = sliced.dst.nb[2]; - return sliced; -} - -} // namespace detail - -} // namespace emel::text::generator::matmul diff --git a/src/emel/text/generator/matmul/events.hpp b/src/emel/text/generator/matmul/events.hpp deleted file mode 100644 index 709509da..00000000 --- a/src/emel/text/generator/matmul/events.hpp +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - -#include -#include - -#include "emel/kernel/events.hpp" -#include "emel/kernel/sm.hpp" -#include "emel/text/generator/matmul/detail.hpp" - -namespace emel::text::generator::matmul::event { - -struct configure_kernel_kind { - emel::kernel::kernel_kind kind = emel::kernel::kernel_kind::x86_64; -}; - -struct dispatch_result { - size_t lane_count = 0u; - bool all_submitted = false; - bool joined = false; - bool all_lanes_accepted = false; -}; - -struct execute_serial { - execute_serial(const emel::kernel::event::op_mul_mat & request_ref, - dispatch_result & result_ref, - bool & accepted_ref) noexcept - : request(request_ref), result(result_ref), accepted(accepted_ref) {} - - const emel::kernel::event::op_mul_mat & request; - dispatch_result & result; - bool & accepted; -}; - -struct execute_parallel { - execute_parallel(const emel::kernel::event::op_mul_mat & request_ref, - dispatch_result & result_ref, - bool & accepted_ref) noexcept - : request(request_ref), result(result_ref), accepted(accepted_ref) {} - - const emel::kernel::event::op_mul_mat & request; - dispatch_result & result; - bool & accepted; -}; - -} // namespace emel::text::generator::matmul::event diff --git a/src/emel/text/generator/matmul/guards.hpp b/src/emel/text/generator/matmul/guards.hpp deleted file mode 100644 index 8022466c..00000000 --- a/src/emel/text/generator/matmul/guards.hpp +++ /dev/null @@ -1,116 +0,0 @@ -#pragma once - -#include "emel/text/generator/matmul/context.hpp" -#include "emel/text/generator/matmul/events.hpp" - -namespace emel::text::generator::matmul::guard { - -inline bool guard_parallel_storage_ready( - const event::execute_parallel & ev, - const action::context & ctx) noexcept { - return ctx.parallel_matmul_lanes.valid() && - action::lane_storage_ready(ctx) && ev.request.src0.ne[1] > 0u; -} - -inline bool guard_uses_x8_row_groups( - const emel::kernel::event::dtype type) noexcept { - const uint8_t code = emel::kernel::detail::dtype_code(type); - return code == emel::kernel::detail::dtype_q4_k_x8_bl4 || - code == emel::kernel::detail::dtype_q4_k_x8_bl8 || - code == emel::kernel::detail::dtype_q6_k_x8 || - code == emel::kernel::detail::dtype_q6_k_x8_q8_prepared || - code == emel::kernel::detail::dtype_q6_k_x8_q8_argmax_prepared; -} - -inline bool guard_uses_x4_row_groups( - const emel::kernel::event::dtype type) noexcept { - const uint8_t code = emel::kernel::detail::dtype_code(type); - return code == emel::kernel::detail::dtype_q8_0_x4_bl4 || - code == emel::kernel::detail::dtype_q8_0_x4_bl8; -} - -struct guard_parallel_ready { - bool operator()(const event::execute_parallel & ev, - const action::context & ctx) const noexcept { - return guard_parallel_storage_ready(ev, ctx); - } -}; - -struct guard_parallel_unavailable { - bool operator()(const event::execute_parallel & ev, - const action::context & ctx) const noexcept { - return !ctx.parallel_matmul_lanes.valid() || - !action::lane_storage_ready(ctx) || ev.request.src0.ne[1] == 0u; - } -}; - -struct guard_parallel_x8_ready { - bool operator()(const event::execute_parallel & ev, - const action::context & ctx) const noexcept { - return guard_parallel_storage_ready(ev, ctx) && - guard_uses_x8_row_groups(ev.request.src0.type); - } -}; - -struct guard_parallel_x4_ready { - bool operator()(const event::execute_parallel & ev, - const action::context & ctx) const noexcept { - return guard_parallel_storage_ready(ev, ctx) && - guard_uses_x4_row_groups(ev.request.src0.type); - } -}; - -struct guard_parallel_unit_ready { - bool operator()(const event::execute_parallel & ev, - const action::context & ctx) const noexcept { - return guard_parallel_storage_ready(ev, ctx) && - !guard_uses_x8_row_groups(ev.request.src0.type) && - !guard_uses_x4_row_groups(ev.request.src0.type); - } -}; - -struct guard_serial_accepted { - bool operator()(const event::execute_serial & ev, - const action::context &) const noexcept { - return ev.result.all_lanes_accepted; - } -}; - -struct guard_serial_rejected { - bool operator()(const event::execute_serial & ev, - const action::context &) const noexcept { - return !ev.result.all_lanes_accepted; - } -}; - -struct guard_parallel_submission_failed { - bool operator()(const event::execute_parallel & ev, - const action::context &) const noexcept { - return !ev.result.all_submitted; - } -}; - -struct guard_parallel_join_failed { - bool operator()(const event::execute_parallel & ev, - const action::context &) const noexcept { - return ev.result.all_submitted && !ev.result.joined; - } -}; - -struct guard_parallel_lane_rejected { - bool operator()(const event::execute_parallel & ev, - const action::context &) const noexcept { - return ev.result.all_submitted && ev.result.joined && - !ev.result.all_lanes_accepted; - } -}; - -struct guard_parallel_all_lanes_accepted { - bool operator()(const event::execute_parallel & ev, - const action::context &) const noexcept { - return ev.result.all_submitted && ev.result.joined && - ev.result.all_lanes_accepted; - } -}; - -} // namespace emel::text::generator::matmul::guard diff --git a/src/emel/text/generator/matmul/sm.hpp b/src/emel/text/generator/matmul/sm.hpp deleted file mode 100644 index daf9d225..00000000 --- a/src/emel/text/generator/matmul/sm.hpp +++ /dev/null @@ -1,156 +0,0 @@ -#pragma once -// benchmark: designed - -#include -#include -#include - -#include "emel/sm.hpp" -#include "emel/text/generator/matmul/actions.hpp" -#include "emel/text/generator/matmul/guards.hpp" - -namespace emel::text::generator::matmul { - -struct state_ready {}; -struct state_serial_result_decision {}; -struct state_parallel_result_decision {}; - -struct model { - auto operator()() const { - namespace sml = stateforward::sml; - - // clang-format off - return sml::make_transition_table( - //------------------------------------------------------------------------------// - // Kernel binding. - sml::state <= *sml::state - + sml::event - / action::effect_configure_kernel_kind - - //------------------------------------------------------------------------------// - // Explicit serial versus parallel matmul execution. - , sml::state <= sml::state - + sml::event - / action::effect_execute_serial - - , sml::state <= sml::state - + sml::completion - [ guard::guard_serial_accepted{} ] - / action::effect_accept_serial_execution - - , sml::state <= sml::state - + sml::completion - [ guard::guard_serial_rejected{} ] - / action::effect_reject_serial_execution - - , sml::state <= sml::state - + sml::event - [ guard::guard_parallel_x8_ready{} ] - / action::effect_execute_parallel_x8 - - , sml::state <= sml::state - + sml::event - [ guard::guard_parallel_x4_ready{} ] - / action::effect_execute_parallel_x4 - - , sml::state <= sml::state - + sml::event - [ guard::guard_parallel_unit_ready{} ] - / action::effect_execute_parallel_unit - - , sml::state <= sml::state - + sml::event - [ guard::guard_parallel_unavailable{} ] - / action::effect_reject_parallel_execution - - , sml::state <= sml::state - + sml::completion - [ guard::guard_parallel_submission_failed{} ] - / action::effect_reject_parallel_execution - - , sml::state <= sml::state - + sml::completion - [ guard::guard_parallel_join_failed{} ] - / action::effect_reject_parallel_execution - - , sml::state <= sml::state - + sml::completion - [ guard::guard_parallel_lane_rejected{} ] - / action::effect_reject_parallel_execution - - , sml::state <= sml::state - + sml::completion - [ guard::guard_parallel_all_lanes_accepted{} ] - / action::effect_accept_parallel_execution - - //------------------------------------------------------------------------------// - // Unexpected events. - , sml::state <= sml::state + sml::unexpected_event - / action::effect_on_unexpected - , sml::state <= sml::state - + sml::unexpected_event - / action::effect_on_unexpected - , sml::state <= sml::state - + sml::unexpected_event - / action::effect_on_unexpected - ); - // clang-format on - } -}; - -struct sm : public emel::sm { - using base_type = emel::sm; - using base_type::is; - using base_type::visit_current_states; - - explicit sm(const execution_policy & policy) : base_type() { - this->context_.parallel_matmul_lanes = policy.parallel_matmul_lanes; - this->context_.kernel_kind = policy.kernel_kind; - this->context_.active_lanes = policy.active_lanes; - if (!action::reserve_lane_storage( - this->context_, policy.parallel_matmul_lanes.lane_capacity)) { - std::terminate(); - } - process_event(event::configure_kernel_kind{policy.kernel_kind}); - } - - bool process_event(const event::configure_kernel_kind & ev) { - return base_type::process_event(ev); - } - - bool process_event(const event::execute_serial & ev) { - return base_type::process_event(ev); - } - - bool process_event(const event::execute_parallel & ev) { - return base_type::process_event(ev); - } - - template - uint64_t kernel_counter_total(counter_fn && counter) const noexcept { - return action::compute_kernel_counter_total( - this->context_, std::forward(counter)); - } - - const emel::kernel::sm & serial_kernel() const noexcept { - return this->context_.kernel; - } - - bool parallel_lanes_available() const noexcept { - return this->context_.parallel_matmul_lanes.valid() && - action::lane_storage_ready(this->context_); - } - - size_t active_lane_count() const noexcept { - return this->context_.active_lanes; - } - - std::span parallel_lane_kernels() const noexcept { - if (this->context_.lane_kernels == nullptr) { - return {}; - } - return {this->context_.lane_kernels.get(), this->context_.lane_capacity}; - } -}; - -} // namespace emel::text::generator::matmul diff --git a/src/emel/text/generator/prefill/guards.hpp b/src/emel/text/generator/prefill/guards.hpp index 7cbbacd4..93242c91 100644 --- a/src/emel/text/generator/prefill/guards.hpp +++ b/src/emel/text/generator/prefill/guards.hpp @@ -61,7 +61,8 @@ inline bool uses_parallel_matmul_lanes(const event::run & ev, const action::context & ctx) noexcept { return ctx.generator.compute.backend.parallel_lanes_enabled && ctx.generator.compute.backend.matmul_actor != nullptr && - ctx.generator.compute.backend.matmul_actor->parallel_lanes_available() && + ctx.generator.compute.backend.matmul_lane_mode == + emel::kernel::matmul::lane_mode::parallel && ev.ctx.prompt_token_count >= ctx.generator.compute.backend.routes.parallel_min_prefill_tokens; } diff --git a/src/emel/text/generator/sm.hpp b/src/emel/text/generator/sm.hpp index fb9ae4e8..b1790d68 100644 --- a/src/emel/text/generator/sm.hpp +++ b/src/emel/text/generator/sm.hpp @@ -51,10 +51,11 @@ inline bool dispatch_prefill_run(void * actor, inline void bind_matmul_actor( action::context & ctx, - std::optional & actor, - const emel::text::generator::matmul::execution_policy & policy) { + std::optional & actor, + const emel::kernel::matmul::execution_policy & policy) { actor.emplace(policy); ctx.matmul_actor = &actor.value(); + ctx.matmul_lane_mode = policy.mode; } inline void bind_pipeline_actors( @@ -74,7 +75,7 @@ inline void bind_pipeline_actors( struct dependencies { const emel::model::generation::contract & generation_contract; emel::text::conditioner::sm & conditioner; - emel::text::generator::matmul::execution_policy matmul_policy; + emel::kernel::matmul::execution_policy matmul_policy; emel::text::generator::runtime_policy runtime_policy; void * formatter_ctx = nullptr; emel::text::formatter::format_fn format_prompt = @@ -1188,7 +1189,7 @@ struct sm : public emel::sm { } private: - std::optional matmul_actor_ = {}; + std::optional matmul_actor_ = {}; std::optional initializer_actor_ = {}; std::optional prefill_actor_ = {}; }; diff --git a/tests/kernel/aarch64_tests.cpp b/tests/kernel/aarch64_tests.cpp index 9972bb48..67cf0fa7 100644 --- a/tests/kernel/aarch64_tests.cpp +++ b/tests/kernel/aarch64_tests.cpp @@ -1588,6 +1588,85 @@ TEST_CASE("kernel_aarch64_q4_k_4rows_neon_matches_scalar") { #endif } +TEST_CASE("kernel_aarch64_q4_k_multirow_preserves_one_row_accumulation_order") { +#if !(defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD)) + CHECK(true); +#else + using emel::kernel::detail::quant::block_q4_k; + using emel::kernel::detail::quant::block_q8_k; + + constexpr size_t ROW_COUNT = 4u; + constexpr size_t MAX_BLOCK_COUNT = 64u; + std::array q4_rows = {}; + std::array q8_blocks = {}; + for (size_t row = 0; row < ROW_COUNT; ++row) { + for (size_t block = 0; block < MAX_BLOCK_COUNT; ++block) { + auto &q4 = q4_rows[row * MAX_BLOCK_COUNT + block]; + q4.d = emel::kernel::detail::quant::fp32_to_fp16( + 0.0009765625f * + static_cast(((row + 3u) * (block + 5u)) % 31u + 1u)); + q4.dmin = emel::kernel::detail::quant::fp32_to_fp16( + 0.00048828125f * + static_cast(((row + 7u) * (block + 11u)) % 29u + 1u)); + for (size_t index = 0; index < q4.scales.size(); ++index) { + q4.scales[index] = static_cast( + (row * 41u + block * 37u + index * 19u + 0x5au) & 0xffu); + } + for (size_t index = 0; index < q4.qs.size(); ++index) { + q4.qs[index] = static_cast( + (row * 23u + block * 17u + index * 29u + 0x96u) & 0xffu); + } + } + } + for (size_t block = 0; block < MAX_BLOCK_COUNT; ++block) { + auto &q8 = q8_blocks[block]; + q8.d = 0.000244140625f * static_cast((block % 23u) + 1u); + for (size_t index = 0; index < q8.qs.size(); ++index) { + q8.qs[index] = static_cast( + static_cast((block * 13u + index * 7u) % 255u) - 127); + } + for (size_t group = 0; group < q8.bsums.size(); ++group) { + int32_t group_sum = 0; + for (size_t index = 0; index < 16u; ++index) { + group_sum += static_cast(q8.qs[group * 16u + index]); + } + q8.bsums[group] = static_cast(group_sum); + } + } + + constexpr std::array BLOCK_COUNTS = {1u, 2u, 7u, 64u}; + for (const uint64_t block_count : BLOCK_COUNTS) { + std::array one_row = {}; + for (size_t row = 0; row < ROW_COUNT; ++row) { + one_row[row] = emel::kernel::aarch64::detail::dot_q4_k_q8_k_row_neon( + q4_rows.data() + row * MAX_BLOCK_COUNT, q8_blocks.data(), + block_count); + } + + std::array two_rows = {}; + emel::kernel::aarch64::detail::dot_q4_k_q8_k_2rows_neon( + q4_rows.data(), q4_rows.data() + MAX_BLOCK_COUNT, q8_blocks.data(), + block_count, two_rows.data()); + CHECK(std::bit_cast(two_rows[0]) == + std::bit_cast(one_row[0])); + CHECK(std::bit_cast(two_rows[1]) == + std::bit_cast(one_row[1])); + + std::array four_rows = {}; + emel::kernel::aarch64::detail::dot_q4_k_q8_k_4rows_neon( + q4_rows.data(), q4_rows.data() + MAX_BLOCK_COUNT, + q4_rows.data() + MAX_BLOCK_COUNT * 2u, + q4_rows.data() + MAX_BLOCK_COUNT * 3u, q8_blocks.data(), block_count, + four_rows.data()); + for (size_t row = 0; row < ROW_COUNT; ++row) { + CHECK(std::bit_cast(four_rows[row]) == + std::bit_cast(one_row[row])); + } + } +#endif +} + TEST_CASE("kernel_aarch64_sm_reports_q2_vectorized_dispatch_at_kernel_seam") { using emel::kernel::detail::quant::block_q2_k; using emel::kernel::detail::quant::block_q3_k; @@ -5161,59 +5240,67 @@ TEST_CASE("kernel_aarch64_detail_branch_paths") { CHECK_FALSE(emel::kernel::aarch64::detail::can_use_neon(unary_ev, true)); } -TEST_CASE("kernel_aarch64_q4_k_vector_q8_rhs_tail_rows_match_scalar") { +TEST_CASE("kernel_aarch64_q4_k_vector_q8_rhs_tail_rows_match_one_row") { using emel::kernel::detail::quant::block_q4_k; using emel::kernel::detail::quant::block_q8_k; using emel::kernel::detail::quant::QK_K; - constexpr uint64_t row_count = 5u; - constexpr uint64_t block_count = 1u; + constexpr uint64_t ROW_COUNT = 5u; + constexpr uint64_t BLOCK_COUNT = 7u; + constexpr uint64_t COL_COUNT = QK_K * BLOCK_COUNT; - std::array native_rows = {}; - for (size_t row = 0; row < native_rows.size(); ++row) { - auto &q4 = native_rows[row]; - q4.d = emel::kernel::detail::quant::fp32_to_fp16( - 0.0009765625f * static_cast(row + 1u)); - q4.dmin = emel::kernel::detail::quant::fp32_to_fp16( - 0.000244140625f * static_cast(row + 3u)); - for (size_t idx = 0; idx < q4.scales.size(); ++idx) { - q4.scales[idx] = static_cast((row * 11u + idx * 7u) & 0xffu); - } - for (size_t idx = 0; idx < q4.qs.size(); ++idx) { - q4.qs[idx] = static_cast((row * 13u + idx * 5u) & 0xffu); + std::array native_rows = {}; + for (size_t row = 0; row < ROW_COUNT; ++row) { + for (size_t block = 0; block < BLOCK_COUNT; ++block) { + auto &q4 = native_rows[row * BLOCK_COUNT + block]; + q4.d = emel::kernel::detail::quant::fp32_to_fp16( + 0.0009765625f * static_cast((row + block) % 31u + 1u)); + q4.dmin = emel::kernel::detail::quant::fp32_to_fp16( + 0.000244140625f * static_cast((row + block * 3u) % 29u + 1u)); + for (size_t idx = 0; idx < q4.scales.size(); ++idx) { + q4.scales[idx] = + static_cast((row * 11u + block * 17u + idx * 7u) & 0xffu); + } + for (size_t idx = 0; idx < q4.qs.size(); ++idx) { + q4.qs[idx] = + static_cast((row * 13u + block * 19u + idx * 5u) & 0xffu); + } } } - std::array rhs_dense = {}; + std::array rhs_dense = {}; for (size_t idx = 0; idx < rhs_dense.size(); ++idx) { rhs_dense[idx] = static_cast(static_cast((idx * 17u) % 127u) - 63) * 0.0078125f; } - std::array rhs_q8 = {}; + std::array rhs_q8 = {}; emel::kernel::detail::quant::quantize_row_q8_k_strided( - rhs_dense.data(), 1u, rhs_q8.data(), static_cast(QK_K)); + rhs_dense.data(), 1u, rhs_q8.data(), static_cast(COL_COUNT)); - std::array reference = {}; - for (size_t row = 0; row < row_count; ++row) { - reference[row] = emel::kernel::detail::dot_q4_k_q8_k_row_scalar( - native_rows.data() + row, rhs_q8.data(), block_count); + std::array reference = {}; + for (size_t row = 0; row < ROW_COUNT; ++row) { + reference[row] = emel::kernel::aarch64::detail::dot_q4_k_q8_k_row_neon( + native_rows.data() + row * BLOCK_COUNT, rhs_q8.data(), BLOCK_COUNT); } - std::array out = {}; + std::array out = {}; const emel::kernel::event::op_mul_mat ev{ - .src0 = - make_quantized_src(native_rows.data(), dtype::q4_k, QK_K, row_count), - .src1 = make_q8_k_vector_src(rhs_q8.data(), QK_K), - .dst = make_dst(out.data(), dtype::f32, 1u, row_count), + .src0 = make_quantized_src(native_rows.data(), dtype::q4_k, COL_COUNT, + ROW_COUNT), + .src1 = make_q8_k_vector_src(rhs_q8.data(), COL_COUNT), + .dst = make_dst(out.data(), dtype::f32, 1u, ROW_COUNT), }; #if defined(__aarch64__) && defined(__ARM_NEON) && \ defined(__ARM_FEATURE_DOTPROD) aarch64_sm machine{}; + allocation_scope allocations{}; REQUIRE(machine.process_event(ev)); + CHECK(allocations.allocations() == 0u); for (size_t idx = 0; idx < out.size(); ++idx) { - CHECK(out[idx] == doctest::Approx(reference[idx]).epsilon(1.0e-4f)); + CHECK(std::bit_cast(out[idx]) == + std::bit_cast(reference[idx])); } #else aarch64_sm machine{}; diff --git a/tests/kernel/attention_tests.cpp b/tests/kernel/attention_tests.cpp new file mode 100644 index 00000000..1270243a --- /dev/null +++ b/tests/kernel/attention_tests.cpp @@ -0,0 +1,360 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../allocation_tracker.hpp" +#include "emel/kernel/attention/sm.hpp" +#include "emel/kernel/detail.hpp" + +namespace { + +namespace attention = emel::kernel::attention; + +inline constexpr int32_t k_heads = 8; +inline constexpr int32_t k_head_dim = 16; +inline constexpr int32_t k_hidden_dim = k_heads * k_head_dim; +inline constexpr int32_t k_capacity = 5; + +struct attention_fixture { + attention_fixture() { + for (std::size_t index = 0u; index < query.size(); ++index) { + query[index] = + static_cast(static_cast(index) - 7) / 16.0f; + } + for (std::size_t index = 0u; index < key.size(); ++index) { + const float key_value = + static_cast(static_cast(index % 19u) - 9) / 32.0f; + const float value_value = + static_cast(static_cast(index % 23u) - 11) / 24.0f; + key[index] = emel::kernel::detail::fp32_to_bf16(key_value); + value[index] = emel::kernel::detail::fp32_to_bf16(value_value); + } + } + + attention::event::head_range_request + request(const int32_t head_begin, const int32_t head_end, + std::span output) const noexcept { + return attention::event::head_range_request{ + .query = std::span{query}, + .key_cache = std::span{key}, + .value_cache = std::span{value}, + .output = output, + .layer_offset = 0u, + .hidden_dim = k_hidden_dim, + .head_dim = k_head_dim, + .head_begin = head_begin, + .head_end = head_end, + .position_capacity = k_capacity, + .physical_begin = 3, + .valid_positions = 4, + }; + } + + std::array(k_hidden_dim)> query = {}; + std::array(k_capacity *k_hidden_dim)> key = + {}; + std::array(k_capacity *k_hidden_dim)> + value = {}; +}; + +struct outcome_counts { + int done = 0; + int error = 0; +}; + +void count_done(void *object, const attention::events::execute_done &) { + ++static_cast(object)->done; +} + +void count_error(void *object, const attention::events::execute_error &) { + ++static_cast(object)->error; +} + +std::array(k_hidden_dim)> +compute_legacy_attention(const attention::event::head_range_request &request) { + std::array(k_hidden_dim)> output = {}; + std::array(k_head_dim)> q_bf16 = {}; + std::array(k_capacity)> scores = {}; + std::array(k_capacity)> weights = {}; + const float scale = 1.0f / std::sqrt(static_cast(request.head_dim)); + + for (int32_t head = request.head_begin; head < request.head_end; ++head) { + const int32_t head_offset = head * request.head_dim; + const float *q_head = request.query.data() + head_offset; + for (int32_t dim = 0; dim < request.head_dim; ++dim) { + q_bf16[static_cast(dim)] = + emel::kernel::detail::fp32_to_bf16(q_head[dim]); + } + scores.fill(-std::numeric_limits::infinity()); + for (int32_t position = 0; position < request.valid_positions; ++position) { + const int32_t unwrapped = request.physical_begin + position; + const int32_t physical = + unwrapped - + static_cast(unwrapped >= request.position_capacity) * + request.position_capacity; + const std::size_t cache_begin = + request.layer_offset + + static_cast(physical * request.hidden_dim + head_offset); + scores[static_cast(physical)] = + emel::kernel::detail::vec_dot_bf16_ggml( + request.head_dim, request.key_cache.data() + cache_begin, + q_bf16.data()) * + scale; + } + emel::kernel::detail::soft_max_row_ggml(request.position_capacity, + scores.data()); + for (int32_t physical = 0; physical < request.position_capacity; + ++physical) { + weights[static_cast(physical)] = + emel::kernel::detail::fp32_to_bf16( + scores[static_cast(physical)]); + } + for (int32_t dim = 0; dim < request.head_dim; ++dim) { + double sum = 0.0; + for (int32_t position = 0; position < request.valid_positions; + ++position) { + const int32_t unwrapped = request.physical_begin + position; + const int32_t physical = + unwrapped - + static_cast(unwrapped >= request.position_capacity) * + request.position_capacity; + const std::size_t value_index = + request.layer_offset + + static_cast(physical * request.hidden_dim + + head_offset + dim); + sum += static_cast( + emel::kernel::detail::bf16_to_fp32( + request.value_cache[value_index]) * + emel::kernel::detail::bf16_to_fp32( + weights[static_cast(physical)])); + } + output[static_cast(head_offset + dim)] = + static_cast(sum); + } + } + return output; +} + +} // namespace + +TEST_CASE( + "kernel attention head actors preserve exact contiguous head output") { + attention_fixture fixture{}; + auto serial_actor = std::make_unique(); + auto lane_actors = std::make_unique>(); + std::array(k_hidden_dim)> serial = {}; + std::array(k_hidden_dim)> partitioned = {}; + + auto serial_request = fixture.request(0, k_heads, std::span{serial}); + attention::event::dispatch_result serial_result{}; + REQUIRE(serial_actor->process_event( + attention::event::execute{serial_request, serial_result})); + REQUIRE(serial_result.accepted); + + for (int32_t head = 0; head < k_heads; ++head) { + const std::size_t begin = static_cast(head * k_head_dim); + auto request = + fixture.request(head, head + 1, + std::span{partitioned.data() + begin, + static_cast(k_head_dim)}); + attention::event::dispatch_result result{}; + REQUIRE((*lane_actors)[static_cast(head)].process_event( + attention::event::execute{request, result})); + REQUIRE(result.accepted); + } + + CHECK(std::equal(serial.begin(), serial.end(), partitioned.begin())); +} + +TEST_CASE("kernel attention value traversal preserves legacy output bits") { + attention_fixture fixture{}; + auto actor = std::make_unique(); + std::array(k_hidden_dim)> output = {}; + auto request = fixture.request(0, k_heads, std::span{output}); + const auto legacy = compute_legacy_attention(request); + attention::event::dispatch_result result{}; + + REQUIRE(actor->process_event(attention::event::execute{request, result})); + REQUIRE(result.accepted); + CHECK(std::equal(output.begin(), output.end(), legacy.begin())); +} + +TEST_CASE("kernel attention dispatch reuses actor scratch without allocation") { + attention_fixture fixture{}; + auto actor = std::make_unique(); + std::array(k_hidden_dim)> output = {}; + auto request = fixture.request(0, k_heads, std::span{output}); + attention::event::dispatch_result result{}; + + std::size_t allocations = 0u; + { + emel::test::allocation::allocation_scope allocation_scope{}; + REQUIRE(actor->process_event(attention::event::execute{request, result})); + allocations = allocation_scope.allocations(); + } + CHECK(result.accepted); + CHECK(allocations == 0u); +} + +TEST_CASE("kernel attention publishes explicit done and error outcomes") { + attention_fixture fixture{}; + auto actor = std::make_unique(); + std::array(k_hidden_dim)> output = {}; + auto request = fixture.request(0, k_heads, std::span{output}); + attention::event::dispatch_result result{}; + outcome_counts counts{}; + attention::event::execute valid{request, result}; + valid.on_done = {&counts, count_done}; + valid.on_error = {&counts, count_error}; + + REQUIRE(actor->process_event(valid)); + CHECK(counts.done == 1); + CHECK(counts.error == 0); + + request.output = request.output.first(1u); + attention::event::execute invalid{request, result}; + invalid.on_done = {&counts, count_done}; + invalid.on_error = {&counts, count_error}; + CHECK_FALSE(actor->process_event(invalid)); + CHECK(counts.done == 1); + CHECK(counts.error == 1); +} + +TEST_CASE("kernel attention rejects an undersized disjoint output span") { + attention_fixture fixture{}; + auto actor = std::make_unique(); + std::array output = {}; + auto request = fixture.request(0, 2, std::span{output}); + attention::event::dispatch_result result{}; + + CHECK_FALSE(actor->process_event(attention::event::execute{request, result})); + CHECK_FALSE(result.accepted); +} + +TEST_CASE("kernel attention rejects invalid public request shapes and spans") { + using request_type = attention::event::head_range_request; + using mutate_type = void (*)(request_type &); + struct invalid_case { + const char *name; + mutate_type mutate; + }; + const std::array cases{ + invalid_case{"nonpositive hidden dim", + [](request_type &request) { request.hidden_dim = 0; }}, + invalid_case{"nonpositive head dim", + [](request_type &request) { request.head_dim = 0; }}, + invalid_case{"negative head begin", + [](request_type &request) { request.head_begin = -1; }}, + invalid_case{ + "empty head range", + [](request_type &request) { request.head_end = request.head_begin; }}, + invalid_case{ + "nonpositive capacity", + [](request_type &request) { request.position_capacity = 0; }}, + invalid_case{"negative physical begin", + [](request_type &request) { request.physical_begin = -1; }}, + invalid_case{"physical begin at capacity", + [](request_type &request) { + request.physical_begin = request.position_capacity; + }}, + invalid_case{"nonpositive valid positions", + [](request_type &request) { request.valid_positions = 0; }}, + invalid_case{"valid positions exceed capacity", + [](request_type &request) { + request.valid_positions = request.position_capacity + 1; + }}, + invalid_case{"layer offset overflows cache requirement", + [](request_type &request) { + request.layer_offset = + std::numeric_limits::max(); + }}, + invalid_case{ + "head end exceeds hidden heads", + [](request_type &request) { request.head_end = k_heads + 1; }}, + invalid_case{"head dim exceeds scratch capacity", + [](request_type &request) { + request.head_dim = attention::action::k_max_head_dim + 1; + request.hidden_dim = request.head_dim; + request.head_end = 1; + }}, + invalid_case{"context exceeds scratch capacity", + [](request_type &request) { + request.position_capacity = + attention::action::k_max_context + 1; + request.physical_begin = 0; + request.valid_positions = 1; + }}, + invalid_case{"query span is undersized", + [](request_type &request) { + request.query = + request.query.first(request.query.size() - 1u); + }}, + invalid_case{"key span is undersized", + [](request_type &request) { + request.key_cache = + request.key_cache.first(request.key_cache.size() - 1u); + }}, + invalid_case{"value span is undersized", + [](request_type &request) { + request.value_cache = request.value_cache.first( + request.value_cache.size() - 1u); + }}, + invalid_case{"output span is undersized", + [](request_type &request) { + request.output = request.output.first(1u); + }}, + }; + + for (const auto &test_case : cases) { + CAPTURE(test_case.name); + attention_fixture fixture{}; + auto actor = std::make_unique(); + std::array(k_hidden_dim)> output = {}; + auto request = fixture.request(0, k_heads, std::span{output}); + test_case.mutate(request); + attention::event::dispatch_result result{.accepted = true}; + CHECK_FALSE( + actor->process_event(attention::event::execute{request, result})); + CHECK_FALSE(result.accepted); + } +} + +TEST_CASE("kernel attention rejects unsafe public span storage") { + attention_fixture fixture{}; + auto actor = std::make_unique(); + std::array(k_hidden_dim)> output = {}; + auto request = fixture.request(0, k_heads, std::span{output}); + attention::event::dispatch_result result{.accepted = true}; + + SUBCASE("sized null query") { + request.query = std::span{static_cast(nullptr), + fixture.query.size()}; + } + SUBCASE("misaligned output") { + alignas(float) std::array storage = {}; + request.output = std::span{ + reinterpret_cast(storage.data() + 1u), output.size()}; + } + SUBCASE("output aliases query") { + request.output = std::span{const_cast(request.query.data()), + output.size()}; + } + SUBCASE("output aliases key cache") { + request.output = std::span{ + reinterpret_cast(fixture.key.data()), output.size()}; + } + SUBCASE("output aliases value cache") { + request.output = std::span{ + reinterpret_cast(fixture.value.data()), output.size()}; + } + + CHECK_FALSE(actor->process_event(attention::event::execute{request, result})); + CHECK_FALSE(result.accepted); +} diff --git a/tests/kernel/f32_matvec_tests.cpp b/tests/kernel/f32_matvec_tests.cpp new file mode 100644 index 00000000..2851e204 --- /dev/null +++ b/tests/kernel/f32_matvec_tests.cpp @@ -0,0 +1,384 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../allocation_tracker.hpp" +#include "emel/kernel/f32_matvec/sm.hpp" + +namespace { + +namespace f32_matvec = emel::kernel::f32_matvec; + +struct unknown_with_result { + f32_matvec::event::dispatch_result &result; +}; + +struct unknown_without_result {}; + +struct outcome_counts { + int done = 0; + int error = 0; + const f32_matvec::event::prepare_f32 *done_request = nullptr; + const f32_matvec::event::prepare_f32 *error_request = nullptr; +}; + +void count_done(void *object, const f32_matvec::events::dispatch_done< + f32_matvec::event::prepare_f32> &outcome) { + auto &counts = *static_cast(object); + ++counts.done; + counts.done_request = &outcome.request; +} + +void count_error(void *object, const f32_matvec::events::dispatch_error< + f32_matvec::event::prepare_f32> &outcome) { + auto &counts = *static_cast(object); + ++counts.error; + counts.error_request = &outcome.request; +} + +template +void check_exact_case(const std::array &weights, + const std::array &input) { + REQUIRE(std::fesetround(FE_TONEAREST) == 0); + REQUIRE(std::fegetround() == FE_TONEAREST); + auto actor = std::make_unique(); + std::array packed{}; + std::array reference{}; + std::array exact{}; + + const f32_matvec::event::prepare_f32_request prepare_request{ + .source = std::span{weights}, + .destination = std::span{packed}, + .inner = inner, + .rows = rows, + }; + f32_matvec::event::dispatch_result prepare_result{}; + REQUIRE(actor->process_event( + f32_matvec::event::prepare_f32{prepare_request, prepare_result})); + + const f32_matvec::event::execute_request reference_request{ + .weights = std::span{weights}, + .input = std::span{input}, + .output = std::span{reference}, + .inner = inner, + .rows = rows, + }; + f32_matvec::event::dispatch_result reference_result{}; + REQUIRE(actor->process_event(f32_matvec::event::execute_reference{ + reference_request, reference_result})); + + const f32_matvec::event::execute_request exact_request{ + .weights = std::span{packed}, + .input = std::span{input}, + .output = std::span{exact}, + .inner = inner, + .rows = rows, + }; + f32_matvec::event::dispatch_result exact_result{}; +#if defined(__aarch64__) || defined(_M_ARM64) + REQUIRE(actor->process_event( + f32_matvec::event::execute_exact_x4{exact_request, exact_result})); + for (std::size_t row = 0u; row < rows; ++row) { + if (std::isnan(reference[row])) { + CHECK(std::isnan(exact[row])); + } else { + CHECK(std::bit_cast(exact[row]) == + std::bit_cast(reference[row])); + } + } +#else + CHECK_FALSE(actor->process_event( + f32_matvec::event::execute_exact_x4{exact_request, exact_result})); + CHECK_FALSE(exact_result.accepted); +#endif +} + +} // namespace + +TEST_CASE("f32 matvec x4 preserves independent row arithmetic and tails") { + constexpr std::size_t inner = 13u; + constexpr std::size_t rows = 7u; + std::array weights{}; + std::array input{}; + for (std::size_t index = 0u; index < weights.size(); ++index) { + weights[index] = + static_cast(static_cast(index % 29u) - 14) / 32.0f; + } + for (std::size_t index = 0u; index < input.size(); ++index) { + input[index] = static_cast(static_cast(index) - 6) / 16.0f; + } + check_exact_case(weights, input); +} + +TEST_CASE("f32 matvec packs f16 rows without changing values") { + constexpr std::size_t inner = 3u; + constexpr std::size_t rows = 5u; + std::array source{}; + for (std::size_t index = 0u; index < source.size(); ++index) { + source[index] = emel::kernel::detail::quant::fp32_to_fp16( + static_cast(static_cast(index) - 7) / 8.0f); + } + std::array packed{}; + auto actor = std::make_unique(); + const f32_matvec::event::prepare_f16_request request{ + .source = std::span{source}, + .destination = std::span{packed}, + .inner = inner, + .rows = rows, + }; + f32_matvec::event::dispatch_result result{}; + REQUIRE( + actor->process_event(f32_matvec::event::prepare_f16{request, result})); + REQUIRE(result.accepted); + for (std::size_t column = 0u; column < inner; ++column) { + for (std::size_t row = 0u; row < 4u; ++row) { + CHECK(packed[column * 4u + row] == + emel::kernel::detail::quant::fp16_to_fp32( + source[row * inner + column])); + } + } + for (std::size_t column = 0u; column < inner; ++column) { + CHECK( + packed[4u * inner + column] == + emel::kernel::detail::quant::fp16_to_fp32(source[4u * inner + column])); + } +} + +TEST_CASE("f32 matvec publishes explicit done and error outcomes") { + std::array source{1.0f, 2.0f, 3.0f, 4.0f}; + std::array destination{}; + auto actor = std::make_unique(); + outcome_counts counts{}; + f32_matvec::event::dispatch_result result{}; + const f32_matvec::event::prepare_f32_request request{ + .source = source, + .destination = destination, + .inner = 2u, + .rows = 2u, + }; + f32_matvec::event::prepare_f32 valid{request, result}; + valid.on_done = {&counts, count_done}; + valid.on_error = {&counts, count_error}; + + REQUIRE(actor->process_event(valid)); + CHECK(counts.done == 1); + CHECK(counts.error == 0); + CHECK(counts.done_request == &valid); + + const f32_matvec::event::prepare_f32_request bad_request{ + .source = source, + .destination = std::span{destination}.first(1u), + .inner = 2u, + .rows = 2u, + }; + f32_matvec::event::prepare_f32 invalid{bad_request, result}; + invalid.on_done = {&counts, count_done}; + invalid.on_error = {&counts, count_error}; + CHECK_FALSE(actor->process_event(invalid)); + CHECK(counts.done == 1); + CHECK(counts.error == 1); + CHECK(counts.error_request == &invalid); +} + +TEST_CASE("f32 matvec x4 preserves NaN classification") { + constexpr std::size_t inner = 1u; + constexpr std::size_t rows = 4u; + const float quiet_nan = std::numeric_limits::quiet_NaN(); + const std::array weights{1.0f, -1.0f, 2.0f, -2.0f}; + const std::array input{quiet_nan}; + check_exact_case(weights, input); +} + +TEST_CASE("f32 matvec x4 preserves subnormal results") { + constexpr std::size_t inner = 1u; + constexpr std::size_t rows = 4u; + const float denorm = std::numeric_limits::denorm_min(); + const std::array weights{denorm, -denorm, 2.0f * denorm, + -2.0f * denorm}; + const std::array input{1.0f}; + check_exact_case(weights, input); + CHECK(std::fpclassify(weights[0] * input[0]) == FP_SUBNORMAL); +} + +TEST_CASE("f32 matvec x4 preserves non-NaN infinity results") { + constexpr std::size_t inner = 1u; + constexpr std::size_t rows = 4u; + const float infinity = std::numeric_limits::infinity(); + const std::array weights{infinity, -infinity, infinity, + -infinity}; + const std::array input{1.0f}; + check_exact_case(weights, input); + CHECK(std::isinf(weights[0] * input[0])); + CHECK_FALSE(std::isnan(weights[0] * input[0])); +} + +TEST_CASE("f32 matvec x4 preserves the reference signed zero result") { + constexpr std::size_t inner = 1u; + constexpr std::size_t rows = 4u; + const std::array weights{-0.0f, 0.0f, -0.0f, 0.0f}; + const std::array input{1.0f}; + check_exact_case(weights, input); + + double sum = 0.0; + sum += static_cast(weights[0] * input[0]); + const float reference = static_cast(sum); + CHECK(reference == 0.0f); + CHECK_FALSE(std::signbit(reference)); +} + +TEST_CASE("f32 matvec prepare rejection never writes caller storage") { + auto actor = std::make_unique(); + std::array storage{}; + storage.fill(7.0f); + const f32_matvec::event::prepare_f32_request undersized{ + .source = std::span{storage}, + .destination = std::span{storage}.first(7u), + .inner = 2u, + .rows = 4u, + }; + f32_matvec::event::dispatch_result result{}; + CHECK_FALSE( + actor->process_event(f32_matvec::event::prepare_f32{undersized, result})); + CHECK_FALSE(result.accepted); + for (const float value : storage) { + CHECK(value == 7.0f); + } + + const f32_matvec::event::prepare_f32_request overlapping{ + .source = std::span{storage}, + .destination = std::span{storage}, + .inner = 2u, + .rows = 4u, + }; + CHECK_FALSE(actor->process_event( + f32_matvec::event::prepare_f32{overlapping, result})); + for (const float value : storage) { + CHECK(value == 7.0f); + } + + const f32_matvec::event::prepare_f32_request overflow{ + .source = std::span{storage}, + .destination = std::span{storage}, + .inner = std::numeric_limits::max(), + .rows = 2u, + }; + CHECK_FALSE( + actor->process_event(f32_matvec::event::prepare_f32{overflow, result})); + + alignas(float) std::array bytes{}; + auto *misaligned = reinterpret_cast(bytes.data() + 1u); + const f32_matvec::event::prepare_f32_request bad_alignment{ + .source = std::span{misaligned, 8u}, + .destination = std::span{storage}, + .inner = 2u, + .rows = 4u, + }; + CHECK_FALSE(actor->process_event( + f32_matvec::event::prepare_f32{bad_alignment, result})); +} + +TEST_CASE("f32 matvec rejects aliased execute spans without writes") { + auto actor = std::make_unique(); + std::array weights{}; + std::array input_and_output{}; + for (std::size_t index = 0u; index < weights.size(); ++index) { + weights[index] = static_cast(index + 1u); + } + input_and_output.fill(1.0f); + const auto before = input_and_output; + const f32_matvec::event::execute_request request{ + .weights = std::span{weights}, + .input = std::span{input_and_output}, + .output = std::span{input_and_output}, + .inner = 8u, + .rows = 8u, + }; + f32_matvec::event::dispatch_result result{.accepted = true}; + CHECK_FALSE(actor->process_event( + f32_matvec::event::execute_exact_x4{request, result})); + CHECK_FALSE(result.accepted); + CHECK(input_and_output == before); + + std::array weights_and_output{}; + for (std::size_t index = 0u; index < weights_and_output.size(); ++index) { + weights_and_output[index] = static_cast(index + 1u); + } + const auto weights_before = weights_and_output; + const f32_matvec::event::execute_request weight_alias_request{ + .weights = std::span{weights_and_output}, + .input = std::span{input_and_output}, + .output = std::span{weights_and_output}.first(8u), + .inner = 8u, + .rows = 8u, + }; + result.accepted = true; + CHECK_FALSE(actor->process_event( + f32_matvec::event::execute_reference{weight_alias_request, result})); + CHECK_FALSE(result.accepted); + CHECK(weights_and_output == weights_before); +} + +TEST_CASE("f32 matvec rejects unexpected events and stale results") { + auto actor = std::make_unique(); + f32_matvec::event::dispatch_result result{.accepted = true}; + CHECK_FALSE(actor->process_event(unknown_with_result{result})); + CHECK_FALSE(result.accepted); + CHECK_FALSE(actor->process_event(unknown_without_result{})); +} + +TEST_CASE("f32 matvec dispatch is allocation-free and reports deltas") { + auto actor = std::make_unique(); + const std::array weights{1.0f, 2.0f, 3.0f, 4.0f, + 5.0f, 6.0f, 7.0f, 8.0f}; + const std::array input{0.5f, -0.25f}; + std::array packed{}; + std::array output{}; + f32_matvec::event::dispatch_result result{}; + const f32_matvec::event::prepare_f32_request prepare_request{ + .source = std::span{weights}, + .destination = std::span{packed}, + .inner = 2u, + .rows = 4u, + }; + const f32_matvec::event::execute_request execute_request{ + .weights = std::span{packed}, + .input = std::span{input}, + .output = std::span{output}, + .inner = 2u, + .rows = 4u, + }; + + std::size_t allocations = 0u; + { + emel::test::allocation::allocation_scope allocation_scope{}; + REQUIRE(actor->process_event( + f32_matvec::event::prepare_f32{prepare_request, result})); +#if defined(__aarch64__) || defined(_M_ARM64) + for (std::size_t iteration = 0u; iteration < 64u; ++iteration) { + REQUIRE(actor->process_event( + f32_matvec::event::execute_exact_x4{execute_request, result})); + } +#endif + allocations = allocation_scope.allocations(); + } + CHECK(allocations == 0u); + + f32_matvec::event::diagnostics diagnostics{}; + REQUIRE(actor->process_event( + f32_matvec::event::capture_diagnostics{diagnostics, result})); + CHECK(diagnostics.prepare_calls == 1u); + CHECK(diagnostics.prepared_floats == 8u); +#if defined(__aarch64__) || defined(_M_ARM64) + CHECK(diagnostics.exact_x4_calls == 64u); +#else + CHECK(diagnostics.exact_x4_calls == 0u); +#endif +} diff --git a/tests/kernel/matmul_tests.cpp b/tests/kernel/matmul_tests.cpp new file mode 100644 index 00000000..6ce0efdb --- /dev/null +++ b/tests/kernel/matmul_tests.cpp @@ -0,0 +1,779 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// View-sliced parallel matmul behavior: pack-group-aligned slice arithmetic +// and bit-exact parity between serial and parallel lane dispatch. Route-level +// proof against maintained model fixtures lives in the generator lifecycle +// tests; this file covers the slicing and fork/join detail surface. + +#include "../allocation_tracker.hpp" +#include "../text/generator/generator_test_policies.hpp" +#include "emel/kernel/matmul/actions.hpp" +#include "emel/kernel/matmul/guards.hpp" +#include "emel/text/generator/actions.hpp" +#include "emel/text/generator/detail.hpp" +#include "emel/text/generator/guards.hpp" + +namespace { + +namespace gen_detail = emel::text::generator::detail; +namespace matmul = emel::kernel::matmul; +using emel::kernel::event::dtype; +using gen_detail::matmul_lane_mode; +using matmul::detail::matmul_row_slice; + +inline constexpr size_t k_test_matmul_lanes = 8u; + +struct outcome_counts { + int done = 0; + int error = 0; + const matmul::event::execute_parallel *done_request = nullptr; + const matmul::event::execute_parallel *error_request = nullptr; +}; + +void count_done(void *object, const matmul::events::execute_done< + matmul::event::execute_parallel> &outcome) { + auto &counts = *static_cast(object); + ++counts.done; + counts.done_request = &outcome.request; +} + +void count_error(void *object, const matmul::events::execute_error< + matmul::event::execute_parallel> &outcome) { + auto &counts = *static_cast(object); + ++counts.error; + counts.error_request = &outcome.request; +} + +emel::model::data::tensor_record make_tensor_record(void *data, + const int32_t type, + const int32_t cols, + const int32_t rows) { + emel::model::data::tensor_record tensor = {}; + tensor.data = data; + tensor.type = type; + tensor.n_dims = 2; + tensor.dims[0] = static_cast(cols); + tensor.dims[1] = static_cast(rows); + tensor.data_size = + gen_detail::row_storage_bytes(tensor, cols) * static_cast(rows); + return tensor; +} + +template +std::array +make_production_slices(const uint64_t rows, std::index_sequence) { + return { + {matmul::action::compute_fixed_row_slice( + rows)...}}; +} + +template +void check_slices_tile_rows(const uint64_t rows) { + CAPTURE(rows); + constexpr uint64_t group_rows_value = group_rows; + const auto slices = make_production_slices( + rows, std::make_index_sequence{}); + uint64_t expected_begin = 0u; + for (size_t lane = 0; lane < lane_count; ++lane) { + CHECK(static_cast(slices[lane].row_begin) == expected_begin); + CHECK(slices[lane].row_count > 0); + CHECK(static_cast(slices[lane].row_begin) % group_rows_value == + 0u); + expected_begin += static_cast(slices[lane].row_count); + } + CHECK(expected_begin == rows); +} + +TEST_CASE("parallel matmul slice group rows match pack formats") { + CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::f32)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::f32)); + CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::f16)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::f16)); + CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q8_0)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q8_0)); + CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q4_k)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q4_k)); + CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q6_k)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q6_k)); + + CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q8_0_x4_bl4)); + CHECK(matmul::guard::guard_uses_x4_row_groups(dtype::q8_0_x4_bl4)); + CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q8_0_x4_bl8)); + CHECK(matmul::guard::guard_uses_x4_row_groups(dtype::q8_0_x4_bl8)); + + CHECK(matmul::guard::guard_uses_x8_row_groups(dtype::q4_k_x8_bl4)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q4_k_x8_bl4)); + CHECK(matmul::guard::guard_uses_x8_row_groups(dtype::q4_k_x8_bl8)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q4_k_x8_bl8)); + CHECK(matmul::guard::guard_uses_x8_row_groups(dtype::q6_k_x8)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q6_k_x8)); + CHECK(matmul::guard::guard_uses_x8_row_groups(dtype::q6_k_x8_q8_prepared)); + CHECK_FALSE( + matmul::guard::guard_uses_x4_row_groups(dtype::q6_k_x8_q8_prepared)); + CHECK(matmul::guard::guard_uses_x8_row_groups( + dtype::q6_k_x8_q8_argmax_prepared)); + CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups( + dtype::q6_k_x8_q8_argmax_prepared)); +} + +TEST_CASE("parallel matmul slices tile rows contiguously and group aligned") { + check_slices_tile_rows<8u, 1u>(13u); + check_slices_tile_rows<2u, 1u>(3u); + check_slices_tile_rows<4u, 1u>(16u); + check_slices_tile_rows<8u, 1u>(256u); + check_slices_tile_rows<8u, 4u>(64u); + check_slices_tile_rows<4u, 4u>(20u); + check_slices_tile_rows<8u, 8u>(100u); + check_slices_tile_rows<8u, 8u>(64u); + check_slices_tile_rows<8u, 8u>(4096u); +} + +TEST_CASE("parallel matmul ragged tail lands in final slice") { + const auto slices = make_production_slices( + 100u, std::make_index_sequence{}); + const auto &tail = slices.back(); + CHECK(static_cast(tail.row_begin) % 8u == 0u); + CHECK(static_cast(tail.row_begin) + + static_cast(tail.row_count) == + 100u); +} + +TEST_CASE("parallel matmul sliced event offsets views by storage groups") { + std::array src0_storage = {}; + std::array dst_storage = {}; + emel::kernel::event::op_mul_mat ev = {}; + ev.src0.data = src0_storage.data(); + ev.src0.type = dtype::q4_k_x8_bl8; + ev.src0.ne = {256u, 64u, 1u, 1u}; + ev.src0.nb = {1u, 128u, 128u * 8u, 128u * 8u}; + ev.dst.data = dst_storage.data(); + ev.dst.type = dtype::f32; + ev.dst.ne = {1u, 64u, 1u, 1u}; + ev.dst.nb = {sizeof(float), sizeof(float), sizeof(float) * 64u, + sizeof(float) * 64u}; + + const matmul_row_slice slice{16, 24}; + const auto sliced = + matmul::detail::compute_sliced_mul_mat_event(ev, 8u, slice); + CHECK(sliced.src0.data == src0_storage.data() + (16u / 8u) * 128u); + CHECK(sliced.src0.ne[1] == 24u); + CHECK(sliced.src0.nb[1] == ev.src0.nb[1]); + CHECK(sliced.src0.nb[2] == 128u * 3u); + CHECK(sliced.dst.data == + reinterpret_cast(dst_storage.data()) + 16u * sizeof(float)); + CHECK(sliced.dst.ne[1] == 24u); + CHECK(sliced.src1.data == ev.src1.data); +} + +TEST_CASE("parallel matmul storage guard rejects malformed parent requests") { + matmul::lane_pool lanes = {}; + matmul::action::context ctx = {}; + ctx.parallel_matmul_lanes = &lanes; + ctx.active_lanes = 8u; + + emel::kernel::event::op_mul_mat request = {}; + request.src0.ne[1] = 8u; + matmul::event::dispatch_result result = {}; + bool accepted = true; + const matmul::event::execute_parallel run{request, result, accepted}; + + CHECK_FALSE(matmul::guard::guard_parallel_storage_ready(run, ctx)); + const auto policy = matmul::make_execution_policy( + lanes, emel::kernel::detect_host_kind(), 8u); + matmul::sm actor{policy}; + REQUIRE(actor.process_event(run)); + CHECK_FALSE(accepted); + CHECK(result.lane_count == 0u); + CHECK(result.submitted_worker_lanes == 0u); + + std::array storage = {}; + request.src0.data = storage.data(); + request.src1.data = storage.data(); + request.dst.data = storage.data(); + request.src0.type = dtype::f32; + request.src1.type = dtype::f32; + request.dst.type = dtype::f32; + request.src0.ne = { + 1u, static_cast(std::numeric_limits::max()) + 1u, 1u, + 1u}; + request.src1.ne = {1u, 1u, 1u, 1u}; + request.dst.ne = {1u, request.src0.ne[1], 1u, 1u}; + request.src0.nb = {sizeof(float), sizeof(float), sizeof(float), + sizeof(float)}; + request.src1.nb = {sizeof(float), sizeof(float), sizeof(float), + sizeof(float)}; + request.dst.nb = {sizeof(float), sizeof(float), sizeof(float), sizeof(float)}; + CHECK_FALSE(matmul::guard::guard_parallel_storage_ready(run, ctx)); + result = {}; + accepted = true; + REQUIRE(actor.process_event(run)); + CHECK_FALSE(accepted); + CHECK(result.lane_count == 0u); + CHECK(result.submitted_worker_lanes == 0u); +} + +TEST_CASE("matmul actor guards and actions expose explicit rejection paths") { + matmul::action::context ctx = {}; + emel::kernel::event::op_mul_mat request = {}; + request.src0.ne[1] = 0u; + matmul::event::dispatch_result serial_result = {}; + matmul::event::dispatch_result parallel_result = {}; + bool serial_accepted = true; + bool parallel_accepted = true; + const matmul::event::execute_serial serial_event{request, serial_result, + serial_accepted}; + const matmul::event::execute_parallel parallel_event{request, parallel_result, + parallel_accepted}; + + CHECK(matmul::guard::guard_parallel_unavailable{}(parallel_event, ctx)); + CHECK_FALSE( + matmul::guard::guard_parallel_ready<4u, 1u>{}(parallel_event, ctx)); + request.src0.ne[1] = 4u; + CHECK(matmul::guard::guard_parallel_unavailable{}(parallel_event, ctx)); + + matmul::lane_pool parallel_lanes = {}; + ctx.parallel_matmul_lanes = ¶llel_lanes; + ctx.active_lanes = 8u; + CHECK(matmul::guard::guard_parallel_request_invalid{}(parallel_event, ctx)); + CHECK_FALSE( + matmul::guard::guard_parallel_ready<4u, 1u>{}(parallel_event, ctx)); + CHECK_FALSE(matmul::guard::guard_parallel_unavailable{}(parallel_event, ctx)); + + std::array weights = {}; + std::array input = {}; + std::array output = {}; + request.src0 = gen_detail::make_src_view(weights.data(), 1u, 4u); + request.src1 = gen_detail::make_src_view(input.data(), 1u, 1u); + request.dst = gen_detail::make_dst_view(output.data(), 1u, 4u); + CHECK_FALSE( + matmul::guard::guard_parallel_request_invalid{}(parallel_event, ctx)); + CHECK(matmul::guard::guard_parallel_ready<4u, 1u>{}(parallel_event, ctx)); + + matmul::action::context over_cap_ctx = {}; + over_cap_ctx.parallel_matmul_lanes = ¶llel_lanes; + over_cap_ctx.active_lanes = 3u; + CHECK_FALSE(matmul::guard::guard_parallel_ready<4u, 1u>{}(parallel_event, + over_cap_ctx)); + CHECK(matmul::guard::guard_parallel_unavailable{}(parallel_event, + over_cap_ctx)); + + serial_result.all_lanes_accepted = false; + CHECK(matmul::guard::guard_serial_rejected{}(serial_event, ctx)); + CHECK_FALSE(matmul::guard::guard_serial_accepted{}(serial_event, ctx)); + matmul::action::effect_reject_serial_execution(serial_event, ctx); + CHECK_FALSE(serial_accepted); + + parallel_result.all_submitted = false; + CHECK(matmul::guard::guard_parallel_submission_failed{}(parallel_event, ctx)); + + parallel_result.all_submitted = true; + parallel_result.lane_count = 0u; + parallel_result.all_lanes_accepted = false; + CHECK(matmul::guard::guard_parallel_lane_rejected{}(parallel_event, ctx)); + CHECK_FALSE( + matmul::guard::guard_parallel_all_lanes_accepted{}(parallel_event, ctx)); + + parallel_result.all_lanes_accepted = true; + CHECK( + matmul::guard::guard_parallel_all_lanes_accepted{}(parallel_event, ctx)); + matmul::action::effect_reject_parallel_execution(parallel_event, ctx); + CHECK_FALSE(parallel_accepted); + + struct event_with_accepted { + bool &accepted; + }; + bool unexpected_accepted = true; + const event_with_accepted unexpected{unexpected_accepted}; + matmul::action::effect_on_unexpected(unexpected, ctx); + CHECK_FALSE(unexpected_accepted); + + struct event_without_accepted {}; + matmul::action::effect_on_unexpected(event_without_accepted{}, ctx); +} + +TEST_CASE("parallel matmul guards select fixed lane and packed row groups") { + matmul::lane_pool lanes = {}; + matmul::action::context ctx = {}; + ctx.parallel_matmul_lanes = &lanes; + ctx.active_lanes = 8u; + emel::kernel::event::op_mul_mat request = {}; + matmul::event::dispatch_result result = {}; + bool accepted = false; + const matmul::event::execute_parallel run{request, result, accepted}; + + request.src0.type = dtype::f32; + request.src0.ne[1] = 8u; + CHECK(matmul::guard::guard_lane_count_selected<8u>(run, ctx, 1u)); + request.src0.ne[1] = 7u; + CHECK_FALSE(matmul::guard::guard_lane_count_selected<8u>(run, ctx, 1u)); + + ctx.active_lanes = 4u; + request.src0.ne[1] = 8u; + CHECK(matmul::guard::guard_lane_count_selected<4u>(run, ctx, 1u)); + ctx.active_lanes = 8u; + request.src0.ne[1] = 4u; + CHECK(matmul::guard::guard_lane_count_selected<4u>(run, ctx, 1u)); + request.src0.ne[1] = 8u; + CHECK_FALSE(matmul::guard::guard_lane_count_selected<4u>(run, ctx, 1u)); + + ctx.active_lanes = 2u; + request.src0.ne[1] = 4u; + CHECK(matmul::guard::guard_lane_count_selected<2u>(run, ctx, 1u)); + ctx.active_lanes = 4u; + request.src0.ne[1] = 2u; + CHECK(matmul::guard::guard_lane_count_selected<2u>(run, ctx, 1u)); + request.src0.ne[1] = 4u; + CHECK_FALSE(matmul::guard::guard_lane_count_selected<2u>(run, ctx, 1u)); + + std::array src0_storage = {}; + std::array src1_storage = {}; + std::array dst_storage = {}; + request.src0.data = src0_storage.data(); + request.src0.ne[0] = 1u; + request.src0.ne[2] = 1u; + request.src0.ne[3] = 1u; + request.src0.nb = {1u, 1u, 64u, 64u}; + request.src1 = gen_detail::make_src_view(src1_storage.data(), 1u, 1u); + request.dst = gen_detail::make_dst_view(dst_storage.data(), 1u, 1u); + const auto set_request_rows = [&request](const dtype type, + const uint64_t rows) noexcept { + request.src0.type = type; + request.src0.ne[1] = rows; + request.dst.ne[1] = rows; + }; + ctx.active_lanes = 8u; + set_request_rows(dtype::f32, 1u); + CHECK(matmul::guard::guard_parallel_no_lane_count{}(run, ctx)); + set_request_rows(dtype::q8_0_x4_bl8, 4u); + CHECK(matmul::guard::guard_parallel_no_lane_count{}(run, ctx)); + set_request_rows(dtype::q4_k_x8_bl8, 8u); + CHECK(matmul::guard::guard_parallel_no_lane_count{}(run, ctx)); + set_request_rows(dtype::q4_k_x8_bl8, 16u); + CHECK_FALSE(matmul::guard::guard_parallel_no_lane_count{}(run, ctx)); + + CHECK(matmul::guard::guard_group_rows_selected<1u>(dtype::f32)); + CHECK_FALSE(matmul::guard::guard_group_rows_selected<1u>(dtype::q8_0_x4_bl8)); + CHECK(matmul::guard::guard_group_rows_selected< + emel::kernel::detail::quant::Q8_0_X4_ROWS>(dtype::q8_0_x4_bl8)); + CHECK(matmul::guard::guard_group_rows_selected< + emel::kernel::detail::quant::Q4_K_X8_ROWS>(dtype::q4_k_x8_bl8)); +} + +void exercise_fixed_packed_format(const dtype type, const uint64_t rows, + const size_t expected_lanes, + const size_t active_lanes) { + matmul::lane_pool pool = {}; + const auto policy = matmul::make_execution_policy( + pool, emel::kernel::detect_host_kind(), active_lanes); + matmul::sm actor{policy}; + std::vector src0_storage(static_cast(rows), 0u); + std::array src1_storage = {}; + std::vector dst_storage(static_cast(rows), 0.0f); + emel::kernel::event::op_mul_mat request = {}; + request.src0.data = src0_storage.data(); + request.src0.type = type; + request.src0.ne = {1u, rows, 1u, 1u}; + request.src0.nb = {1u, 1u, rows, rows}; + request.src1 = gen_detail::make_src_view(src1_storage.data(), 1u, 1u); + request.dst = gen_detail::make_dst_view(dst_storage.data(), 1u, rows); + matmul::event::dispatch_result result = {}; + bool accepted = true; + const matmul::event::execute_parallel run{request, result, accepted}; + REQUIRE(actor.process_event(run)); + CHECK(result.lane_count == expected_lanes); + CHECK_FALSE(accepted); +} + +TEST_CASE("parallel matmul dispatches explicit packed format lane bodies") { + exercise_fixed_packed_format(dtype::q8_0_x4_bl8, 8u, 2u, 2u); + exercise_fixed_packed_format(dtype::q8_0_x4_bl8, 16u, 4u, 4u); + exercise_fixed_packed_format(dtype::q8_0_x4_bl8, 32u, 8u, 8u); + exercise_fixed_packed_format(dtype::q4_k_x8_bl8, 16u, 2u, 2u); + exercise_fixed_packed_format(dtype::q4_k_x8_bl8, 32u, 4u, 4u); + exercise_fixed_packed_format(dtype::q4_k_x8_bl8, 64u, 8u, 8u); +} + +TEST_CASE("parallel matmul uses the concrete canonical lane pool") { + matmul::lane_pool pool = {}; + CHECK_FALSE(pool.is_current_thread_worker()); + const auto policy = matmul::make_auto_execution_policy(pool); + CHECK(policy.parallel_matmul_lanes == &pool); + CHECK(policy.active_lanes == 8u); +} + +struct parallel_backend_fixture { + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; + emel::kernel::matmul::execution_policy policy = + emel::kernel::matmul::make_auto_execution_policy(parallel_matmul_lanes); + emel::kernel::matmul::sm matmul_actor{policy}; + gen_detail::native_backend backend = {}; + + parallel_backend_fixture() { + backend.kernel_kind = policy.kernel_kind; + backend.routes = emel::text::generator::test::k_generation_route_policy; + backend.kernel.set_kind(backend.kernel_kind); + backend.matmul_actor = &matmul_actor; + backend.matmul_lane_mode = emel::kernel::matmul::lane_mode::parallel; + matmul_actor.process_event( + emel::kernel::matmul::event::configure_kernel_kind{ + backend.kernel_kind}); + } +}; + +void check_fixed_parallel_lane_count(const size_t expected_lanes) { + constexpr int32_t rows = 16; + constexpr int32_t cols = 8; + std::array(rows * cols)> weights = {}; + std::array(cols)> input = {}; + std::array(rows)> output = {}; + std::fill(weights.begin(), weights.end(), 0.25f); + std::fill(input.begin(), input.end(), 0.5f); + + auto record = make_tensor_record(weights.data(), + emel::kernel::detail::dtype_f32, cols, rows); + gen_detail::tensor_matrix matrix{&record, rows, cols}; + emel::kernel::event::op_mul_mat request{ + .src0 = gen_detail::make_src_view(matrix), + .src1 = gen_detail::make_src_view(input.data(), 1u, input.size()), + .dst = gen_detail::make_dst_view(output.data(), 1u, output.size()), + }; + + matmul::lane_pool pool = {}; + const auto policy = matmul::make_execution_policy( + pool, emel::kernel::detect_host_kind(), expected_lanes); + matmul::sm actor{policy}; + matmul::event::dispatch_result result = {}; + bool accepted = false; + const matmul::event::execute_parallel run{request, result, accepted}; + REQUIRE(actor.process_event(run)); + REQUIRE(accepted); + CHECK(result.lane_count == expected_lanes); + CHECK(result.all_submitted); + CHECK(result.submitted_worker_lanes == expected_lanes - 1u); + CHECK(result.drained_worker_lanes == result.submitted_worker_lanes); + CHECK(result.all_lanes_accepted); +} + +TEST_CASE("parallel matmul explicitly dispatches two and four lane bodies") { + check_fixed_parallel_lane_count(2u); + check_fixed_parallel_lane_count(4u); +} + +TEST_CASE( + "parallel matmul partial submission rejection drains accepted workers") { + constexpr int32_t rows = 8; + constexpr int32_t cols = 1; + std::array(rows * cols)> weights = {}; + std::array(cols)> input = {2.0f}; + std::array(rows)> output = {}; + weights.fill(1.0f); + output.fill(-1.0f); + + matmul::lane_pool pool = {}; + matmul::lane_pool::join_group blocker_group{}; + std::atomic blocker_entered = false; + std::atomic release_blocker = false; + REQUIRE(pool.try_submit(blocker_group, [&]() noexcept { + blocker_entered.store(true, std::memory_order_release); + while (!release_blocker.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + })); + for (size_t attempt = 0u; attempt < 100000u; ++attempt) { + if (blocker_entered.load(std::memory_order_acquire)) { + break; + } + std::this_thread::yield(); + } + REQUIRE(blocker_entered.load(std::memory_order_acquire)); + + auto record = make_tensor_record(weights.data(), + emel::kernel::detail::dtype_f32, cols, rows); + gen_detail::tensor_matrix matrix{&record, rows, cols}; + emel::kernel::event::op_mul_mat request{ + .src0 = gen_detail::make_src_view(matrix), + .src1 = gen_detail::make_src_view(input.data(), 1u, input.size()), + .dst = gen_detail::make_dst_view(output.data(), 1u, output.size()), + }; + const auto policy = + matmul::make_execution_policy(pool, emel::kernel::detect_host_kind(), 8u); + matmul::sm actor{policy}; + matmul::event::dispatch_result result = {}; + bool accepted = true; + const matmul::event::execute_parallel run{request, result, accepted}; + + bool dispatched = false; + size_t allocation_count = 0u; + { + emel::test::allocation::allocation_scope allocations{}; + dispatched = actor.process_event(run); + allocation_count = allocations.allocations(); + } + REQUIRE(dispatched); + CHECK_FALSE(accepted); + CHECK_FALSE(result.all_submitted); + CHECK(result.submitted_worker_lanes == 6u); + CHECK(result.drained_worker_lanes == result.submitted_worker_lanes); + CHECK_FALSE(result.all_lanes_accepted); + CHECK(allocation_count == 0u); + for (size_t row = 0u; row < 7u; ++row) { + CHECK(output[row] == doctest::Approx(2.0f)); + } + CHECK(output[7] == doctest::Approx(-1.0f)); + + release_blocker.store(true, std::memory_order_release); + CHECK(blocker_group.wait()); + + output.fill(-1.0f); + result = {}; + accepted = false; + REQUIRE(actor.process_event(run)); + CHECK(accepted); + CHECK(result.all_submitted); + CHECK(result.submitted_worker_lanes == 7u); + CHECK(result.drained_worker_lanes == 7u); + CHECK(result.all_lanes_accepted); + for (const float value : output) { + CHECK(value == doctest::Approx(2.0f)); + } +} + +TEST_CASE( + "parallel matmul capability accepts only supported fixed lane counts") { + matmul::lane_pool two_lane_pool = {}; + auto two_lane_policy = matmul::make_execution_policy( + two_lane_pool, emel::kernel::detect_host_kind(), 2u); + CHECK(two_lane_policy.mode == matmul::lane_mode::parallel); + CHECK(matmul::guard::supports_parallel_execution(two_lane_policy)); + two_lane_policy.active_lanes = 1u; + CHECK_FALSE(matmul::guard::supports_parallel_execution(two_lane_policy)); + + auto three_lane_policy = matmul::make_execution_policy( + two_lane_pool, emel::kernel::detect_host_kind(), 3u); + CHECK(three_lane_policy.mode == matmul::lane_mode::serial); + CHECK_FALSE(matmul::guard::supports_parallel_execution(three_lane_policy)); + three_lane_policy.active_lanes = 2u; + CHECK_FALSE(matmul::guard::supports_parallel_execution(three_lane_policy)); + + auto disengaged_policy = matmul::make_execution_policy( + two_lane_pool, emel::kernel::detect_host_kind(), 2u); + disengaged_policy.parallel_matmul_lanes = nullptr; + CHECK_FALSE(matmul::guard::supports_parallel_execution(disengaged_policy)); +} + +TEST_CASE("parallel matmul f32 gemv matches serial dispatch bit exact") { + constexpr int32_t rows = 61; + constexpr int32_t cols = 32; + std::vector weights(static_cast(rows) * + static_cast(cols)); + for (size_t idx = 0; idx < weights.size(); ++idx) { + weights[idx] = 0.25f * static_cast((idx * 31u + 7u) % 17u) - 2.0f; + } + std::vector input(static_cast(cols)); + for (size_t idx = 0; idx < input.size(); ++idx) { + input[idx] = 0.5f * static_cast((idx * 13u + 3u) % 11u) - 2.5f; + } + + auto record = make_tensor_record(weights.data(), + emel::kernel::detail::dtype_f32, cols, rows); + gen_detail::tensor_matrix matrix{&record, rows, cols}; + + parallel_backend_fixture fixture; + std::vector out_serial(static_cast(rows), -1.0f); + std::vector out_parallel(static_cast(rows), -2.0f); + + emel::kernel::event::op_mul_mat serial_ev{ + .src0 = gen_detail::make_src_view(matrix), + .src1 = gen_detail::make_src_view(input.data(), static_cast(1u), + static_cast(input.size())), + .dst = gen_detail::make_dst_view( + out_serial.data(), static_cast(1u), + static_cast(out_serial.size())), + }; + emel::kernel::event::op_mul_mat parallel_ev = serial_ev; + parallel_ev.dst = + gen_detail::make_dst_view(out_parallel.data(), static_cast(1u), + static_cast(out_parallel.size())); + + CHECK(gen_detail::compute_mul_mat(fixture.backend, + serial_ev)); + CHECK(gen_detail::compute_mul_mat(fixture.backend, + parallel_ev)); + CHECK(std::memcmp(out_serial.data(), out_parallel.data(), + out_serial.size() * sizeof(float)) == 0); + + std::fill(out_parallel.begin(), out_parallel.end(), -2.0f); + matmul::event::dispatch_result result = {}; + bool accepted = false; + outcome_counts outcomes{}; + matmul::event::execute_parallel run{parallel_ev, result, accepted}; + run.on_done = {&outcomes, count_done}; + run.on_error = {&outcomes, count_error}; + bool dispatched = false; + size_t allocation_count = 0u; + { + emel::test::allocation::allocation_scope allocations{}; + dispatched = fixture.matmul_actor.process_event(run); + allocation_count = allocations.allocations(); + } + + CHECK(dispatched); + CHECK(accepted); + CHECK(result.all_submitted); + CHECK(result.submitted_worker_lanes == result.drained_worker_lanes); + CHECK(result.all_lanes_accepted); + CHECK(result.lane_count > 1u); + CHECK(outcomes.done == 1); + CHECK(outcomes.error == 0); + CHECK(outcomes.done_request == &run); + CHECK(allocation_count == 0u); + CHECK(std::memcmp(out_serial.data(), out_parallel.data(), + out_serial.size() * sizeof(float)) == 0); + + parallel_ev.src0.ne[1] = 0u; + matmul::event::execute_parallel rejected{parallel_ev, result, accepted}; + rejected.on_done = {&outcomes, count_done}; + rejected.on_error = {&outcomes, count_error}; + REQUIRE(fixture.matmul_actor.process_event(rejected)); + CHECK_FALSE(accepted); + CHECK(outcomes.done == 1); + CHECK(outcomes.error == 1); + CHECK(outcomes.error_request == &rejected); +} + +TEST_CASE("parallel matmul q8_0 gemv matches serial dispatch bit exact") { + constexpr int32_t rows = 61; + constexpr int32_t cols = 64; + constexpr size_t blocks_per_row = + static_cast(cols) / static_cast(gen_detail::quant::QK8_0); + std::vector weights(static_cast(rows) * + blocks_per_row); + for (size_t block = 0; block < weights.size(); ++block) { + weights[block].d = gen_detail::quant::fp32_to_fp16( + 0.01f + 0.001f * static_cast(block % 7u)); + for (size_t idx = 0; idx < weights[block].qs.size(); ++idx) { + weights[block].qs[idx] = static_cast( + static_cast((block * 37u + idx * 5u) % 255u) - 127); + } + } + std::vector input(static_cast(cols)); + for (size_t idx = 0; idx < input.size(); ++idx) { + input[idx] = 0.125f * static_cast((idx * 7u + 1u) % 19u) - 1.0f; + } + + auto record = make_tensor_record( + weights.data(), emel::kernel::detail::dtype_q8_0, cols, rows); + gen_detail::tensor_matrix matrix{&record, rows, cols}; + + parallel_backend_fixture fixture; + std::vector out_serial(static_cast(rows), -1.0f); + std::vector out_parallel(static_cast(rows), -2.0f); + + emel::kernel::event::op_mul_mat serial_ev{ + .src0 = gen_detail::make_src_view(matrix), + .src1 = gen_detail::make_src_view(input.data(), static_cast(1u), + static_cast(input.size())), + .dst = gen_detail::make_dst_view( + out_serial.data(), static_cast(1u), + static_cast(out_serial.size())), + }; + emel::kernel::event::op_mul_mat parallel_ev = serial_ev; + parallel_ev.dst = + gen_detail::make_dst_view(out_parallel.data(), static_cast(1u), + static_cast(out_parallel.size())); + + CHECK(gen_detail::compute_mul_mat(fixture.backend, + serial_ev)); + CHECK(gen_detail::compute_mul_mat(fixture.backend, + parallel_ev)); + CHECK(std::memcmp(out_serial.data(), out_parallel.data(), + out_serial.size() * sizeof(float)) == 0); +} + +TEST_CASE("parallel matmul route guard rejects disengaged lane pool") { + emel::text::generator::action::context ctx = {}; + ctx.compute.backend.n_embd = + emel::text::generator::test::k_generation_route_policy + .parallel_min_gemv_dim; + emel::text::generator::event::generate_ctx run_ctx = {}; + std::array messages = {}; + std::array output = {}; + size_t output_length = 0u; + const emel::text::generator::event::generate request{ + std::span(messages), 1, + std::span(output), output_length}; + const emel::text::generator::event::generate_run run{request, run_ctx}; + + CHECK_FALSE(emel::text::generator::guard::guard_decode_parallel_lanes_ready{}( + run, ctx)); + + emel::kernel::matmul::lane_pool serial_matmul_lanes = {}; + emel::kernel::matmul::execution_policy serial_policy{ + .parallel_matmul_lanes = &serial_matmul_lanes, + .kernel_kind = emel::kernel::detect_host_kind(), + .active_lanes = 1u, + }; + emel::kernel::matmul::sm serial_matmul_actor{serial_policy}; + ctx.compute.backend.matmul_actor = &serial_matmul_actor; + ctx.compute.backend.matmul_lane_mode = + emel::kernel::matmul::lane_mode::serial; + CHECK_FALSE(emel::text::generator::guard::guard_decode_parallel_lanes_ready{}( + run, ctx)); + + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; + auto parallel_policy = + emel::kernel::matmul::make_auto_execution_policy(parallel_matmul_lanes); + emel::kernel::matmul::sm parallel_matmul_actor{parallel_policy}; + ctx.compute.backend.matmul_actor = ¶llel_matmul_actor; + ctx.compute.backend.matmul_lane_mode = + emel::kernel::matmul::lane_mode::parallel; + CHECK(emel::text::generator::guard::guard_decode_parallel_lanes_ready{}(run, + ctx)); + ctx.compute.backend.parallel_lanes_enabled = false; + CHECK_FALSE(emel::text::generator::guard::guard_decode_parallel_lanes_ready{}( + run, ctx)); +} + +TEST_CASE( + "benchmark lane configuration guards and actions toggle parallel lanes") { + emel::text::generator::action::context ctx = {}; + const emel::text::generator::event::configure_benchmark_lane single{ + emel::text::generator::benchmark_lane::single}; + const emel::text::generator::event::configure_benchmark_lane multithreaded{ + emel::text::generator::benchmark_lane::multithreaded}; + + CHECK( + emel::text::generator::guard::guard_benchmark_lane_single{}(single, ctx)); + CHECK_FALSE(emel::text::generator::guard::guard_benchmark_lane_single{}( + multithreaded, ctx)); + CHECK(emel::text::generator::guard::guard_benchmark_lane_multithreaded{}( + multithreaded, ctx)); + CHECK_FALSE( + emel::text::generator::guard::guard_benchmark_lane_multithreaded{}(single, + ctx)); + + ctx.compute.backend.parallel_lanes_enabled = true; + ctx.benchmark_parallel_lanes_enabled = true; + emel::text::generator::action::effect_disable_parallel_benchmark_lanes(single, + ctx); + CHECK_FALSE(ctx.compute.backend.parallel_lanes_enabled); + CHECK_FALSE(ctx.benchmark_parallel_lanes_enabled); + ctx.compute.backend.parallel_lanes_enabled = true; + emel::text::generator::action::apply_benchmark_lane_policy(ctx); + CHECK_FALSE(ctx.compute.backend.parallel_lanes_enabled); + emel::text::generator::action::effect_enable_parallel_benchmark_lanes( + multithreaded, ctx); + CHECK(ctx.compute.backend.parallel_lanes_enabled); + CHECK(ctx.benchmark_parallel_lanes_enabled); +} + +} // namespace diff --git a/tests/sm/sm_policy_tests.cpp b/tests/sm/sm_policy_tests.cpp index a8c38e22..46bc5bf7 100644 --- a/tests/sm/sm_policy_tests.cpp +++ b/tests/sm/sm_policy_tests.cpp @@ -737,6 +737,128 @@ TEST_CASE("fork_join_lane_pool_wait_returns_after_worker_slot_reusable") { CHECK(ran.load(std::memory_order_acquire) == k_rounds); } +TEST_CASE("fork_join_lane_pool_batch_uses_distinct_workers_and_drains") { + using pool_type = emel::policy::fork_join_lane_pool<2u, 128u, 64u>; + pool_type pool{}; + pool_type::join_group group{}; + std::atomic entered{0}; + std::atomic release{false}; + + const size_t submitted = pool.try_submit_batch( + group, + [&]() noexcept { + entered.fetch_add(1, std::memory_order_release); + while (!release.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }, + [&]() noexcept { + entered.fetch_add(1, std::memory_order_release); + while (!release.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + + CHECK(submitted == 2u); + require_eventually("batch tasks did not enter distinct workers", [&]() { + return entered.load(std::memory_order_acquire) == 2; + }); + release.store(true, std::memory_order_release); + CHECK(group.wait()); +} + +TEST_CASE("fork_join_lane_pool_starts_only_the_runtime_worker_budget") { + using pool_type = emel::policy::fork_join_lane_pool<7u, 128u, 64u>; + pool_type pool{1u}; + CHECK(pool.active_worker_count() == 1u); + + pool_type::join_group group{}; + std::atomic ran{0}; + const size_t submitted = pool.try_submit_batch( + group, + [&]() noexcept { ran.fetch_add(1, std::memory_order_relaxed); }, + [&]() noexcept { ran.fetch_add(1, std::memory_order_relaxed); }); + + CHECK(submitted == 1u); + CHECK_FALSE(group.wait()); + CHECK(ran.load(std::memory_order_acquire) == 1); +} + +TEST_CASE("fork_join_lane_pool_batch_rejects_partial_claim_and_reuses_slots") { + using pool_type = emel::policy::fork_join_lane_pool<2u, 128u, 64u>; + pool_type pool{}; + pool_type::join_group partial_group{}; + std::atomic entered{0}; + std::atomic release{false}; + std::atomic unclaimed_ran{false}; + + const size_t submitted = pool.try_submit_batch( + partial_group, + [&]() noexcept { + entered.fetch_add(1, std::memory_order_release); + while (!release.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }, + [&]() noexcept { + entered.fetch_add(1, std::memory_order_release); + while (!release.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }, + [&]() noexcept { unclaimed_ran.store(true, std::memory_order_release); }); + + CHECK(submitted == 2u); + require_eventually("partial batch tasks did not enter", [&]() { + return entered.load(std::memory_order_acquire) == 2; + }); + CHECK_FALSE(unclaimed_ran.load(std::memory_order_acquire)); + release.store(true, std::memory_order_release); + CHECK_FALSE(partial_group.wait()); + + pool_type::join_group reuse_group{}; + std::atomic reused{0}; + size_t reused_count = 0u; + bool reuse_joined = false; + size_t allocation_count = 0u; + { + allocation_scope allocations{}; + reused_count = pool.try_submit_batch( + reuse_group, + [&]() noexcept { reused.fetch_add(1, std::memory_order_relaxed); }, + [&]() noexcept { reused.fetch_add(1, std::memory_order_relaxed); }); + reuse_joined = reuse_group.wait(); + allocation_count = allocations.allocations(); + } + CHECK(reused_count == 2u); + CHECK(reuse_joined); + CHECK(reused.load(std::memory_order_acquire) == 2); + CHECK(allocation_count == 0u); +} + +TEST_CASE("fork_join_lane_pool_batch_rejects_same_pool_worker_submit") { + using pool_type = emel::policy::fork_join_lane_pool<1u, 128u, 64u>; + pool_type pool{}; + pool_type::join_group outer_group{}; + std::atomic nested_submitted{1u}; + std::atomic nested_joined{true}; + std::atomic nested_ran{false}; + + REQUIRE(pool.try_submit(outer_group, [&]() noexcept { + pool_type::join_group nested_group{}; + nested_submitted.store( + pool.try_submit_batch(nested_group, [&]() noexcept { + nested_ran.store(true, std::memory_order_release); + }), + std::memory_order_release); + nested_joined.store(nested_group.wait(), std::memory_order_release); + })); + CHECK(outer_group.wait()); + CHECK(nested_submitted.load(std::memory_order_acquire) == 0u); + CHECK_FALSE(nested_joined.load(std::memory_order_acquire)); + CHECK_FALSE(nested_ran.load(std::memory_order_acquire)); +} + TEST_CASE("co_sm_contextful_wrapper_injects_context") { namespace sml = stateforward::sml; context_co_probe ctx{.value = 40}; diff --git a/tests/speech/codec/mimi_lifecycle_tests.cpp b/tests/speech/codec/mimi_lifecycle_tests.cpp index 8cc75a91..f4f237a3 100644 --- a/tests/speech/codec/mimi_lifecycle_tests.cpp +++ b/tests/speech/codec/mimi_lifecycle_tests.cpp @@ -26,6 +26,52 @@ namespace { namespace mimi = emel::speech::codec::mimi; +template +concept mutable_bind_model_field = + requires(contract_type &contract, const emel::model::data *model) { + contract.model = model; + }; + +template +concept mutable_bind_prepared_field = + requires(contract_type &contract) { contract.prepared_floats = 1u; }; + +template +concept mutable_bind_state_field = + requires(contract_type &contract) { contract.state_floats = 1u; }; + +template +concept mutable_bind_workspace_field = + requires(contract_type &contract) { contract.workspace_floats = 1u; }; + +template +concept mutable_bind_frame_field = + requires(contract_type &contract) { contract.frame_floats = 1u; }; + +template +concept mutable_bind_f32_field = + requires(contract_type &contract) { contract.f32_valid = true; }; + +template +concept mutable_bind_native_field = + requires(contract_type &contract) { contract.native_valid = true; }; + +template +concept mutable_bind_q8_field = + requires(contract_type &contract) { contract.q8_valid = true; }; + +static_assert(!std::is_aggregate_v); +static_assert(!std::is_default_constructible_v); +static_assert(!std::is_trivially_copyable_v); +static_assert(!mutable_bind_model_field); +static_assert(!mutable_bind_prepared_field); +static_assert(!mutable_bind_state_field); +static_assert(!mutable_bind_workspace_field); +static_assert(!mutable_bind_frame_field); +static_assert(!mutable_bind_f32_field); +static_assert(!mutable_bind_native_field); +static_assert(!mutable_bind_q8_field); + // modeled-event observation: initialize_done carries the codec dims, and // the done callbacks count completed frames (no context peeking) int32_t g_frame_samples = 0; @@ -211,13 +257,19 @@ TEST_CASE("mimi codec facade initializes, encodes, and decodes frames") { mimi::sm machine{}; namespace sml = stateforward::sml; CHECK(machine.is(sml::state)); + mimi::event::diagnostics diagnostics_before{}; + REQUIRE(machine.process_event( + mimi::event::capture_diagnostics{diagnostics_before})); + CHECK(diagnostics_before.projection_prepare_calls == 0u); emel::error::type err = emel::error::cast(mimi::error::none); g_frame_samples = 0; g_n_q = 0; g_encode_done_count = 0; g_decode_done_count = 0; - mimi::event::initialize init{*loaded.model, std::span{arenas.prepared}, + mimi::event::initialize init{*loaded.model, + mimi::make_bind_contract(*loaded.model), + std::span{arenas.prepared}, std::span{arenas.state}, std::span{arenas.workspace}, std::span{arenas.frame}}; @@ -229,6 +281,11 @@ TEST_CASE("mimi codec facade initializes, encodes, and decodes frames") { check_session_ready(machine); CHECK(g_frame_samples == 1920); CHECK(g_n_q == 2); + mimi::event::diagnostics diagnostics_initialized{}; + REQUIRE(machine.process_event( + mimi::event::capture_diagnostics{diagnostics_initialized})); + CHECK(diagnostics_initialized.projection_prepare_calls > 0u); + CHECK(diagnostics_initialized.projection_prepared_floats > 0u); const auto pcm = deterministic_pcm(g_frame_samples); std::vector codes(static_cast(g_n_q), -1); @@ -255,6 +312,17 @@ TEST_CASE("mimi codec facade initializes, encodes, and decodes frames") { CHECK(err == emel::error::cast(mimi::error::none)); check_session_ready(machine); CHECK(g_decode_done_count == 1u); + mimi::event::diagnostics diagnostics_after_frames{}; + REQUIRE(machine.process_event( + mimi::event::capture_diagnostics{diagnostics_after_frames})); +#if defined(__aarch64__) || defined(_M_ARM64) + CHECK(diagnostics_after_frames.projection_exact_x4_calls > + diagnostics_initialized.projection_exact_x4_calls); + CHECK(diagnostics_after_frames.legacy_f32_projection_calls == 0u); +#else + CHECK(diagnostics_after_frames.projection_exact_x4_calls == 0u); + CHECK(diagnostics_after_frames.legacy_f32_projection_calls > 0u); +#endif REQUIRE(machine.process_event(mimi::event::reset_stream{})); const size_t partial_count = @@ -331,7 +399,9 @@ TEST_CASE("mimi codec facade reports bind failures explicitly") { mimi::sm machine{}; namespace sml = stateforward::sml; emel::error::type err = emel::error::cast(mimi::error::none); - mimi::event::initialize init{*loaded.model, std::span{arenas.prepared}, + mimi::event::initialize init{*loaded.model, + mimi::make_bind_contract(*loaded.model), + std::span{arenas.prepared}, std::span{arenas.state}, std::span{arenas.workspace}, std::span{arenas.frame}}; @@ -357,6 +427,7 @@ TEST_CASE("mimi codec facade reports bind failures explicitly") { err = emel::error::cast(mimi::error::none); g_init_error = emel::error::cast(mimi::error::none); mimi::event::initialize lm_init{*lm_loaded.model, + mimi::make_bind_contract(*lm_loaded.model), std::span{lm_arenas.prepared}, std::span{lm_arenas.state}, std::span{lm_arenas.workspace}, @@ -385,7 +456,9 @@ TEST_CASE("mimi codec initialize rejects out-of-contract model metadata") { mimi::sm machine{}; emel::error::type err = emel::error::cast(mimi::error::none); g_init_error = emel::error::cast(mimi::error::none); - mimi::event::initialize init{model, std::span{arenas.prepared}, + mimi::event::initialize init{model, + mimi::make_bind_contract(model), + std::span{arenas.prepared}, std::span{arenas.state}, std::span{arenas.workspace}, std::span{arenas.frame}}; @@ -605,8 +678,10 @@ TEST_CASE("mimi codec initialize rejects null arena storage") { // walk and reset_streaming_state write through every arena. mimi::event::initialize init{ *loaded.model, + mimi::make_bind_contract(*loaded.model), std::span{static_cast(nullptr), arenas.prepared.size()}, - std::span{arenas.state}, std::span{arenas.workspace}, + std::span{arenas.state}, + std::span{arenas.workspace}, std::span{arenas.frame}}; init.error_out = &err; CHECK_FALSE(machine.process_event(init)); @@ -614,6 +689,113 @@ TEST_CASE("mimi codec initialize rejects null arena storage") { CHECK(machine.is(sml::state)); } +TEST_CASE("mimi codec bind contracts are opaque and model bound") { + auto loaded = load_mimi_fixture_or_skip(); + if (loaded.model == nullptr) { + return; + } + + auto other_loaded = load_mimi_fixture_or_skip(); + if (other_loaded.model == nullptr) { + return; + } + + codec_arenas arenas{}; + size_arenas(*other_loaded.model, arenas); + auto contract = mimi::make_bind_contract(*loaded.model); + + namespace sml = stateforward::sml; + mimi::sm machine{}; + emel::error::type err = emel::error::cast(mimi::error::none); + mimi::event::initialize init{*other_loaded.model, + contract, + std::span{arenas.prepared}, + std::span{arenas.state}, + std::span{arenas.workspace}, + std::span{arenas.frame}}; + init.error_out = &err; + CHECK_FALSE(machine.process_event(init)); + CHECK(err == emel::error::cast(mimi::error::bind_failed)); + CHECK(machine.is(sml::state)); +} + +TEST_CASE("mimi codec bind contracts reject same-address model mutation") { + auto loaded = load_mimi_fixture_or_skip(); + if (loaded.model == nullptr) { + return; + } + + codec_arenas arenas{}; + size_arenas(*loaded.model, arenas); + auto contract = mimi::make_bind_contract(*loaded.model); + SUBCASE("Mimi hyperparameters") { ++loaded.model->mimi.sample_rate; } + SUBCASE("tensor records") { + REQUIRE(loaded.model->n_tensors > 0u); + ++loaded.model->tensors[0].file_offset; + } + SUBCASE("used tensor-name storage") { + REQUIRE(loaded.model->name_bytes_used > 0u); + loaded.model->name_storage[loaded.model->name_bytes_used - 1u] ^= 1; + } + + namespace sml = stateforward::sml; + mimi::sm machine{}; + emel::error::type err = emel::error::cast(mimi::error::none); + mimi::event::initialize init{*loaded.model, + contract, + std::span{arenas.prepared}, + std::span{arenas.state}, + std::span{arenas.workspace}, + std::span{arenas.frame}}; + init.error_out = &err; + CHECK_FALSE(machine.process_event(init)); + CHECK(err == emel::error::cast(mimi::error::bind_failed)); + CHECK(machine.is(sml::state)); +} + +TEST_CASE("mimi codec initialize rejects arenas overlapping model tensors") { + auto loaded = load_mimi_fixture_or_skip(); + if (loaded.model == nullptr) { + return; + } + + codec_arenas arenas{}; + size_arenas(*loaded.model, arenas); + REQUIRE_FALSE(arenas.prepared.empty()); + + emel::model::data::tensor_record *projection = nullptr; + for (uint32_t index = 0u; index < loaded.model->n_tensors; ++index) { + auto &tensor = loaded.model->tensors[index]; + const auto name = emel::model::tensor_name_view(*loaded.model, tensor); + if (name.ends_with(".self_attn.in_projs.0.weight")) { + projection = &tensor; + break; + } + } + REQUIRE(projection != nullptr); + REQUIRE(projection->data_size <= arenas.prepared.size() * sizeof(float)); + projection->data = arenas.prepared.data(); + + namespace sml = stateforward::sml; + mimi::sm machine{}; + emel::error::type err = emel::error::cast(mimi::error::none); + mimi::event::initialize init{*loaded.model, + mimi::make_bind_contract(*loaded.model), + std::span{arenas.prepared}, + std::span{arenas.state}, + std::span{arenas.workspace}, + std::span{arenas.frame}}; + init.error_out = &err; + CHECK_FALSE(machine.process_event(init)); + CHECK(err == emel::error::cast(mimi::error::arena_capacity)); + CHECK(machine.is(sml::state)); + + mimi::event::diagnostics diagnostics{}; + REQUIRE(machine.process_event(mimi::event::capture_diagnostics{diagnostics})); + CHECK(diagnostics.projection_prepare_calls == 0u); + CHECK(diagnostics.projection_prepared_floats == 0u); +} + TEST_CASE("mimi codec facade reports unexpected event ordering as errors") { auto loaded = load_mimi_fixture_or_skip(); if (loaded.model == nullptr) { @@ -632,7 +814,9 @@ TEST_CASE("mimi codec facade reports unexpected event ordering as errors") { CHECK(machine.is(sml::state)); emel::error::type err = emel::error::cast(mimi::error::none); - mimi::event::initialize init{*loaded.model, std::span{arenas.prepared}, + mimi::event::initialize init{*loaded.model, + mimi::make_bind_contract(*loaded.model), + std::span{arenas.prepared}, std::span{arenas.state}, std::span{arenas.workspace}, std::span{arenas.frame}}; @@ -643,10 +827,12 @@ TEST_CASE("mimi codec facade reports unexpected event ordering as errors") { // initialize while a session is ready is equally out of order; the // dispatch must fail and surface the explicit unexpected_event error. emel::error::type second_err = emel::error::cast(mimi::error::none); - mimi::event::initialize second{ - *loaded.model, std::span{arenas.prepared}, - std::span{arenas.state}, std::span{arenas.workspace}, - std::span{arenas.frame}}; + mimi::event::initialize second{*loaded.model, + mimi::make_bind_contract(*loaded.model), + std::span{arenas.prepared}, + std::span{arenas.state}, + std::span{arenas.workspace}, + std::span{arenas.frame}}; second.error_out = &second_err; CHECK_FALSE(machine.process_event(second)); CHECK(second_err == emel::error::cast(mimi::error::unexpected_event)); @@ -671,7 +857,9 @@ TEST_CASE("mimi codec facade rejects malformed frame requests explicitly") { emel::error::type err = emel::error::cast(mimi::error::none); g_frame_samples = 0; g_n_q = 0; - mimi::event::initialize init{*loaded.model, std::span{arenas.prepared}, + mimi::event::initialize init{*loaded.model, + mimi::make_bind_contract(*loaded.model), + std::span{arenas.prepared}, std::span{arenas.state}, std::span{arenas.workspace}, std::span{arenas.frame}}; @@ -775,10 +963,12 @@ TEST_CASE("mimi codec facade streams the f16 and q8 operand-class fixtures") { emel::error::type err = emel::error::cast(mimi::error::none); g_frame_samples = 0; g_n_q = 0; - mimi::event::initialize init{ - *loaded.model, std::span{arenas.prepared}, - std::span{arenas.state}, std::span{arenas.workspace}, - std::span{arenas.frame}}; + mimi::event::initialize init{*loaded.model, + mimi::make_bind_contract(*loaded.model), + std::span{arenas.prepared}, + std::span{arenas.state}, + std::span{arenas.workspace}, + std::span{arenas.frame}}; init.error_out = &err; init.on_done = emel::callback::from<&on_initialize_done>(); diff --git a/tests/speech/generator/frame_lifecycle_tests.cpp b/tests/speech/generator/frame_lifecycle_tests.cpp new file mode 100644 index 00000000..7b8d15d6 --- /dev/null +++ b/tests/speech/generator/frame_lifecycle_tests.cpp @@ -0,0 +1,696 @@ +#include "doctest/doctest.h" + +#include +#include +#include +#include +#include +#include + +#include "../../allocation_tracker.hpp" +#include "emel/batch/planner/events.hpp" +#include "emel/error/error.hpp" +#include "emel/speech/generator/frame/sm.hpp" + +namespace { + +namespace frame = emel::speech::generator::frame; +namespace sml = stateforward::sml; + +constexpr size_t PUBLIC_CODEBOOK_COUNT = 2u; +constexpr size_t MODEL_TOKEN_COUNT = 4u; +constexpr size_t PREDICTED_CODEBOOK_COUNT = 3u; +constexpr int32_t MAX_FRAME_TRANSITIONS = 8; +constexpr int32_t MAX_CROSS_ACTOR_CALLS = 6; +constexpr auto FAKE_DISPATCH_BUDGET = std::chrono::milliseconds{10}; + +enum class phase_id : int32_t { + tokenize = 1, + plan = 2, + predict = 3, + graph = 4, + sample = 5, + detokenize = 6, +}; + +struct phase_trace { + std::array entries{}; + size_t size = 0; + bool overflow = false; + + void record(const phase_id phase) noexcept { + if (size < entries.size()) { + entries[size++] = phase; + } else { + overflow = true; + } + } + + void reset() noexcept { + entries.fill({}); + size = 0; + overflow = false; + } +}; + +struct sml_trace { + std::array entries{}; + size_t size = 0; + int32_t transitions = 0; + bool overflow = false; + + void record(const uint8_t value) noexcept { + if (size < entries.size()) { + entries[size++] = value; + } else { + overflow = true; + } + } + + void reset() noexcept { + entries.fill(0); + size = 0; + transitions = 0; + overflow = false; + } + + template + void log_process_event(const event_type &) noexcept { + record(1); + } + + template + void log_guard(const guard_type &, const event_type &, + const bool accepted) noexcept { + record(accepted ? 3 : 2); + } + + template + void log_action(const action_type &, const event_type &) noexcept { + record(4); + } + + template + void log_state_change(const source_state &, + const destination_state &) noexcept { + ++transitions; + record(5); + } +}; + +struct fake_workspace {}; + +struct fake_tokenize { + std::span audio_tokens = {}; + std::span model_tokens_out = {}; + int32_t &error_out; +}; + +struct fake_predict { + fake_predict(std::span tokens_ref, + fake_workspace &workspace_ref, + const int32_t planned_step_size_ref, + const int32_t planned_output_count_ref) noexcept + : tokens(tokens_ref), workspace(workspace_ref), + planned_step_size(planned_step_size_ref), + planned_output_count(planned_output_count_ref) {} + + std::span tokens; + fake_workspace &workspace; + int32_t planned_step_size = 0; + int32_t planned_output_count = 0; + emel::error::type *error_out = nullptr; +}; + +struct fake_graph { + fake_graph(fake_workspace &workspace_ref, + std::span tokens_ref) noexcept + : workspace(workspace_ref), tokens(tokens_ref) {} + + fake_workspace &workspace; + std::span tokens; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; +}; + +struct fake_sample { + fake_sample(fake_workspace &workspace_ref, + std::span tokens_ref, + std::span tokens_out_ref, + int32_t &text_token_out_ref) noexcept + : workspace(workspace_ref), tokens(tokens_ref), + tokens_out(tokens_out_ref), text_token_out(text_token_out_ref) {} + + fake_workspace &workspace; + std::span tokens; + std::span tokens_out; + int32_t &text_token_out; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; +}; + +struct fake_detokenize { + int32_t text_token = -1; + std::span audio_tokens = {}; + int32_t &text_token_out; + std::span audio_tokens_out = {}; + bool &produced_out; + int32_t &error_out; +}; + +struct fake_tokenizer_actor { + phase_trace *trace = nullptr; + int32_t tokenize_calls = 0; + int32_t detokenize_calls = 0; + bool tokenize_accepted = true; + bool detokenize_accepted = true; + bool produce = true; + int32_t tokenize_error = 0; + int32_t detokenize_error = 0; + + bool process_event(const fake_tokenize &request) noexcept { + trace->record(phase_id::tokenize); + ++tokenize_calls; + std::copy(request.audio_tokens.begin(), request.audio_tokens.end(), + request.model_tokens_out.begin()); + request.error_out = tokenize_error; + return tokenize_accepted; + } + + bool process_event(const fake_detokenize &request) noexcept { + trace->record(phase_id::detokenize); + ++detokenize_calls; + for (size_t index = 0; index < request.audio_tokens_out.size(); ++index) { + request.audio_tokens_out[index] = request.audio_tokens[index]; + } + request.text_token_out = request.text_token; + request.produced_out = produce; + request.error_out = detokenize_error; + return detokenize_accepted; + } +}; + +struct fake_planner_actor { + phase_trace *trace = nullptr; + int32_t calls = 0; + bool accepted = true; + bool report_error = false; + + bool process_event( + const emel::batch::planner::event::plan_request &request) noexcept { + trace->record(phase_id::plan); + ++calls; + if (accepted) { + if (report_error) { + request.on_error(emel::batch::planner::events::plan_error{ + .err = + emel::error::cast(emel::batch::planner::error::invalid_request), + .request = &request, + }); + } else { + const std::array step_sizes{1}; + request.on_done(emel::batch::planner::events::plan_done{ + .request = &request, + .step_sizes = step_sizes.data(), + .step_count = 1, + .total_outputs = 1, + }); + } + } + return accepted; + } +}; + +struct fake_predictor_actor { + phase_trace *trace = nullptr; + int32_t calls = 0; + int32_t last_plan_step_size = 0; + int32_t last_plan_output_count = 0; + bool accepted = true; + emel::error::type error = 0; + + bool process_event(const fake_predict &request) noexcept { + trace->record(phase_id::predict); + ++calls; + last_plan_step_size = request.planned_step_size; + last_plan_output_count = request.planned_output_count; + *request.error_out = error; + return accepted; + } +}; + +struct fake_graph_actor { + phase_trace *trace = nullptr; + int32_t calls = 0; + bool accepted = true; + emel::error::type error = 0; + emel::error::type graph_error = 0; + + bool process_event(const fake_graph &request) noexcept { + trace->record(phase_id::graph); + ++calls; + *request.error_out = error; + *request.graph_error_out = graph_error; + return accepted; + } +}; + +struct fake_sampler_actor { + phase_trace *trace = nullptr; + int32_t calls = 0; + bool accepted = true; + emel::error::type error = 0; + emel::error::type graph_error = 0; + + bool process_event(const fake_sample &request) noexcept { + trace->record(phase_id::sample); + ++calls; + for (size_t index = 0; index < request.tokens_out.size(); ++index) { + request.tokens_out[index] = 111 + static_cast(index); + } + request.text_token_out = 42; + *request.error_out = error; + *request.graph_error_out = graph_error; + return accepted; + } +}; + +struct fake_dependencies { + using tokenize_event = fake_tokenize; + using predict_event = fake_predict; + using graph_event = fake_graph; + using sample_event = fake_sample; + using detokenize_event = fake_detokenize; + + fake_planner_actor &planner; + fake_tokenizer_actor &tokenizer; + fake_predictor_actor &predictor; + fake_graph_actor &graph; + fake_sampler_actor &sampler; + fake_workspace &prediction_workspace; + std::span model_codes = {}; + std::span predicted_codes = {}; + int32_t codebook_count = 0; + emel::batch::planner::event::plan_mode frame_plan_mode = + emel::batch::planner::event::plan_mode::simple; + int32_t frame_plan_steps = 0; + int32_t frame_plan_token_count = 0; + bool frame_plan_output_all = false; +}; + +template struct fixture_machine { + using type = frame::traced_sm; +}; + +template <> struct fixture_machine { + using type = frame::sm; +}; + +template struct fixture_base { + phase_trace phases{}; + std::array model_codes{}; + std::array predicted_codes{}; + fake_planner_actor planner{.trace = &phases}; + fake_tokenizer_actor tokenizer{.trace = &phases}; + fake_predictor_actor predictor{.trace = &phases}; + fake_graph_actor graph{.trace = &phases}; + fake_sampler_actor sampler{.trace = &phases}; + fake_workspace prediction_workspace{}; + fake_dependencies dependencies{ + .planner = planner, + .tokenizer = tokenizer, + .predictor = predictor, + .graph = graph, + .sampler = sampler, + .prediction_workspace = prediction_workspace, + .model_codes = std::span{model_codes}, + .predicted_codes = std::span{predicted_codes}, + .codebook_count = static_cast(PUBLIC_CODEBOOK_COUNT), + .frame_plan_mode = emel::batch::planner::event::plan_mode::simple, + .frame_plan_steps = 1, + .frame_plan_token_count = 1, + .frame_plan_output_all = true, + }; + typename fixture_machine::type machine; + + template + explicit fixture_base(args &...args_in) : machine{dependencies, args_in...} {} +}; + +using fixture = fixture_base<>; + +struct callback_probe { + int32_t done_calls = 0; + int32_t error_calls = 0; + int32_t text_token = -1; + emel::error::type last_error = 0; + + void done(const frame::events::run_done &ev) noexcept { + ++done_calls; + text_token = ev.text_token; + } + + void error(const frame::events::run_error &ev) noexcept { + ++error_calls; + last_error = ev.err; + } +}; + +struct unexpected_request { + emel::error::type &error_out; +}; + +template +concept accepts_internal_run = requires( + machine_type &machine, const frame::detail::run_frame &runtime_ev) { + machine.process_event(runtime_ev); +}; + +static_assert(!accepts_internal_run>); + +template struct forbidden_queue {}; +struct forbidden_lock {}; + +template +concept accepts_actor_policy = + requires(dependencies_type &dependencies, policy_type &policy) { + frame::sm{dependencies, policy}; + }; + +static_assert(!accepts_actor_policy>); +static_assert(!accepts_actor_policy>); +static_assert( + !accepts_actor_policy>); +static_assert(!accepts_actor_policy>); + +emel::error::type error_code(const frame::error value) noexcept { + return frame::action::error_code(value); +} + +} // namespace + +TEST_CASE("speech_generator_frame_runs_the_middle_pipeline_synchronously") { + fixture test{}; + callback_probe probe{}; + const std::array encoded_tokens{11, 12}; + std::array generated_tokens{}; + int32_t text_token = -1; + emel::error::type err = -1; + const auto on_done = + emel::callback::from< + callback_probe, &callback_probe::done>(&probe); + const frame::event::run request{encoded_tokens, generated_tokens, text_token, + err, on_done}; + + REQUIRE(test.machine.process_event(request)); + CHECK(test.machine.is(sml::state)); + CHECK(test.tokenizer.tokenize_calls == 1); + CHECK(test.planner.calls == 1); + CHECK(test.predictor.calls == 1); + CHECK(test.graph.calls == 1); + CHECK(test.sampler.calls == 1); + CHECK(test.tokenizer.detokenize_calls == 1); + CHECK(test.predictor.last_plan_step_size == 1); + CHECK(test.predictor.last_plan_output_count == 1); + CHECK(generated_tokens == std::array{111, 112}); + CHECK(text_token == 42); + CHECK(err == error_code(frame::error::none)); + CHECK(probe.done_calls == 1); + CHECK(probe.error_calls == 0); + CHECK(probe.text_token == 42); +} + +TEST_CASE("speech_generator_frame_rejects_invalid_spans_and_restores_idle") { + SUBCASE("encoded input is narrower than the public codebook contract") { + fixture test{}; + callback_probe probe{}; + const std::array encoded_tokens{11}; + std::array generated_tokens{}; + int32_t text_token = 7; + emel::error::type err = 0; + const auto on_error = + emel::callback::from< + callback_probe, &callback_probe::error>(&probe); + const frame::event::run request{ + encoded_tokens, generated_tokens, text_token, err, {}, on_error}; + + CHECK_FALSE(test.machine.process_event(request)); + CHECK(test.machine.is(sml::state)); + CHECK(err == error_code(frame::error::invalid_request)); + CHECK(text_token == -1); + CHECK(probe.error_calls == 1); + CHECK(test.tokenizer.tokenize_calls == 0); + } + + SUBCASE("generated output capacity must equal the public codebook contract") { + fixture test{}; + const std::array encoded_tokens{11, 12}; + std::array generated_tokens{31, 32, + 991}; + int32_t text_token = 7; + emel::error::type err = 0; + const frame::event::run request{encoded_tokens, generated_tokens, + text_token, err}; + + CHECK_FALSE(test.machine.process_event(request)); + CHECK(test.machine.is(sml::state)); + CHECK(err == error_code(frame::error::invalid_request)); + CHECK(text_token == -1); + CHECK(generated_tokens == + std::array{31, 32, 991}); + CHECK(test.tokenizer.tokenize_calls == 0); + } +} + +TEST_CASE("speech_generator_frame_reports_each_child_phase_failure") { + const auto run_failure = [](fixture &test, const frame::error expected) { + callback_probe probe{}; + const std::array encoded_tokens{11, 12}; + std::array generated_tokens{}; + int32_t text_token = 7; + emel::error::type err = 0; + const auto on_error = + emel::callback::from< + callback_probe, &callback_probe::error>(&probe); + const frame::event::run request{ + encoded_tokens, generated_tokens, text_token, err, {}, on_error}; + + CHECK_FALSE(test.machine.process_event(request)); + CHECK(test.machine.is(sml::state)); + CHECK(err == error_code(expected)); + CHECK(text_token == -1); + CHECK(probe.error_calls == 1); + CHECK(probe.last_error == error_code(expected)); + }; + + SUBCASE("tokenize") { + fixture test{}; + test.tokenizer.tokenize_accepted = false; + run_failure(test, frame::error::tokenize_failed); + } + SUBCASE("plan") { + fixture test{}; + test.planner.accepted = false; + run_failure(test, frame::error::planning_failed); + } + SUBCASE("predict") { + fixture test{}; + test.predictor.accepted = false; + run_failure(test, frame::error::predict_failed); + } + SUBCASE("graph") { + fixture test{}; + test.graph.accepted = false; + run_failure(test, frame::error::graph_failed); + } + SUBCASE("sample") { + fixture test{}; + test.sampler.accepted = false; + run_failure(test, frame::error::sample_failed); + } + SUBCASE("detokenize") { + fixture test{}; + test.tokenizer.detokenize_accepted = false; + run_failure(test, frame::error::detokenize_failed); + } +} + +TEST_CASE("speech_generator_frame_maps_accepted_child_error_channels") { + const auto run_failure = [](fixture &test, const frame::error expected) { + const std::array encoded_tokens{11, 12}; + std::array generated_tokens{}; + int32_t text_token = 7; + emel::error::type err = 0; + const frame::event::run request{encoded_tokens, generated_tokens, + text_token, err}; + + CHECK_FALSE(test.machine.process_event(request)); + CHECK(test.machine.is(sml::state)); + CHECK(err == error_code(expected)); + CHECK(text_token == -1); + }; + + SUBCASE("tokenize error with accepted dispatch") { + fixture test{}; + test.tokenizer.tokenize_error = 9; + run_failure(test, frame::error::tokenize_failed); + } + SUBCASE("planner error callback with accepted dispatch") { + fixture test{}; + test.planner.report_error = true; + run_failure(test, frame::error::planning_failed); + } + SUBCASE("predict error with accepted dispatch") { + fixture test{}; + test.predictor.error = 9; + run_failure(test, frame::error::predict_failed); + } + SUBCASE("graph error channel with accepted dispatch") { + fixture test{}; + test.graph.graph_error = 9; + run_failure(test, frame::error::graph_failed); + } + SUBCASE("graph child error channel with accepted dispatch") { + fixture test{}; + test.graph.error = 9; + run_failure(test, frame::error::graph_failed); + } + SUBCASE("sample graph error channel with accepted dispatch") { + fixture test{}; + test.sampler.graph_error = 9; + run_failure(test, frame::error::sample_failed); + } + SUBCASE("sample child error channel with accepted dispatch") { + fixture test{}; + test.sampler.error = 9; + run_failure(test, frame::error::sample_failed); + } + SUBCASE("detokenize error with accepted dispatch") { + fixture test{}; + test.tokenizer.detokenize_error = 9; + run_failure(test, frame::error::detokenize_failed); + } +} + +TEST_CASE("speech_generator_frame_treats_missing_output_as_pending_error") { + fixture test{}; + test.tokenizer.produce = false; + callback_probe probe{}; + const std::array encoded_tokens{11, 12}; + std::array generated_tokens{}; + int32_t text_token = 7; + emel::error::type err = 0; + const auto on_error = + emel::callback::from< + callback_probe, &callback_probe::error>(&probe); + const frame::event::run request{ + encoded_tokens, generated_tokens, text_token, err, {}, on_error}; + + CHECK_FALSE(test.machine.process_event(request)); + CHECK(test.machine.is(sml::state)); + CHECK(err == error_code(frame::error::frame_pending)); + CHECK(text_token == -1); + CHECK(probe.error_calls == 1); +} + +TEST_CASE("speech_generator_frame_handles_unexpected_external_events") { + fixture test{}; + emel::error::type err = 0; + + REQUIRE(test.machine.process_event(unexpected_request{err})); + CHECK(err == error_code(frame::error::unexpected_event)); + CHECK(test.machine.is(sml::state)); +} + +TEST_CASE( + "speech_generator_frame_trace_is_deterministic_and_statically_bounded") { + sml_trace logger{}; + fixture_base test{logger}; + const std::array encoded_tokens{11, 12}; + std::array generated_tokens{}; + int32_t text_token = -1; + emel::error::type err = 0; + const frame::event::run request{encoded_tokens, generated_tokens, text_token, + err}; + const std::array expected_phase_trace{ + phase_id::tokenize, phase_id::plan, phase_id::predict, + phase_id::graph, phase_id::sample, phase_id::detokenize}; + + logger.reset(); + test.phases.reset(); + REQUIRE(test.machine.process_event(request)); + const auto first_sml_trace = logger.entries; + const size_t first_sml_trace_size = logger.size; + const auto first_phase_trace = test.phases.entries; + const size_t first_phase_trace_size = test.phases.size; + + CHECK(logger.transitions == MAX_FRAME_TRANSITIONS); + CHECK(logger.transitions <= MAX_FRAME_TRANSITIONS); + CHECK_FALSE(logger.overflow); + CHECK(test.phases.size == static_cast(MAX_CROSS_ACTOR_CALLS)); + CHECK_FALSE(test.phases.overflow); + CHECK(test.phases.entries == expected_phase_trace); + + logger.reset(); + test.phases.reset(); + REQUIRE(test.machine.process_event(request)); + + CHECK(logger.entries == first_sml_trace); + CHECK(logger.size == first_sml_trace_size); + CHECK(logger.transitions == MAX_FRAME_TRANSITIONS); + CHECK_FALSE(logger.overflow); + CHECK(test.phases.entries == first_phase_trace); + CHECK(test.phases.size == first_phase_trace_size); + CHECK_FALSE(test.phases.overflow); + CHECK(test.machine.is(sml::state)); +} + +TEST_CASE("speech_generator_frame_meets_the_fake_dispatch_time_budget") { + fixture test{}; + const std::array encoded_tokens{11, 12}; + std::array generated_tokens{}; + int32_t text_token = -1; + emel::error::type err = 0; + const frame::event::run request{encoded_tokens, generated_tokens, text_token, + err}; + std::chrono::steady_clock::duration maximum_elapsed{}; + + for (int32_t iteration = 0; iteration < 64; ++iteration) { + const auto start = std::chrono::steady_clock::now(); + const bool accepted = test.machine.process_event(request); + const auto elapsed = std::chrono::steady_clock::now() - start; + REQUIRE(accepted); + maximum_elapsed = std::max(maximum_elapsed, elapsed); + } + + CHECK(maximum_elapsed < FAKE_DISPATCH_BUDGET); + CHECK(test.machine.is(sml::state)); +} + +TEST_CASE("speech_generator_frame_repeated_dispatch_is_allocation_free") { + fixture test{}; + const std::array encoded_tokens{11, 12}; + std::array generated_tokens{}; + int32_t text_token = -1; + emel::error::type err = 0; + const frame::event::run request{encoded_tokens, generated_tokens, text_token, + err}; + REQUIRE(test.machine.process_event(request)); + + bool accepted = true; + size_t allocation_count = 0; + { + emel::test::allocation::allocation_scope allocations; + for (int32_t iteration = 0; iteration < 64; ++iteration) { + accepted = test.machine.process_event(request) && accepted; + } + allocation_count = allocations.allocations(); + } + + CHECK(accepted); + CHECK(allocation_count == 0u); + CHECK(test.machine.is(sml::state)); + CHECK(text_token == 42); +} diff --git a/tests/speech/generator/wavefront_lifecycle_tests.cpp b/tests/speech/generator/wavefront_lifecycle_tests.cpp new file mode 100644 index 00000000..434bce92 --- /dev/null +++ b/tests/speech/generator/wavefront_lifecycle_tests.cpp @@ -0,0 +1,1376 @@ +#include "doctest/doctest.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "../../allocation_tracker.hpp" +#include "emel/batch/planner/events.hpp" +#include "emel/speech/generator/frame/sm.hpp" +#include "emel/speech/generator/sm.hpp" + +namespace { + +namespace frame = emel::speech::generator::frame; +namespace generator = emel::speech::generator; +namespace sml = stateforward::sml; + +constexpr size_t FRAME_SAMPLES = 4u; +constexpr size_t CODEBOOKS = 2u; +constexpr int32_t MAX_WAVEFRONT_TRANSITIONS = 3; +constexpr auto FAKE_WAVEFRONT_DISPATCH_BUDGET = std::chrono::milliseconds{100}; + +struct fake_workspace {}; + +struct fake_tokenize { + std::span audio_tokens; + std::span model_tokens_out; + int32_t &error_out; +}; + +struct fake_predict { + fake_predict(std::span tokens_ref, + fake_workspace &workspace_ref, int32_t step_ref, + int32_t outputs_ref) noexcept + : tokens(tokens_ref), workspace(workspace_ref), step(step_ref), + outputs(outputs_ref) {} + std::span tokens; + fake_workspace &workspace; + int32_t step; + int32_t outputs; + emel::error::type *error_out = nullptr; +}; + +struct fake_graph { + fake_graph(fake_workspace &workspace_ref, + std::span tokens_ref) noexcept + : workspace(workspace_ref), tokens(tokens_ref) {} + fake_workspace &workspace; + std::span tokens; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; +}; + +struct fake_sample { + fake_sample(fake_workspace &workspace_ref, + std::span tokens_ref, + std::span tokens_out_ref, + int32_t &text_token_out_ref) noexcept + : workspace(workspace_ref), tokens(tokens_ref), + tokens_out(tokens_out_ref), text_token_out(text_token_out_ref) {} + fake_workspace &workspace; + std::span tokens; + std::span tokens_out; + int32_t &text_token_out; + emel::error::type *error_out = nullptr; + emel::error::type *graph_error_out = nullptr; +}; + +struct fake_detokenize { + int32_t text_token; + std::span audio_tokens; + int32_t &text_token_out; + std::span audio_tokens_out; + bool &produced_out; + int32_t &error_out; +}; + +struct overlap_probe { + std::atomic active_workers = 0; + std::atomic expected_workers = 0; + std::atomic hold_workers = false; + std::atomic release_workers = false; + std::atomic overlap_observed = false; +}; + +struct sml_trace { + std::array entries{}; + size_t size = 0u; + int32_t transitions = 0; + bool overflow = false; + + void record(const uint8_t value) noexcept { + if (size < entries.size()) { + entries[size++] = value; + } else { + overflow = true; + } + } + + void reset() noexcept { + entries.fill(0u); + size = 0u; + transitions = 0; + overflow = false; + } + + template + void log_process_event(const event_type &) noexcept { + record(1u); + } + + template + void log_guard(const guard_type &, const event_type &, + const bool accepted) noexcept { + record(accepted ? 3u : 2u); + } + + template + void log_action(const action_type &, const event_type &) noexcept { + record(4u); + } + + template + void log_state_change(const source_state &, + const destination_state &) noexcept { + ++transitions; + record(5u); + } +}; + +struct fake_tokenizer_actor { + overlap_probe *overlap = nullptr; + bool accepted = true; + bool produce = true; + int32_t error = 0; + + bool process_event(const fake_tokenize &request) noexcept { + while (overlap->hold_workers.load(std::memory_order_acquire) && + overlap->active_workers.load(std::memory_order_acquire) < + overlap->expected_workers.load(std::memory_order_acquire)) { + emel::policy::cpu_relax(); + } + overlap->overlap_observed.store( + overlap->hold_workers.load(std::memory_order_acquire), + std::memory_order_release); + overlap->release_workers.store(true, std::memory_order_release); + std::copy(request.audio_tokens.begin(), request.audio_tokens.end(), + request.model_tokens_out.begin()); + request.error_out = error; + return accepted; + } + + bool process_event(const fake_detokenize &request) noexcept { + std::copy(request.audio_tokens.begin(), request.audio_tokens.end(), + request.audio_tokens_out.begin()); + request.text_token_out = request.text_token; + request.produced_out = produce; + request.error_out = error; + return accepted; + } +}; + +struct fake_planner_actor { + bool accepted = true; + + bool process_event( + const emel::batch::planner::event::plan_request &request) noexcept { + const std::array steps{1}; + request.on_done(emel::batch::planner::events::plan_done{ + .request = &request, + .step_sizes = steps.data(), + .step_count = 1, + .total_outputs = 1, + }); + return accepted; + } +}; + +struct fake_predictor_actor { + bool accepted = true; + emel::error::type error = 0; + bool process_event(const fake_predict &request) noexcept { + *request.error_out = error; + return accepted; + } +}; + +struct fake_graph_actor { + bool accepted = true; + bool process_event(const fake_graph &request) noexcept { + *request.error_out = 0; + *request.graph_error_out = 0; + return accepted; + } +}; + +struct fake_sampler_actor { + bool accepted = true; + bool process_event(const fake_sample &request) noexcept { + for (size_t lane = 0u; lane < request.tokens_out.size(); ++lane) { + request.tokens_out[lane] = request.tokens[lane] + 100; + } + request.text_token_out = request.tokens[0] + 1000; + *request.error_out = 0; + *request.graph_error_out = 0; + return accepted; + } +}; + +struct frame_dependencies { + using tokenize_event = fake_tokenize; + using predict_event = fake_predict; + using graph_event = fake_graph; + using sample_event = fake_sample; + using detokenize_event = fake_detokenize; + + fake_planner_actor &planner; + fake_tokenizer_actor &tokenizer; + fake_predictor_actor &predictor; + fake_graph_actor &graph; + fake_sampler_actor &sampler; + fake_workspace &prediction_workspace; + std::span model_codes; + std::span predicted_codes; + int32_t codebook_count; + emel::batch::planner::event::plan_mode frame_plan_mode; + int32_t frame_plan_steps; + int32_t frame_plan_token_count; + bool frame_plan_output_all; +}; + +struct fake_encode { + fake_encode(std::span pcm_ref, + std::span tokens_ref) noexcept + : pcm(pcm_ref), tokens(tokens_ref) {} + std::span pcm; + std::span tokens; + emel::error::type *error_out = nullptr; +}; + +struct fake_decode { + fake_decode(std::span tokens_ref, + std::span pcm_ref) noexcept + : tokens(tokens_ref), pcm(pcm_ref) {} + std::span tokens; + std::span pcm; + emel::error::type *error_out = nullptr; +}; + +struct fake_reset { + explicit fake_reset(emel::error::type &error_out_ref) noexcept + : error_out(error_out_ref) {} + + emel::error::type &error_out; +}; + +struct fake_encoder_actor { + overlap_probe *overlap = nullptr; + bool accepted = true; + emel::error::type error = 0; + bool reset_accepted = true; + emel::error::type reset_error = 0; + int32_t reset_calls = 0; + bool processed_since_reset = false; + + bool process_event(const fake_encode &request) noexcept { + processed_since_reset = true; + overlap->active_workers.fetch_add(1, std::memory_order_acq_rel); + while (overlap->hold_workers.load(std::memory_order_acquire) && + !overlap->release_workers.load(std::memory_order_acquire)) { + emel::policy::cpu_relax(); + } + for (size_t lane = 0u; lane < request.tokens.size(); ++lane) { + request.tokens[lane] = + static_cast(request.pcm[0]) + static_cast(lane); + } + *request.error_out = error; + overlap->active_workers.fetch_sub(1, std::memory_order_acq_rel); + return accepted; + } + + bool process_event(const fake_reset &request) noexcept { + ++reset_calls; + processed_since_reset = false; + request.error_out = reset_error; + return reset_accepted; + } +}; + +struct fake_decoder_actor { + overlap_probe *overlap = nullptr; + bool accepted = true; + emel::error::type error = 0; + bool reset_accepted = true; + emel::error::type reset_error = 0; + int32_t reset_calls = 0; + bool processed_since_reset = false; + + bool process_event(const fake_decode &request) noexcept { + processed_since_reset = true; + overlap->active_workers.fetch_add(1, std::memory_order_acq_rel); + while (overlap->hold_workers.load(std::memory_order_acquire) && + !overlap->release_workers.load(std::memory_order_acquire)) { + emel::policy::cpu_relax(); + } + std::fill(request.pcm.begin(), request.pcm.end(), + static_cast(request.tokens[0])); + *request.error_out = error; + overlap->active_workers.fetch_sub(1, std::memory_order_acq_rel); + return accepted; + } + + bool process_event(const fake_reset &request) noexcept { + ++reset_calls; + processed_since_reset = false; + request.error_out = reset_error; + return reset_accepted; + } +}; + +struct wavefront_dependencies { + using generator_mode = generator::action::mode::wavefront; + using wavefront_encode_event = fake_encode; + using wavefront_encode_reset_event = fake_reset; + using wavefront_middle_reset_event = frame::event::reset; + using wavefront_decode_event = fake_decode; + using wavefront_decode_reset_event = fake_reset; + static constexpr size_t wavefront_frame_capacity = FRAME_SAMPLES; + static constexpr size_t wavefront_codebook_capacity = CODEBOOKS; + + fake_encoder_actor &wavefront_encoder; + frame::sm &wavefront_middle; + fake_decoder_actor &wavefront_decoder; + generator::action::wavefront_stage_pool *stage_pool; + generator::action::wavefront_diagnostics &stage_diagnostics; + generator::action::wavefront_stage_mode stage_mode; + int32_t frame_samples; + int32_t codebook_count; +}; + +struct fixture { + overlap_probe overlap{}; + std::array model_codes{}; + std::array predicted_codes{}; + fake_planner_actor planner{}; + fake_tokenizer_actor tokenizer{.overlap = &overlap}; + fake_predictor_actor predictor{}; + fake_graph_actor graph{}; + fake_sampler_actor sampler{}; + fake_workspace workspace{}; + frame_dependencies middle_dependencies{ + .planner = planner, + .tokenizer = tokenizer, + .predictor = predictor, + .graph = graph, + .sampler = sampler, + .prediction_workspace = workspace, + .model_codes = model_codes, + .predicted_codes = predicted_codes, + .codebook_count = static_cast(CODEBOOKS), + .frame_plan_mode = emel::batch::planner::event::plan_mode::simple, + .frame_plan_steps = 1, + .frame_plan_token_count = 1, + .frame_plan_output_all = true, + }; + frame::sm middle{middle_dependencies}; + fake_encoder_actor encoder{.overlap = &overlap}; + fake_decoder_actor decoder{.overlap = &overlap}; + generator::action::wavefront_stage_pool pool{}; + generator::action::wavefront_diagnostics diagnostics{}; + wavefront_dependencies dependencies; + generator::sm machine; + int32_t last_text_token = -1; + int32_t last_sample_count = 0; + + explicit fixture( + const int32_t frame_samples = static_cast(FRAME_SAMPLES), + const int32_t codebook_count = static_cast(CODEBOOKS), + const generator::action::wavefront_stage_mode stage_mode = + generator::action::wavefront_stage_mode::parallel) + : dependencies{.wavefront_encoder = encoder, + .wavefront_middle = middle, + .wavefront_decoder = decoder, + .stage_pool = &pool, + .stage_diagnostics = diagnostics, + .stage_mode = stage_mode, + .frame_samples = frame_samples, + .codebook_count = codebook_count}, + machine{dependencies} {} + + bool + push(const uint64_t sequence, bool &produced, + generator::event::wavefront_attribution &output, + std::array &pcm_out, + std::array &tokens_out, emel::error::type &err, + emel::callback + on_done = {}) { + std::array pcm_in{}; + pcm_in.fill(static_cast(sequence + 1u)); + last_text_token = -1; + last_sample_count = 0; + generator::event::wavefront_frame request{ + pcm_in, + pcm_out, + tokens_out, + {.sequence = sequence, .source = 99u}, + output, + last_text_token, + last_sample_count, + produced, + err}; + request.on_done = on_done; + return machine.process_event(request); + } + + bool + flush(bool &produced, bool &complete, + generator::event::wavefront_attribution &output, + std::array &pcm_out, + std::array &tokens_out, emel::error::type &err, + emel::callback + on_done = {}) { + last_text_token = -1; + last_sample_count = 0; + generator::event::wavefront_flush request{ + pcm_out, tokens_out, output, last_text_token, + last_sample_count, produced, complete, err}; + request.on_done = on_done; + return machine.process_event(request); + } +}; + +struct callback_probe { + bool caller_active = false; + int32_t calls = 0; + bool all_calls_inside_dispatch = true; + emel::error::type last_error = 0u; + + void done(const generator::events::wavefront_frame_done &) noexcept { + ++calls; + all_calls_inside_dispatch = all_calls_inside_dispatch && caller_active; + } + + void flush_done(const generator::events::wavefront_flush_done &) noexcept { + ++calls; + all_calls_inside_dispatch = all_calls_inside_dispatch && caller_active; + } + + void + frame_error(const generator::events::wavefront_frame_error &ev) noexcept { + ++calls; + all_calls_inside_dispatch = all_calls_inside_dispatch && caller_active; + last_error = ev.err; + } + + void + flush_error(const generator::events::wavefront_flush_error &ev) noexcept { + ++calls; + all_calls_inside_dispatch = all_calls_inside_dispatch && caller_active; + last_error = ev.err; + } +}; + +template +concept accepts_internal_wavefront = requires( + machine_type &machine, const generator::detail::wavefront_frame_run &ev) { + machine.process_event(ev); +}; + +static_assert( + !accepts_internal_wavefront>); + +struct production_wavefront_dependencies { + using generator_mode = generator::action::mode::wavefront; + using wavefront_encode_event = fake_encode; + using wavefront_encode_reset_event = fake_reset; + using wavefront_middle_reset_event = frame::event::reset; + using wavefront_decode_event = fake_decode; + using wavefront_decode_reset_event = fake_reset; + static constexpr size_t wavefront_frame_capacity = FRAME_SAMPLES; + static constexpr size_t wavefront_codebook_capacity = CODEBOOKS; + + fake_encoder_actor &wavefront_encoder; + frame::sm &wavefront_middle; + fake_decoder_actor &wavefront_decoder; + generator::action::wavefront_stage_pool *stage_pool; + generator::action::wavefront_stage_mode stage_mode; + int32_t frame_samples; + int32_t codebook_count; +}; + +static_assert(generator::action::wavefront_dependencies< + production_wavefront_dependencies>); + +template +concept has_wavefront_diagnostics = + requires(dependencies_type &deps) { deps.stage_diagnostics; }; + +static_assert(!has_wavefront_diagnostics); + +emel::error::type error_code(const generator::error value) noexcept { + return generator::action::error_code(value); +} + +emel::error::type combined_error(const generator::error value, + const generator::error qualifier) noexcept { + return error_code(value) | error_code(qualifier); +} + +struct wavefront_io { + std::array pcm{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + bool complete = false; + emel::error::type err = 0u; +}; + +void prime_frames(fixture &test, wavefront_io &io, const uint64_t frame_count) { + for (uint64_t sequence = 0u; sequence < frame_count; ++sequence) { + REQUIRE( + test.push(sequence, io.produced, io.output, io.pcm, io.tokens, io.err)); + } +} + +} // namespace + +TEST_CASE( + "speech_generator_wavefront_models_zero_through_many_frame_lifecycle") { + SUBCASE("zero frames completes without output") { + fixture test{}; + std::array pcm{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = true; + bool complete = false; + emel::error::type err = -1; + REQUIRE(test.flush(produced, complete, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK(complete); + CHECK(test.machine.is(sml::state)); + } + + for (const uint64_t frame_total : {1u, 2u, 3u, 9u}) { + CAPTURE(frame_total); + fixture test{}; + std::array pcm{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + emel::error::type err = -1; + uint64_t output_sequence = 0u; + for (uint64_t sequence = 0u; sequence < frame_total; ++sequence) { + bool produced = true; + REQUIRE(test.push(sequence, produced, output, pcm, tokens, err)); + CHECK(produced == (sequence >= 2u)); + if (produced) { + CHECK(output.sequence == output_sequence++); + CHECK(output.source == 99u); + CHECK(pcm[0] == + doctest::Approx(static_cast(output.sequence + 101u))); + CHECK(tokens[0] == static_cast(output.sequence + 101u)); + CHECK(tokens[1] == static_cast(output.sequence + 102u)); + CHECK(test.last_text_token == + static_cast(output.sequence + 1001u)); + CHECK(test.last_sample_count == static_cast(FRAME_SAMPLES)); + } + } + + bool complete = false; + while (!complete) { + bool produced = false; + REQUIRE(test.flush(produced, complete, output, pcm, tokens, err)); + if (produced) { + CHECK(output.sequence == output_sequence++); + CHECK(output.source == 99u); + CHECK(pcm[0] == + doctest::Approx(static_cast(output.sequence + 101u))); + CHECK(tokens[0] == static_cast(output.sequence + 101u)); + CHECK(tokens[1] == static_cast(output.sequence + 102u)); + CHECK(test.last_text_token == + static_cast(output.sequence + 1001u)); + CHECK(test.last_sample_count == static_cast(FRAME_SAMPLES)); + } + } + CHECK(output_sequence == frame_total); + CHECK(test.diagnostics.submissions.load() == test.diagnostics.joins.load()); + } +} + +TEST_CASE("speech_generator_wavefront_emits_done_callbacks_for_each_public_" + "lifecycle_phase") { + for (const uint64_t frame_count : {0u, 1u, 2u, 3u}) { + CAPTURE(frame_count); + fixture test{}; + wavefront_io io{}; + callback_probe probe{}; + const auto frame_done = + emel::callback:: + from(&probe); + const auto flush_done = + emel::callback:: + from(&probe); + + probe.caller_active = true; + for (uint64_t sequence = 0u; sequence < frame_count; ++sequence) { + REQUIRE(test.push(sequence, io.produced, io.output, io.pcm, io.tokens, + io.err, frame_done)); + } + while (!io.complete) { + REQUIRE(test.flush(io.produced, io.complete, io.output, io.pcm, io.tokens, + io.err, flush_done)); + } + probe.caller_active = false; + + CHECK(probe.calls > 0); + CHECK(probe.all_calls_inside_dispatch); + CHECK(test.machine.is(sml::state)); + } +} + +TEST_CASE("speech_generator_wavefront_rejects_each_invalid_public_request_" + "shape") { + const auto check_invalid_frame = [](fixture &test, + std::span pcm_in, + std::span pcm_out, + std::span tokens_out, + const uint64_t sequence) { + generator::event::wavefront_attribution output{}; + int32_t text_token = 7; + int32_t samples = 7; + bool produced = true; + emel::error::type err = 0u; + const generator::event::wavefront_frame request{ + pcm_in, pcm_out, tokens_out, {.sequence = sequence, .source = 99u}, + output, text_token, samples, produced, + err}; + + CHECK_FALSE(test.machine.process_event(request)); + CHECK_FALSE(produced); + CHECK(err == error_code(generator::error::invalid_request)); + }; + + const auto check_invalid_flush = [](fixture &test, std::span pcm_out, + std::span tokens_out) { + generator::event::wavefront_attribution output{}; + int32_t text_token = 7; + int32_t samples = 7; + bool produced = true; + bool complete = true; + emel::error::type err = 0u; + const generator::event::wavefront_flush request{ + pcm_out, tokens_out, output, text_token, + samples, produced, complete, err}; + + CHECK_FALSE(test.machine.process_event(request)); + CHECK_FALSE(produced); + CHECK_FALSE(complete); + CHECK(err == error_code(generator::error::invalid_request)); + }; + + std::array pcm{}; + std::array tokens{}; + SUBCASE("invalid persistent dimensions") { + fixture zero_samples{0, static_cast(CODEBOOKS)}; + check_invalid_frame(zero_samples, pcm, pcm, tokens, 0u); + + fixture zero_codebooks{static_cast(FRAME_SAMPLES), 0}; + check_invalid_frame(zero_codebooks, pcm, pcm, tokens, 0u); + + fixture excess_samples{static_cast(FRAME_SAMPLES + 1u), + static_cast(CODEBOOKS)}; + check_invalid_frame(excess_samples, pcm, pcm, tokens, 0u); + + fixture excess_codebooks{static_cast(FRAME_SAMPLES), + static_cast(CODEBOOKS + 1u)}; + check_invalid_frame(excess_codebooks, pcm, pcm, tokens, 0u); + } + + SUBCASE("invalid frame spans and attribution") { + fixture narrow_input{}; + check_invalid_frame(narrow_input, + std::span{pcm}.first(FRAME_SAMPLES - 1u), + pcm, tokens, 0u); + + fixture narrow_output{}; + check_invalid_frame(narrow_output, pcm, + std::span{pcm}.first(FRAME_SAMPLES - 1u), tokens, + 0u); + + fixture narrow_tokens{}; + check_invalid_frame(narrow_tokens, pcm, pcm, + std::span{tokens}.first(CODEBOOKS - 1u), 0u); + + fixture invalid_sequence{}; + check_invalid_frame(invalid_sequence, pcm, pcm, tokens, + generator::event::k_invalid_wavefront_sequence); + + fixture out_of_order{}; + check_invalid_frame(out_of_order, pcm, pcm, tokens, 1u); + } + + SUBCASE("invalid flush spans") { + fixture narrow_output{}; + check_invalid_flush( + narrow_output, std::span{pcm}.first(FRAME_SAMPLES - 1u), tokens); + + fixture narrow_tokens{}; + check_invalid_flush(narrow_tokens, pcm, + std::span{tokens}.first(CODEBOOKS - 1u)); + } +} + +TEST_CASE( + "speech_generator_wavefront_observes_three_stage_overlap_and_callbacks") { + fixture test{}; + std::array pcm{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + emel::error::type err = -1; + REQUIRE(test.push(0u, produced, output, pcm, tokens, err)); + REQUIRE(test.push(1u, produced, output, pcm, tokens, err)); + + test.overlap.expected_workers.store(2); + test.overlap.release_workers.store(false); + test.overlap.hold_workers.store(true); + callback_probe probe{}; + std::array pcm_in{}; + pcm_in.fill(3.0f); + int32_t text_token = -1; + int32_t samples = 0; + generator::event::wavefront_frame request{ + pcm_in, pcm, tokens, {.sequence = 2u, .source = 99u}, + output, text_token, samples, produced, + err}; + request.on_done = + decltype(request.on_done)::from( + &probe); + probe.caller_active = true; + REQUIRE(test.machine.process_event(request)); + probe.caller_active = false; + + CHECK(test.overlap.overlap_observed.load()); + CHECK(produced); + CHECK(output.sequence == 0u); + CHECK(probe.calls == 1); + CHECK(probe.all_calls_inside_dispatch); + CHECK(test.diagnostics.worker_entries.load() == + test.diagnostics.worker_exits.load()); + CHECK(test.diagnostics.submissions.load() == test.diagnostics.joins.load()); +} + +TEST_CASE("speech_generator_wavefront_serial_stage_mode_uses_no_workers") { + fixture test{static_cast(FRAME_SAMPLES), + static_cast(CODEBOOKS), + generator::action::wavefront_stage_mode::serial}; + wavefront_io io{}; + REQUIRE(test.push(0u, io.produced, io.output, io.pcm, io.tokens, io.err)); + REQUIRE(test.push(1u, io.produced, io.output, io.pcm, io.tokens, io.err)); + REQUIRE(test.push(2u, io.produced, io.output, io.pcm, io.tokens, io.err)); + CHECK(test.diagnostics.submissions.load() == 0u); + CHECK(test.diagnostics.joins.load() == 0u); + CHECK(test.diagnostics.worker_entries.load() == 0u); + CHECK(test.diagnostics.worker_exits.load() == 0u); + + REQUIRE( + test.machine.process_event(generator::event::wavefront_reset{io.err})); + CHECK(test.diagnostics.submissions.load() == 0u); + CHECK(test.diagnostics.joins.load() == 0u); + CHECK(test.diagnostics.worker_entries.load() == 0u); + CHECK(test.diagnostics.worker_exits.load() == 0u); +} + +TEST_CASE( + "speech_generator_wavefront_drains_partial_submit_and_phase_failures") { + SUBCASE("encode failure publishes no output") { + fixture test{}; + test.encoder.error = 7; + std::array pcm{}; + pcm.fill(-9.0f); + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = true; + emel::error::type err = 0; + CHECK_FALSE(test.push(0u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK(pcm[0] == -9.0f); + CHECK(err == error_code(generator::error::encode_failed)); + } + + SUBCASE("middle and decode failures publish no output") { + fixture middle_failure{}; + std::array pcm{}; + pcm.fill(-9.0f); + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + emel::error::type err = 0; + REQUIRE(middle_failure.push(0u, produced, output, pcm, tokens, err)); + middle_failure.tokenizer.error = 8; + CHECK_FALSE(middle_failure.push(1u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK(pcm[0] == -9.0f); + CHECK(err == error_code(generator::error::middle_failed)); + + fixture decode_failure{}; + REQUIRE(decode_failure.push(0u, produced, output, pcm, tokens, err)); + REQUIRE(decode_failure.push(1u, produced, output, pcm, tokens, err)); + decode_failure.decoder.error = 9; + CHECK_FALSE(decode_failure.push(2u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK(pcm[0] == -9.0f); + CHECK(err == error_code(generator::error::decode_failed)); + } + + SUBCASE("partial submit joins the accepted task then fails") { + fixture test{}; + std::array pcm{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + emel::error::type err = 0; + REQUIRE(test.push(0u, produced, output, pcm, tokens, err)); + REQUIRE(test.push(1u, produced, output, pcm, tokens, err)); + + generator::action::wavefront_stage_pool::join_group blocker_group{}; + std::atomic blocker_started = false; + std::atomic release_blocker = false; + REQUIRE(test.pool.try_submit(blocker_group, [&]() noexcept { + blocker_started.store(true, std::memory_order_release); + while (!release_blocker.load(std::memory_order_acquire)) { + emel::policy::cpu_relax(); + } + })); + while (!blocker_started.load(std::memory_order_acquire)) { + emel::policy::cpu_relax(); + } + const uint64_t submissions_before = test.diagnostics.submissions.load(); + const uint64_t joins_before = test.diagnostics.joins.load(); + CHECK_FALSE(test.push(2u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK(err == error_code(generator::error::stage_submit_failed)); + CHECK(test.diagnostics.submissions.load() - submissions_before == 1u); + CHECK(test.diagnostics.joins.load() - joins_before == 1u); + release_blocker.store(true, std::memory_order_release); + CHECK(blocker_group.wait()); + } +} + +TEST_CASE( + "speech_generator_wavefront_preserves_failure_provenance_across_drains") { + SUBCASE("child rejection and middle non-production are distinct") { + fixture encode_rejection{}; + encode_rejection.encoder.accepted = false; + std::array pcm{}; + pcm.fill(-9.0f); + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = true; + emel::error::type err = 0u; + CHECK_FALSE(encode_rejection.push(0u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK(err == combined_error(generator::error::encode_failed, + generator::error::unsupported_request)); + CHECK(pcm[0] == -9.0f); + + fixture non_production{}; + REQUIRE(non_production.push(0u, produced, output, pcm, tokens, err)); + non_production.tokenizer.produce = false; + CHECK_FALSE(non_production.push(1u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK(err == error_code(generator::error::non_production_failed)); + CHECK(pcm[0] == -9.0f); + } + + SUBCASE("middle and decode drain failures carry the drain qualifier") { + fixture middle_drain{}; + std::array pcm{}; + pcm.fill(-9.0f); + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + bool complete = false; + emel::error::type err = 0u; + REQUIRE(middle_drain.push(0u, produced, output, pcm, tokens, err)); + middle_drain.tokenizer.error = 7; + CHECK_FALSE( + middle_drain.flush(produced, complete, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK_FALSE(complete); + CHECK(err == combined_error(generator::error::middle_failed, + generator::error::drain_failed)); + CHECK(pcm[0] == -9.0f); + + fixture final_decode{}; + REQUIRE(final_decode.push(0u, produced, output, pcm, tokens, err)); + REQUIRE(final_decode.flush(produced, complete, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK_FALSE(complete); + final_decode.decoder.error = 9; + CHECK_FALSE( + final_decode.flush(produced, complete, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK_FALSE(complete); + CHECK(err == combined_error(generator::error::decode_failed, + generator::error::drain_failed)); + CHECK(pcm[0] == -9.0f); + } + + SUBCASE("drain and final-decode submit failures carry the drain qualifier") { + fixture drain{}; + std::array pcm{}; + pcm.fill(-9.0f); + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + bool complete = false; + emel::error::type err = 0u; + REQUIRE(drain.push(0u, produced, output, pcm, tokens, err)); + REQUIRE(drain.push(1u, produced, output, pcm, tokens, err)); + + bool drain_accepted = true; + generator::action::wavefront_stage_pool::join_group drain_group{}; + REQUIRE(drain.pool.try_submit(drain_group, [&]() noexcept { + drain_accepted = + drain.flush(produced, complete, output, pcm, tokens, err); + })); + REQUIRE(drain_group.wait()); + CHECK_FALSE(drain_accepted); + CHECK_FALSE(produced); + CHECK_FALSE(complete); + CHECK(err == combined_error(generator::error::stage_submit_failed, + generator::error::drain_failed)); + CHECK(pcm[0] == -9.0f); + + fixture final_decode{}; + REQUIRE(final_decode.push(0u, produced, output, pcm, tokens, err)); + REQUIRE(final_decode.flush(produced, complete, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK_FALSE(complete); + + bool final_accepted = true; + generator::action::wavefront_stage_pool::join_group final_group{}; + REQUIRE(final_decode.pool.try_submit(final_group, [&]() noexcept { + final_accepted = + final_decode.flush(produced, complete, output, pcm, tokens, err); + })); + REQUIRE(final_group.wait()); + CHECK_FALSE(final_accepted); + CHECK_FALSE(produced); + CHECK_FALSE(complete); + CHECK(err == combined_error(generator::error::stage_submit_failed, + generator::error::drain_failed)); + CHECK(pcm[0] == -9.0f); + } +} + +TEST_CASE("speech_generator_wavefront_classifies_child_failures_in_each_" + "active_frame_phase") { + const auto check_encode_rejection = [](const uint64_t primed_frames) { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, primed_frames); + test.encoder.accepted = false; + + CHECK_FALSE(test.push(primed_frames, io.produced, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK(io.err == combined_error(generator::error::encode_failed, + generator::error::unsupported_request)); + }; + const auto check_encode_failure = [](const uint64_t primed_frames) { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, primed_frames); + test.encoder.error = 17u; + + CHECK_FALSE(test.push(primed_frames, io.produced, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK(io.err == error_code(generator::error::encode_failed)); + }; + const auto check_middle_failure = [](const uint64_t primed_frames) { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, primed_frames); + test.tokenizer.error = 19u; + + CHECK_FALSE(test.push(primed_frames, io.produced, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK(io.err == error_code(generator::error::middle_failed)); + }; + const auto check_decode_rejection = [](const uint64_t primed_frames) { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, primed_frames); + test.decoder.accepted = false; + + CHECK_FALSE(test.push(primed_frames, io.produced, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK(io.err == combined_error(generator::error::decode_failed, + generator::error::unsupported_request)); + }; + const auto check_decode_failure = [](const uint64_t primed_frames) { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, primed_frames); + test.decoder.error = 23u; + + CHECK_FALSE(test.push(primed_frames, io.produced, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK(io.err == error_code(generator::error::decode_failed)); + }; + + SUBCASE("fill one") { + check_encode_rejection(1u); + check_encode_failure(1u); + } + SUBCASE("steady even") { + check_encode_rejection(2u); + check_encode_failure(2u); + check_middle_failure(2u); + check_decode_rejection(2u); + check_decode_failure(2u); + } + SUBCASE("steady odd") { + check_encode_rejection(3u); + check_encode_failure(3u); + check_middle_failure(3u); + check_decode_rejection(3u); + check_decode_failure(3u); + } +} + +TEST_CASE("speech_generator_wavefront_classifies_child_failures_in_each_" + "drain_phase") { + const auto check_middle_failure = [](const uint64_t primed_frames) { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, primed_frames); + test.tokenizer.error = 29u; + + CHECK_FALSE(test.flush(io.produced, io.complete, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK_FALSE(io.complete); + CHECK(io.err == combined_error(generator::error::middle_failed, + generator::error::drain_failed)); + }; + const auto check_decode_rejection = [](const uint64_t primed_frames) { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, primed_frames); + test.decoder.accepted = false; + + CHECK_FALSE(test.flush(io.produced, io.complete, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK_FALSE(io.complete); + CHECK(io.err == (error_code(generator::error::decode_failed) | + error_code(generator::error::unsupported_request) | + error_code(generator::error::drain_failed))); + }; + const auto check_decode_failure = [](const uint64_t primed_frames) { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, primed_frames); + test.decoder.error = 31u; + + CHECK_FALSE(test.flush(io.produced, io.complete, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK_FALSE(io.complete); + CHECK(io.err == combined_error(generator::error::decode_failed, + generator::error::drain_failed)); + }; + + SUBCASE("drain model") { check_middle_failure(1u); } + SUBCASE("drain even") { + check_middle_failure(2u); + check_decode_rejection(2u); + check_decode_failure(2u); + } + SUBCASE("drain odd") { + check_middle_failure(3u); + check_decode_rejection(3u); + check_decode_failure(3u); + } + + SUBCASE("final decode lane zero") { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, 1u); + REQUIRE(test.flush(io.produced, io.complete, io.output, io.pcm, io.tokens, + io.err)); + REQUIRE(test.machine.is( + sml::state)); + test.decoder.accepted = false; + CHECK_FALSE(test.flush(io.produced, io.complete, io.output, io.pcm, + io.tokens, io.err)); + CHECK(io.err == (error_code(generator::error::decode_failed) | + error_code(generator::error::unsupported_request) | + error_code(generator::error::drain_failed))); + } + + SUBCASE("final decode lane one") { + fixture test{}; + wavefront_io io{}; + prime_frames(test, io, 2u); + REQUIRE(test.flush(io.produced, io.complete, io.output, io.pcm, io.tokens, + io.err)); + REQUIRE(test.machine.is( + sml::state)); + test.decoder.error = 37u; + CHECK_FALSE(test.flush(io.produced, io.complete, io.output, io.pcm, + io.tokens, io.err)); + CHECK(io.err == combined_error(generator::error::decode_failed, + generator::error::drain_failed)); + } +} + +TEST_CASE("speech_generator_wavefront_rejects_wrong_lifecycle_with_immediate_" + "callbacks") { + fixture test{}; + std::array pcm{}; + pcm.fill(-9.0f); + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = true; + bool complete = false; + emel::error::type err = 0u; + REQUIRE(test.flush(produced, complete, output, pcm, tokens, err)); + REQUIRE(complete); + + std::array pcm_in{}; + int32_t text_token = -1; + int32_t samples = 0; + callback_probe probe{}; + generator::event::wavefront_frame frame_request{ + pcm_in, pcm, tokens, {.sequence = 0u, .source = 99u}, + output, text_token, samples, produced, + err}; + frame_request.on_error = decltype(frame_request.on_error)::from< + callback_probe, &callback_probe::frame_error>(&probe); + probe.caller_active = true; + CHECK_FALSE(test.machine.process_event(frame_request)); + probe.caller_active = false; + CHECK_FALSE(produced); + CHECK(err == error_code(generator::error::unsupported_request)); + CHECK(probe.calls == 1); + CHECK(probe.all_calls_inside_dispatch); + CHECK(probe.last_error == err); + CHECK(test.machine.is(sml::state)); + + generator::event::wavefront_flush flush_request{ + pcm, tokens, output, text_token, samples, produced, complete, err}; + flush_request.on_error = decltype(flush_request.on_error)::from< + callback_probe, &callback_probe::flush_error>(&probe); + probe.caller_active = true; + CHECK_FALSE(test.machine.process_event(flush_request)); + probe.caller_active = false; + CHECK_FALSE(produced); + CHECK_FALSE(complete); + CHECK(err == error_code(generator::error::unsupported_request)); + CHECK(probe.calls == 2); + CHECK(probe.all_calls_inside_dispatch); + + REQUIRE(test.machine.process_event(generator::event::wavefront_reset{err})); + CHECK(test.machine.is(sml::state)); + REQUIRE(test.push(0u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + REQUIRE(test.flush(produced, complete, output, pcm, tokens, err)); + CHECK(test.machine.is( + sml::state)); + CHECK_FALSE(test.push(1u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + CHECK(err == error_code(generator::error::unsupported_request)); +} + +TEST_CASE("speech_generator_wavefront_detects_corrupt_lane_attribution") { + fixture test{}; + generator::action::context ctx{test.dependencies}; + ctx.encoded_lane0_attribution = {.sequence = 7u, .source = 99u}; + generator::detail::wavefront_run_ctx run_ctx{}; + run_ctx.all_submitted = true; + run_ctx.joined = true; + run_ctx.encode_accepted = true; + std::array pcm_in{}; + std::array pcm_out{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + int32_t text_token = -1; + int32_t samples = 0; + bool produced = false; + emel::error::type err = 0u; + const generator::event::wavefront_frame request{ + pcm_in, pcm_out, tokens, {.sequence = 0u, .source = 99u}, + output, text_token, samples, produced, + err}; + const generator::detail::wavefront_frame_run runtime_ev{request, run_ctx}; + using phase_fill0 = generator::guard::guard_wavefront_phase_succeeded< + wavefront_dependencies, generator::action::lane_zero, + generator::action::lane_zero, generator::action::lane_zero, true, false, + false, false>; + CHECK(generator::guard::guard_wavefront_attribution_failed< + generator::detail::wavefront_frame_run, wavefront_dependencies, + phase_fill0>{}(runtime_ev, ctx)); +} + +TEST_CASE("speech_generator_wavefront_reset_covers_every_stage_actor") { + fixture test{}; + std::array pcm{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + emel::error::type err = 0u; + + REQUIRE(test.push(0u, produced, output, pcm, tokens, err)); + REQUIRE(test.push(1u, produced, output, pcm, tokens, err)); + CHECK(test.encoder.processed_since_reset); + + test.encoder.reset_error = 17u; + CHECK_FALSE( + test.machine.process_event(generator::event::wavefront_reset{err})); + CHECK(err == error_code(generator::error::internal_error)); + CHECK(test.encoder.reset_calls == 1); + CHECK(test.decoder.reset_calls == 1); + CHECK_FALSE(test.encoder.processed_since_reset); + CHECK_FALSE(test.decoder.processed_since_reset); + CHECK(test.machine.is(sml::state)); + + test.encoder.reset_error = 0u; + REQUIRE(test.machine.process_event(generator::event::wavefront_reset{err})); + CHECK(err == error_code(generator::error::none)); + CHECK(test.encoder.reset_calls == 2); + CHECK(test.decoder.reset_calls == 2); + CHECK(test.machine.is(sml::state)); + REQUIRE(test.push(0u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); + + emel::error::type middle_err = 9u; + REQUIRE(test.middle.process_event(frame::event::reset{middle_err})); + CHECK(middle_err == frame::action::error_code(frame::error::none)); +} + +TEST_CASE("speech_generator_wavefront_reset_classifies_each_fake_child_" + "rejection_and_failure") { + const auto check_reset_failure = [](fixture &test) { + emel::error::type err = 0u; + CHECK_FALSE( + test.machine.process_event(generator::event::wavefront_reset{err})); + CHECK(err == error_code(generator::error::internal_error)); + CHECK(test.machine.is(sml::state)); + }; + + SUBCASE("encoder rejection") { + fixture test{}; + test.encoder.reset_accepted = false; + check_reset_failure(test); + } + SUBCASE("decoder rejection") { + fixture test{}; + test.decoder.reset_accepted = false; + check_reset_failure(test); + } + SUBCASE("decoder failure") { + fixture test{}; + test.decoder.reset_error = 41u; + check_reset_failure(test); + } +} + +TEST_CASE( + "speech_generator_wavefront_reset_and_repeated_dispatch_allocate_zero") { + fixture test{}; + std::array pcm{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + emel::error::type err = 0; + + size_t allocations = 0u; + { + emel::test::allocation::allocation_scope scope; + for (uint64_t sequence = 0u; sequence < 64u; ++sequence) { + REQUIRE(test.push(sequence, produced, output, pcm, tokens, err)); + } + allocations = scope.allocations(); + } + CHECK(allocations == 0u); + CHECK(test.diagnostics.submissions.load() == test.diagnostics.joins.load()); + + REQUIRE(test.machine.process_event(generator::event::wavefront_reset{err})); + CHECK(test.encoder.reset_calls == 1); + CHECK(test.decoder.reset_calls == 1); + CHECK(test.machine.is(sml::state)); + REQUIRE(test.push(0u, produced, output, pcm, tokens, err)); + CHECK_FALSE(produced); +} + +TEST_CASE("speech_generator_wavefront_trace_is_deterministic_and_statically_" + "bounded") { + fixture test{}; + sml_trace logger{}; + using traced_machine = + emel::sm, + generator::action::context, + sml::logger>; + generator::action::context ctx{test.dependencies}; + traced_machine machine{ctx, logger}; + std::array pcm_in{}; + std::array pcm_out{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + int32_t text_token = -1; + int32_t samples = 0; + bool produced = false; + emel::error::type err = 0u; + const generator::event::wavefront_frame request{ + pcm_in, pcm_out, tokens, {.sequence = 0u, .source = 99u}, + output, text_token, samples, produced, + err}; + + logger.reset(); + generator::detail::wavefront_run_ctx first_ctx{}; + REQUIRE(machine.process_event( + generator::detail::wavefront_frame_run{request, first_ctx})); + const auto first_trace = logger.entries; + const size_t first_size = logger.size; + const int32_t first_transitions = logger.transitions; + CHECK(first_transitions <= MAX_WAVEFRONT_TRANSITIONS); + CHECK_FALSE(logger.overflow); + + const generator::event::wavefront_reset reset_request{err}; + generator::detail::wavefront_reset_ctx reset_ctx{}; + REQUIRE(machine.process_event( + generator::detail::event_wavefront_reset_run{reset_request, reset_ctx})); + logger.reset(); + generator::detail::wavefront_run_ctx second_ctx{}; + REQUIRE(machine.process_event( + generator::detail::wavefront_frame_run{request, second_ctx})); + CHECK(logger.entries == first_trace); + CHECK(logger.size == first_size); + CHECK(logger.transitions == first_transitions); + CHECK(logger.transitions <= MAX_WAVEFRONT_TRANSITIONS); + CHECK_FALSE(logger.overflow); +} + +TEST_CASE("speech_generator_wavefront_meets_the_fake_dispatch_time_budget") { + fixture test{}; + std::array pcm{}; + std::array tokens{}; + generator::event::wavefront_attribution output{}; + bool produced = false; + emel::error::type err = 0u; + std::chrono::steady_clock::duration maximum_elapsed{}; + + for (uint64_t sequence = 0u; sequence < 64u; ++sequence) { + const auto start = std::chrono::steady_clock::now(); + const bool accepted = + test.push(sequence, produced, output, pcm, tokens, err); + const auto elapsed = std::chrono::steady_clock::now() - start; + REQUIRE(accepted); + maximum_elapsed = std::max(maximum_elapsed, elapsed); + } + + CHECK(maximum_elapsed < FAKE_WAVEFRONT_DISPATCH_BUDGET); + CHECK(test.diagnostics.submissions.load() == test.diagnostics.joins.load()); +} diff --git a/tests/speech/predictor/moshi_executor_streaming_tests.cpp b/tests/speech/predictor/moshi_executor_streaming_tests.cpp index 3a05f84f..98942627 100644 --- a/tests/speech/predictor/moshi_executor_streaming_tests.cpp +++ b/tests/speech/predictor/moshi_executor_streaming_tests.cpp @@ -78,6 +78,13 @@ TEST_CASE( emel::error::cast(memory_streaming::error::none))); emel::kernel::sm kernel{}; + emel::kernel::matmul::execution_policy matmul_policy{ + .parallel_matmul_lanes = nullptr, + .kernel_kind = emel::kernel::detect_host_kind(), + .active_lanes = 1u, + .mode = emel::kernel::matmul::lane_mode::serial, + }; + emel::kernel::matmul::sm matmul{matmul_policy}; executor::sm machine{ executor::dependencies{ .kv = @@ -104,6 +111,7 @@ TEST_CASE( .depformer_positions = &depformer_positions, }, .kernel = kernel, + .matmul = matmul, .policy = executor::action::policies{ .rms_norm_epsilon = 1.0e-8f, diff --git a/tests/speech/predictor/moshi_lifecycle_tests.cpp b/tests/speech/predictor/moshi_lifecycle_tests.cpp index c93ce6cf..1c3c39a0 100644 --- a/tests/speech/predictor/moshi_lifecycle_tests.cpp +++ b/tests/speech/predictor/moshi_lifecycle_tests.cpp @@ -1,15 +1,18 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include +#include "../../allocation_tracker.hpp" #include "../../memory/recording_kv_actor.hpp" #include "emel/logits/sampler/sm.hpp" #include "emel/memory/streaming/sm.hpp" @@ -31,12 +34,39 @@ using emel::speech::predictor::moshi::test::repo_root; inline constexpr emel::error::type k_no_error = emel::error::cast(moshi::error::none); +struct serial_matmul_fixture { + emel::kernel::matmul::execution_policy policy{ + .parallel_matmul_lanes = nullptr, + .kernel_kind = emel::kernel::detect_host_kind(), + .active_lanes = 1u, + .mode = emel::kernel::matmul::lane_mode::serial, + }; + emel::kernel::matmul::sm actor{policy}; +}; + +struct parallel_matmul_fixture { + emel::kernel::matmul::lane_pool pool = {}; + emel::kernel::matmul::execution_policy policy = + emel::kernel::matmul::make_execution_policy( + pool, emel::kernel::detect_host_kind(), 2u); + emel::kernel::matmul::sm actor{policy}; +}; + moshi_executor::dependencies make_executor_dependencies( - emel::kernel::sm &kernel, const moshi_executor::kv_views &kv = {}, - emel::logits::sampler::sm *sampler = nullptr) noexcept { + emel::kernel::sm &kernel, emel::kernel::matmul::sm &matmul, + const moshi_executor::kv_views &kv = {}, + emel::logits::sampler::sm *sampler = nullptr, + const emel::kernel::matmul::lane_mode matmul_lane_mode = + emel::kernel::matmul::lane_mode::serial, + emel::kernel::matmul::lane_pool *attention_lanes = nullptr, + const std::size_t active_attention_lanes = 1u) noexcept { return moshi_executor::dependencies{ .kv = kv, .kernel = kernel, + .matmul = matmul, + .matmul_lane_mode = matmul_lane_mode, + .attention_lanes = attention_lanes, + .active_attention_lanes = active_attention_lanes, .sampler = sampler, .policy = moshi_executor::action::policies{ @@ -90,6 +120,23 @@ void configure_predictor_initialize(moshi::event::initialize &request) { request.delay_cache_row_capacity = maximum_delay + 3; } +TEST_CASE("speech Moshi matmul route guards are mutually exclusive") { + emel::kernel::sm serial_kernel{}; + emel::kernel::sm parallel_kernel{}; + serial_matmul_fixture serial_matmul{}; + parallel_matmul_fixture parallel_matmul{}; + moshi_executor::action::context serial_ctx{ + make_executor_dependencies(serial_kernel, serial_matmul.actor)}; + moshi_executor::action::context parallel_ctx{make_executor_dependencies( + parallel_kernel, parallel_matmul.actor, {}, nullptr, + emel::kernel::matmul::lane_mode::parallel)}; + + CHECK(moshi_executor::guard::guard_serial_matmul_mode(serial_ctx)); + CHECK_FALSE(moshi_executor::guard::guard_parallel_matmul_mode(serial_ctx)); + CHECK_FALSE(moshi_executor::guard::guard_serial_matmul_mode(parallel_ctx)); + CHECK(moshi_executor::guard::guard_parallel_matmul_mode(parallel_ctx)); +} + struct recording_graph_executor { bool initialized = false; int32_t reset_count = 0; @@ -884,7 +931,9 @@ TEST_CASE( REQUIRE(lm.dep_q >= 2); emel::kernel::sm kernel{}; - moshi_executor::action::context ctx{make_executor_dependencies(kernel)}; + serial_matmul_fixture matmul{}; + moshi_executor::action::context ctx{ + make_executor_dependencies(kernel, matmul.actor)}; moshi_executor::event::initialize request{*fixture.model}; moshi_executor::event::initialize_ctx request_ctx{}; moshi_executor::event::initialize_run runtime_ev{request, request_ctx}; @@ -1152,8 +1201,10 @@ TEST_CASE("speech_moshi_executor_initializes_lm_fixture") { } emel::kernel::sm kernel{}; + serial_matmul_fixture matmul{}; emel::logits::sampler::sm sampler{}; - moshi_executor::sm executor{make_executor_dependencies(kernel, {}, &sampler)}; + moshi_executor::sm executor{ + make_executor_dependencies(kernel, matmul.actor, {}, &sampler)}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.error_out = &err; @@ -1169,7 +1220,9 @@ TEST_CASE("speech_moshi_executor_rejects_missing_numeric_policy") { } emel::kernel::sm kernel{}; - moshi_executor::sm executor{moshi_executor::dependencies{.kernel = kernel}}; + serial_matmul_fixture matmul{}; + moshi_executor::sm executor{ + moshi_executor::dependencies{.kernel = kernel, .matmul = matmul.actor}}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.error_out = &err; @@ -1185,7 +1238,8 @@ TEST_CASE("speech_moshi_executor_rejects_missing_capacity") { } emel::kernel::sm kernel{}; - auto dependencies = make_executor_dependencies(kernel); + serial_matmul_fixture matmul{}; + auto dependencies = make_executor_dependencies(kernel, matmul.actor); dependencies.capacity.hidden_dim = 0u; moshi_executor::sm executor{dependencies}; emel::error::type err = emel::error::cast(moshi_executor::error::none); @@ -1202,7 +1256,8 @@ TEST_CASE("speech_moshi_executor_rejects_undersized_hidden_capacity") { return; } emel::kernel::sm kernel{}; - auto dependencies = make_executor_dependencies(kernel); + serial_matmul_fixture matmul{}; + auto dependencies = make_executor_dependencies(kernel, matmul.actor); dependencies.capacity.hidden_dim = static_cast(fixture.model->moshi_lm.dim - 1); moshi_executor::sm executor{dependencies}; @@ -1222,8 +1277,10 @@ TEST_CASE("speech_moshi_executor_rejects_undersized_sampling_capacity") { const auto rejected = [&fixture](const uint64_t sampling_card, const uint64_t sampling_top_k) { emel::kernel::sm kernel{}; + serial_matmul_fixture matmul{}; emel::logits::sampler::sm sampler{}; - auto dependencies = make_executor_dependencies(kernel, {}, &sampler); + auto dependencies = + make_executor_dependencies(kernel, matmul.actor, {}, &sampler); dependencies.capacity.sampling_card = sampling_card; dependencies.capacity.sampling_top_k = sampling_top_k; moshi_executor::sm executor{dependencies}; @@ -1252,8 +1309,10 @@ TEST_CASE("speech_moshi_executor_models_zero_seed_and_top_k_clamp") { } emel::kernel::sm kernel{}; + serial_matmul_fixture matmul{}; emel::logits::sampler::sm sampler{}; - moshi_executor::sm executor{make_executor_dependencies(kernel, {}, &sampler)}; + moshi_executor::sm executor{ + make_executor_dependencies(kernel, matmul.actor, {}, &sampler)}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.sampling_enabled = true; @@ -1275,7 +1334,8 @@ TEST_CASE("speech_moshi_executor_normalizes_nonzero_sampling_seed") { } emel::kernel::sm kernel{}; - auto dependencies = make_executor_dependencies(kernel); + serial_matmul_fixture matmul{}; + auto dependencies = make_executor_dependencies(kernel, matmul.actor); moshi_executor::action::context ctx{dependencies}; moshi_executor::event::initialize init{*fixture.model}; init.sampling_seed = 2147483647u; @@ -1297,9 +1357,10 @@ TEST_CASE("speech_moshi_executor_rejects_invalid_sampling_configuration") { [&fixture](const float audio_temperature, const float text_temperature, const int32_t audio_top_k, const int32_t text_top_k) { emel::kernel::sm kernel{}; + serial_matmul_fixture matmul{}; emel::logits::sampler::sm sampler{}; moshi_executor::sm executor{ - make_executor_dependencies(kernel, {}, &sampler)}; + make_executor_dependencies(kernel, matmul.actor, {}, &sampler)}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.sampling_enabled = true; @@ -1326,7 +1387,8 @@ TEST_CASE("speech_moshi_machines_report_unexpected_events_after_init") { } emel::kernel::sm kernel{}; - moshi_executor::sm executor{make_executor_dependencies(kernel)}; + serial_matmul_fixture matmul{}; + moshi_executor::sm executor{make_executor_dependencies(kernel, matmul.actor)}; emel::error::type executor_err = k_no_error; moshi_executor::event::initialize executor_init{*fixture.model}; executor_init.error_out = &executor_err; @@ -1356,7 +1418,8 @@ TEST_CASE("speech_moshi_executor_requires_sampler_for_sampling") { } emel::kernel::sm kernel{}; - moshi_executor::sm executor{make_executor_dependencies(kernel)}; + serial_matmul_fixture matmul{}; + moshi_executor::sm executor{make_executor_dependencies(kernel, matmul.actor)}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.sampling_enabled = true; @@ -1415,7 +1478,8 @@ TEST_CASE("speech_moshi_executor_rejects_graph_step_before_initialize") { } emel::kernel::sm kernel{}; - moshi_executor::sm executor{make_executor_dependencies(kernel)}; + serial_matmul_fixture matmul{}; + moshi_executor::sm executor{make_executor_dependencies(kernel, matmul.actor)}; std::array input = {0, 1, 2, 3}; std::array output = {}; int32_t text_token = -1; @@ -1618,7 +1682,9 @@ TEST_CASE("speech_moshi_executor_runtime_choice_uses_explicit_transitions") { CHECK(source.find("guard_embedding_view_bound") != std::string::npos); CHECK(source.find("guard_embedding_row_succeeded") != std::string::npos); CHECK(source.find("guard_projection_view_bound") != std::string::npos); - CHECK(source.find("guard_depformer_input_projection_bound") != + CHECK(source.find("guard_depformer_input_projection_serial") != + std::string::npos); + CHECK(source.find("guard_depformer_input_projection_parallel") != std::string::npos); CHECK(source.find("guard_depformer_text_input_projection_succeeded") != std::string::npos); @@ -1908,8 +1974,9 @@ TEST_CASE("speech_moshi_generator_and_executor_cover_explicit_error_guards") { snapshot.sequence_kv_blocks[0][0] = 0; emel::kernel::sm guard_kernel{}; + serial_matmul_fixture guard_matmul{}; moshi_executor::action::context executor_ctx{ - make_executor_dependencies(guard_kernel)}; + make_executor_dependencies(guard_kernel, guard_matmul.actor)}; std::array input = {}; std::array output = {}; std::array embedding = {}; @@ -2229,7 +2296,8 @@ TEST_CASE("speech_moshi_executor_embeds_input_before_unsupported_transformer") { } emel::kernel::sm kernel{}; - moshi_executor::sm executor{make_executor_dependencies(kernel)}; + serial_matmul_fixture matmul{}; + moshi_executor::sm executor{make_executor_dependencies(kernel, matmul.actor)}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.error_out = &err; @@ -2272,7 +2340,9 @@ TEST_CASE( const auto views = prepare_kv_views(*fixture.model, &probe, nullptr, &temporal_positions); emel::kernel::sm kernel{}; - moshi_executor::sm executor{make_executor_dependencies(kernel, views)}; + serial_matmul_fixture matmul{}; + moshi_executor::sm executor{ + make_executor_dependencies(kernel, matmul.actor, views)}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.error_out = &err; @@ -2354,10 +2424,12 @@ TEST_CASE("speech_moshi_executor_accepts_personaplex_voice_embedding_step") { initialize_streaming_position(temporal_positions); initialize_streaming_position(depformer_positions); emel::kernel::sm kernel{}; + serial_matmul_fixture matmul{}; const auto views = prepare_kv_views(*fixture.model, &temporal_probe, &depformer_probe, &temporal_positions, &depformer_positions); - moshi_executor::sm executor{make_executor_dependencies(kernel, views)}; + moshi_executor::sm executor{ + make_executor_dependencies(kernel, matmul.actor, views)}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.error_out = &err; @@ -2419,10 +2491,12 @@ TEST_CASE("speech_moshi_executor_generates_audio_tokens_with_injected_kv") { initialize_streaming_position(temporal_positions); initialize_streaming_position(depformer_positions); emel::kernel::sm kernel{}; + serial_matmul_fixture matmul{}; const auto views = prepare_kv_views(*fixture.model, &temporal_probe, &depformer_probe, &temporal_positions, &depformer_positions); - moshi_executor::sm executor{make_executor_dependencies(kernel, views)}; + moshi_executor::sm executor{ + make_executor_dependencies(kernel, matmul.actor, views)}; emel::error::type err = emel::error::cast(moshi_executor::error::none); moshi_executor::event::initialize init{*fixture.model}; init.error_out = &err; @@ -2498,8 +2572,9 @@ TEST_CASE( const auto &lm = fixture.model->moshi_lm; emel::kernel::sm kernel{}; + serial_matmul_fixture matmul{}; moshi_executor::action::context executor_ctx{ - make_executor_dependencies(kernel)}; + make_executor_dependencies(kernel, matmul.actor)}; executor_ctx.session.hidden_dim = lm.dim; executor_ctx.session.text_card = lm.text_card; @@ -2655,97 +2730,226 @@ TEST_CASE( executor_ctx)); } -TEST_CASE("speech_moshi_executor_sampling_rng_is_actor_owned") { - auto first_fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); - auto second_fixture = load_fixture_or_skip("moshi-tiny-lm.gguf"); - if (first_fixture.model == nullptr || second_fixture.model == nullptr) { - return; - } +struct attention_parent_fixture { + emel::speech::predictor::moshi::test::loaded_fixture loaded = {}; + std::unique_ptr temporal = {}; + std::unique_ptr temporal_positions = {}; + std::unique_ptr depformer_positions = {}; + std::unique_ptr kernel = {}; + std::unique_ptr matmul = {}; + std::unique_ptr pool = {}; + std::unique_ptr executor = {}; + std::vector input = {}; + std::vector input_embedding = {}; + std::vector temporal_state_with_canaries = {}; + std::vector audio_output_with_canaries = {}; + int32_t text_token = -1; + emel::error::type err = emel::error::cast(moshi_executor::error::none); +}; - temporal_kv_probe first_temporal{}; - temporal_kv_probe second_temporal{}; - depformer_kv_probe first_depformer{}; - depformer_kv_probe second_depformer{}; - memory_streaming::sm first_temporal_positions{memory_streaming::dependencies{ - .capacity = first_fixture.model->moshi_lm.context}}; - memory_streaming::sm first_depformer_positions{memory_streaming::dependencies{ - .capacity = first_fixture.model->moshi_lm.depformer_context}}; - memory_streaming::sm second_temporal_positions{memory_streaming::dependencies{ - .capacity = second_fixture.model->moshi_lm.context}}; - memory_streaming::sm second_depformer_positions{ - memory_streaming::dependencies{ - .capacity = second_fixture.model->moshi_lm.depformer_context}}; - initialize_streaming_position(first_temporal_positions); - initialize_streaming_position(first_depformer_positions); - initialize_streaming_position(second_temporal_positions); - initialize_streaming_position(second_depformer_positions); - emel::kernel::sm first_kernel{}; - emel::kernel::sm second_kernel{}; - emel::logits::sampler::sm first_sampler{}; - emel::logits::sampler::sm second_sampler{}; - const auto first_views = - prepare_kv_views(*first_fixture.model, &first_temporal, &first_depformer, - &first_temporal_positions, &first_depformer_positions); - const auto second_views = prepare_kv_views( - *second_fixture.model, &second_temporal, &second_depformer, - &second_temporal_positions, &second_depformer_positions); - moshi_executor::sm first{ - make_executor_dependencies(first_kernel, first_views, &first_sampler)}; - moshi_executor::sm second{ - make_executor_dependencies(second_kernel, second_views, &second_sampler)}; - emel::error::type first_err = emel::error::cast(moshi_executor::error::none); - emel::error::type second_err = emel::error::cast(moshi_executor::error::none); - moshi_executor::event::initialize first_init{*first_fixture.model}; - moshi_executor::event::initialize second_init{*second_fixture.model}; - for (auto *init : {&first_init, &second_init}) { - init->sampling_enabled = true; - init->sampling_audio_temperature = 0.8f; - init->sampling_text_temperature = 0.7f; - init->sampling_audio_top_k = 250; - init->sampling_text_top_k = 25; - init->sampling_seed = 1234u; - } - first_init.error_out = &first_err; - second_init.error_out = &second_err; - REQUIRE(first.process_event(first_init)); - REQUIRE(second.process_event(second_init)); +std::unique_ptr +make_attention_parent_fixture(const std::size_t lanes) { + auto fixture = std::make_unique(); + fixture->loaded = load_fixture_or_skip("moshi-tiny-lm.gguf"); + if (fixture->loaded.model == nullptr) { + return fixture; + } + auto &model = *fixture->loaded.model; + model.moshi_lm.num_heads = 8; + fixture->temporal = std::make_unique(); + fixture->temporal_positions = std::make_unique( + memory_streaming::dependencies{.capacity = model.moshi_lm.context}); + fixture->depformer_positions = + std::make_unique(memory_streaming::dependencies{ + .capacity = model.moshi_lm.depformer_context}); + initialize_streaming_position(*fixture->temporal_positions); + initialize_streaming_position(*fixture->depformer_positions); + fixture->kernel = std::make_unique(); + fixture->matmul = std::make_unique(); + if (lanes > 1u) { + fixture->pool = std::make_unique(); + } + const auto views = prepare_kv_views(model, fixture->temporal.get(), nullptr, + fixture->temporal_positions.get(), + fixture->depformer_positions.get()); + fixture->executor = + std::make_unique(make_executor_dependencies( + *fixture->kernel, fixture->matmul->actor, views, nullptr, + emel::kernel::matmul::lane_mode::serial, fixture->pool.get(), lanes)); + moshi_executor::event::initialize initialize{model}; + initialize.error_out = &fixture->err; + REQUIRE(fixture->executor->process_event(initialize)); + fixture->input.assign(static_cast(model.moshi_lm.n_q + 1), -1); + fixture->input[0] = model.moshi_lm.text_padding_id; + fixture->input_embedding.resize(static_cast(model.moshi_lm.dim)); + for (std::size_t index = 0u; index < fixture->input_embedding.size(); + ++index) { + fixture->input_embedding[index] = + static_cast(static_cast(index) - 16) / 32.0f; + } + fixture->temporal_state_with_canaries.assign( + static_cast(model.moshi_lm.dim) + 2u, -12345.0f); + fixture->audio_output_with_canaries.assign( + static_cast(model.moshi_lm.dep_q) + 2u, -12345); + return fixture; +} - std::array input = {}; - std::array first_output = {}; - std::array second_output = {}; - input.fill(-1); - input[0] = first_fixture.model->moshi_lm.text_padding_id; - int32_t first_text_token = -1; - int32_t second_text_token = -1; +bool run_attention_parent_prediction(attention_parent_fixture &fixture, + std::size_t &allocations) { + auto &model = *fixture.loaded.model; + std::fill(fixture.temporal_state_with_canaries.begin() + 1, + fixture.temporal_state_with_canaries.end() - 1, 0.0f); + std::fill(fixture.audio_output_with_canaries.begin() + 1, + fixture.audio_output_with_canaries.end() - 1, -1); + fixture.text_token = -1; + fixture.err = emel::error::cast(moshi_executor::error::none); emel::memory::view::snapshot snapshot{}; snapshot.max_sequences = 1; snapshot.sequence_active[0] = 1; snapshot.sequence_length_values[0] = 1; snapshot.sequence_kv_block_count[0] = 1; snapshot.sequence_kv_blocks[0][0] = 0; - const auto input_span = std::span{ - input.data(), static_cast(first_fixture.model->moshi_lm.n_q + 1)}; - moshi::event::graph_step first_step{ - *first_fixture.model, snapshot, input_span, - std::span{ - first_output.data(), - static_cast(first_fixture.model->moshi_lm.dep_q)}, - first_text_token}; - moshi::event::graph_step second_step{ - *second_fixture.model, snapshot, input_span, - std::span{ - second_output.data(), - static_cast(second_fixture.model->moshi_lm.dep_q)}, - second_text_token}; - first_step.error_out = &first_err; - second_step.error_out = &second_err; - - REQUIRE(first.process_event(first_step)); - REQUIRE(second.process_event(second_step)); - CHECK(first_text_token == second_text_token); - CHECK(std::equal(first_output.begin(), - first_output.begin() + first_fixture.model->moshi_lm.dep_q, - second_output.begin())); + moshi::event::graph_step step{ + model, snapshot, std::span{fixture.input}, + std::span{fixture.audio_output_with_canaries.data() + 1, + static_cast(model.moshi_lm.dep_q)}, + fixture.text_token}; + step.phase = moshi::event::graph_phase::prediction; + step.input_embedding = std::span{fixture.input_embedding}; + step.temporal_state = + std::span{fixture.temporal_state_with_canaries.data() + 1, + static_cast(model.moshi_lm.dim)}; + step.error_out = &fixture.err; + bool accepted = false; + { + emel::test::allocation::allocation_scope allocation_scope{}; + accepted = fixture.executor->process_event(step); + allocations = allocation_scope.allocations(); + } + return accepted; +} + +TEST_CASE("speech Moshi attention parent routes are exact disjoint and " + "allocation free") { + auto serial = make_attention_parent_fixture(1u); + auto two = make_attention_parent_fixture(2u); + auto four = make_attention_parent_fixture(4u); + auto eight = make_attention_parent_fixture(8u); + if (serial->loaded.model == nullptr || two->loaded.model == nullptr || + four->loaded.model == nullptr || eight->loaded.model == nullptr) { + return; + } + + std::size_t serial_allocations = 0u; + std::size_t two_allocations = 0u; + std::size_t four_allocations = 0u; + std::size_t eight_allocations = 0u; + REQUIRE(run_attention_parent_prediction(*serial, serial_allocations)); + REQUIRE(run_attention_parent_prediction(*two, two_allocations)); + REQUIRE(run_attention_parent_prediction(*four, four_allocations)); + REQUIRE(run_attention_parent_prediction(*eight, eight_allocations)); + CHECK(serial_allocations == 0u); + CHECK(two_allocations == 0u); + CHECK(four_allocations == 0u); + CHECK(eight_allocations == 0u); + CHECK(two->temporal_state_with_canaries == + serial->temporal_state_with_canaries); + CHECK(four->temporal_state_with_canaries == + serial->temporal_state_with_canaries); + CHECK(eight->temporal_state_with_canaries == + serial->temporal_state_with_canaries); + CHECK(two->temporal->key_cache == serial->temporal->key_cache); + CHECK(two->temporal->value_cache == serial->temporal->value_cache); + CHECK(four->temporal->key_cache == serial->temporal->key_cache); + CHECK(four->temporal->value_cache == serial->temporal->value_cache); + CHECK(eight->temporal->key_cache == serial->temporal->key_cache); + CHECK(eight->temporal->value_cache == serial->temporal->value_cache); + for (const auto *fixture : {two.get(), four.get(), eight.get()}) { + CHECK(fixture->temporal_state_with_canaries.front() == -12345.0f); + CHECK(fixture->temporal_state_with_canaries.back() == -12345.0f); + CHECK(fixture->audio_output_with_canaries.front() == -12345); + CHECK(fixture->audio_output_with_canaries.back() == -12345); + } +} + +TEST_CASE("speech Moshi unsupported attention lane count fails explicitly") { + auto fixture = make_attention_parent_fixture(3u); + if (fixture->loaded.model == nullptr) { + return; + } + + std::size_t allocations = 0u; + CHECK_FALSE(run_attention_parent_prediction(*fixture, allocations)); + CHECK(allocations == 0u); + CHECK(fixture->err == + emel::error::cast(moshi_executor::error::graph_execution_unsupported)); +} + +TEST_CASE( + "speech Moshi attention partial submission drains accepted children") { + auto fixture = make_attention_parent_fixture(8u); + if (fixture->loaded.model == nullptr) { + return; + } + + std::atomic blocker_entered = false; + std::atomic release_blocker = false; + std::atomic drain_probe_entered = 0u; + std::atomic release_drain_probes = false; + emel::kernel::matmul::lane_pool::join_group blocker_group{}; + emel::kernel::matmul::lane_pool::join_group drain_probe_group{}; + struct release_tasks { + std::atomic &release_blocker; + std::atomic &release_drain_probes; + emel::kernel::matmul::lane_pool::join_group &blocker_group; + emel::kernel::matmul::lane_pool::join_group &drain_probe_group; + ~release_tasks() { + release_drain_probes.store(true, std::memory_order_release); + (void)drain_probe_group.wait(); + release_blocker.store(true, std::memory_order_release); + (void)blocker_group.wait(); + } + } cleanup{release_blocker, release_drain_probes, blocker_group, + drain_probe_group}; + + REQUIRE(fixture->pool->try_submit(blocker_group, [&]() noexcept { + blocker_entered.store(true, std::memory_order_release); + while (!release_blocker.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + })); + for (std::size_t attempt = 0u; attempt < 100000u; ++attempt) { + if (blocker_entered.load(std::memory_order_acquire)) { + break; + } + std::this_thread::yield(); + } + REQUIRE(blocker_entered.load(std::memory_order_acquire)); + + std::size_t allocations = 0u; + CHECK_FALSE(run_attention_parent_prediction(*fixture, allocations)); + CHECK(allocations == 0u); + CHECK(fixture->err == + emel::error::cast(moshi_executor::error::graph_execution_unsupported)); + + std::size_t drain_probe_submitted = 0u; + for (std::size_t index = 0u; index < 6u; ++index) { + drain_probe_submitted += static_cast( + fixture->pool->try_submit(drain_probe_group, [&]() noexcept { + drain_probe_entered.fetch_add(1u, std::memory_order_acq_rel); + while (!release_drain_probes.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + })); + } + CHECK(drain_probe_submitted == 6u); + for (std::size_t attempt = 0u; attempt < 100000u; ++attempt) { + if (drain_probe_entered.load(std::memory_order_acquire) == 6u) { + break; + } + std::this_thread::yield(); + } + CHECK(drain_probe_entered.load(std::memory_order_acquire) == 6u); } TEST_CASE( @@ -2758,7 +2962,8 @@ TEST_CASE( emel::memory::test::recording_kv_actor kv{}; emel::memory::hybrid::sm memory{emel::memory::hybrid::bind_kv_actor(kv)}; emel::kernel::sm kernel{}; - moshi_executor::sm executor{make_executor_dependencies(kernel)}; + serial_matmul_fixture matmul{}; + moshi_executor::sm executor{make_executor_dependencies(kernel, matmul.actor)}; moshi::sm generator{make_predictor_dependencies(executor, memory)}; emel::error::type err = k_no_error; diff --git a/tests/text/generator/action_guard_tests.cpp b/tests/text/generator/action_guard_tests.cpp index ca8b2a6a..f116719a 100644 --- a/tests/text/generator/action_guard_tests.cpp +++ b/tests/text/generator/action_guard_tests.cpp @@ -16,7 +16,7 @@ #include "emel/text/generator/actions.hpp" #include "emel/text/generator/guards.hpp" #include "emel/text/generator/initializer/guards.hpp" -#include "emel/text/generator/matmul/sm.hpp" +#include "emel/kernel/matmul/sm.hpp" #include "emel/text/generator/prefill/actions.hpp" #include "emel/text/generator/prefill/guards.hpp" #include "generator_test_policies.hpp" @@ -229,11 +229,11 @@ struct native_quantized_route_fixture { struct compute_guard_fixture { emel::text::generator::action::context context = {}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; - emel::text::generator::matmul::execution_policy matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; + emel::kernel::matmul::execution_policy matmul_policy = + emel::kernel::matmul::make_auto_execution_policy( parallel_matmul_lanes); - emel::text::generator::matmul::sm matmul_actor{matmul_policy}; + emel::kernel::matmul::sm matmul_actor{matmul_policy}; emel::model::data model = {}; std::array reserve_tensors = {}; @@ -262,6 +262,7 @@ struct compute_guard_fixture { auto &backend = context.compute.backend; backend.matmul_actor = &matmul_actor; + backend.matmul_lane_mode = emel::kernel::matmul::lane_mode::parallel; backend.model = &model; backend.n_embd = 4; backend.n_head = 1; diff --git a/tests/text/generator/detail_tests.cpp b/tests/text/generator/detail_tests.cpp index 1c7e9e2c..20045ebf 100644 --- a/tests/text/generator/detail_tests.cpp +++ b/tests/text/generator/detail_tests.cpp @@ -43,21 +43,19 @@ using emel::text::generator::detail::quant::QK8_0; using emel::text::generator::detail::quant::QK_K; struct matmul_actor_fixture { - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> - parallel_matmul_lanes = {}; - emel::text::generator::matmul::execution_policy policy = - emel::text::generator::matmul::make_auto_execution_policy( - parallel_matmul_lanes); - emel::text::generator::matmul::sm actor{policy}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; + emel::kernel::matmul::execution_policy policy = + emel::kernel::matmul::make_auto_execution_policy(parallel_matmul_lanes); + emel::kernel::matmul::sm actor{policy}; }; void bind_test_matmul_actor( emel::text::generator::detail::native_backend &backend, matmul_actor_fixture &matmul) { backend.matmul_actor = &matmul.actor; + backend.matmul_lane_mode = emel::kernel::matmul::lane_mode::parallel; matmul.actor.process_event( - emel::text::generator::matmul::event::configure_kernel_kind{ - backend.kernel_kind}); + emel::kernel::matmul::event::configure_kernel_kind{backend.kernel_kind}); } uint16_t fp16_bits(const float value) { @@ -515,7 +513,7 @@ struct chunk4_prefill_runtime_fixture { backend.kernel_kind = emel::kernel::kernel_kind::aarch64; backend.matmul_actor = &matmul.actor; matmul.actor.process_event( - emel::text::generator::matmul::event::configure_kernel_kind{ + emel::kernel::matmul::event::configure_kernel_kind{ backend.kernel_kind}); backend.model = &model; backend.n_vocab = k_vocab; @@ -732,7 +730,7 @@ struct hybrid_chunked_q8_runtime_fixture { backend.kernel_kind = emel::kernel::kernel_kind::aarch64; backend.matmul_actor = &matmul.actor; matmul.actor.process_event( - emel::text::generator::matmul::event::configure_kernel_kind{ + emel::kernel::matmul::event::configure_kernel_kind{ backend.kernel_kind}); backend.model = &model; backend.n_vocab = k_vocab; @@ -4275,14 +4273,14 @@ TEST_CASE("generator_detail_run_kernel_flash_prefill_parallel_chunk8_keeps_" // The parallel route must keep every prefill matmul on the matmul lane // actors; the matmul actor's serial kernel stays untouched. CHECK(parallel_fixture->backend.kernel.optimized_q4_dispatch_count() == 0u); - CHECK(parallel_fixture->matmul.actor.serial_kernel() - .optimized_q4_dispatch_count() == 0u); - uint64_t lane_q4_dispatches = 0u; - for (const auto &lane_kernel : - parallel_fixture->matmul.actor.parallel_lane_kernels()) { - lane_q4_dispatches += lane_kernel.optimized_q4_dispatch_count(); - } - CHECK(lane_q4_dispatches > 0u); + emel::kernel::matmul::event::diagnostics matmul_diagnostics = {}; + bool diagnostics_accepted = false; + const emel::kernel::matmul::event::capture_diagnostics capture{ + matmul_diagnostics, diagnostics_accepted}; + REQUIRE(parallel_fixture->backend.matmul_actor->process_event(capture)); + REQUIRE(diagnostics_accepted); + CHECK(matmul_diagnostics.serial_optimized_q4_dispatch_calls == 0u); + CHECK(matmul_diagnostics.parallel_optimized_q4_dispatch_calls > 0u); // Row-sliced lanes write disjoint dst rows and reorder no reductions, so // the parallel output is bit-identical to the serial dispatch. diff --git a/tests/text/generator/determinism_tests.cpp b/tests/text/generator/determinism_tests.cpp index 46a2d6ad..8a5dd103 100644 --- a/tests/text/generator/determinism_tests.cpp +++ b/tests/text/generator/determinism_tests.cpp @@ -19,9 +19,10 @@ namespace { namespace gen_detail = emel::text::generator::detail; +namespace matmul_detail = emel::kernel::matmul::detail; using emel::kernel::event::dtype; using gen_detail::matmul_lane_mode; -using gen_detail::matmul_row_slice; +using matmul_detail::matmul_row_slice; inline constexpr size_t k_test_matmul_lanes = 8u; @@ -41,7 +42,7 @@ emel::model::data::tensor_record make_tensor_record(void *data, } // Contiguous group-aligned partition of `rows` into exactly `slice_count` -// slices: the same shape the production compute_matmul_row_slices emits, +// slices: the same shape the production fixed-lane actor emits, // parameterized over the slice count so the tests can sweep every effective // lane count instead of only one fixed topology. size_t @@ -125,7 +126,7 @@ bool outputs_identical(const std::vector &lhs, } uint64_t matmul_group_rows_for_test(const dtype type) noexcept { - namespace matmul = emel::text::generator::matmul; + namespace matmul = emel::kernel::matmul; if (matmul::guard::guard_uses_x8_row_groups(type)) { return emel::kernel::detail::quant::Q4_K_X8_ROWS; } @@ -147,7 +148,7 @@ bool run_row_sliced(emel::kernel::sm &kernel, const matmul_case &fixture, group_rows, slice_count, slices); bool all_ok = true; for (size_t lane = 0; lane < lanes; ++lane) { - const auto sliced = gen_detail::compute_sliced_mul_mat_event( + const auto sliced = matmul_detail::compute_sliced_mul_mat_event( full, group_rows, slices[lane]); all_ok = all_ok && kernel.process_event(sliced); } @@ -204,16 +205,16 @@ TEST_CASE("determinism: row-sliced matmul bitwise invariant across slice " TEST_CASE("determinism: parallel fork/join dispatch bitwise repeatable and " "serial-identical") { const matmul_case fixture(dtype::q8_0, 64, 61, 1); - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; - auto policy = emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; + auto policy = emel::kernel::matmul::make_auto_execution_policy( parallel_matmul_lanes); - emel::text::generator::matmul::sm matmul_actor{policy}; + emel::kernel::matmul::sm matmul_actor{policy}; gen_detail::native_backend backend = {}; backend.kernel_kind = policy.kernel_kind; backend.kernel.set_kind(backend.kernel_kind); backend.matmul_actor = &matmul_actor; matmul_actor.process_event( - emel::text::generator::matmul::event::configure_kernel_kind{backend.kernel_kind}); + emel::kernel::matmul::event::configure_kernel_kind{backend.kernel_kind}); std::vector serial_output = {}; const auto serial_ev = fixture.event_for(serial_output); diff --git a/tests/text/generator/lifecycle_tests.cpp b/tests/text/generator/lifecycle_tests.cpp index 7ab5ca25..4071af1a 100644 --- a/tests/text/generator/lifecycle_tests.cpp +++ b/tests/text/generator/lifecycle_tests.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -7,17 +6,18 @@ #include #include #include +#include #include #include #include #include +#include "../../allocation_tracker.hpp" +#include "../../memory/recording_kv_actor.hpp" #include "emel/docs/detail.hpp" #include "emel/emel.h" -#include "emel/text/generator/errors.hpp" -#include "emel/text/generator/initializer/sm.hpp" -#include "emel/text/generator/prefill/sm.hpp" -#include "emel/text/generator/sm.hpp" +#include "emel/graph/tensor/errors.hpp" +#include "emel/graph/tensor/events.hpp" #include "emel/kernel/detail.hpp" #include "emel/kernel/events.hpp" #include "emel/model/data.hpp" @@ -28,12 +28,12 @@ #include "emel/model/llama/detail.hpp" #include "emel/model/loader/errors.hpp" #include "emel/model/qwen3/detail.hpp" -#include "emel/graph/tensor/errors.hpp" -#include "emel/graph/tensor/events.hpp" #include "emel/text/formatter/format.hpp" +#include "emel/text/generator/errors.hpp" +#include "emel/text/generator/initializer/sm.hpp" +#include "emel/text/generator/prefill/sm.hpp" +#include "emel/text/generator/sm.hpp" #include "emel/text/tokenizer/sm.hpp" -#include "../../allocation_tracker.hpp" -#include "../../memory/recording_kv_actor.hpp" #include "generator_test_policies.hpp" namespace { @@ -67,36 +67,40 @@ struct callback_tracker { bool initialize_error_called = false; bool generate_done_called = false; bool generate_error_called = false; - const emel::text::generator::event::initialize * initialize_request = nullptr; - const emel::text::generator::event::generate * generate_request = nullptr; + const emel::text::generator::event::initialize *initialize_request = nullptr; + const emel::text::generator::event::generate *generate_request = nullptr; int32_t tokens_generated = -1; size_t output_length = 0; emel::error::type err = emel::error::cast(emel::text::generator::error::none); }; -void on_initialize_done(void * owner, const emel::text::generator::events::initialize_done & ev) { - auto * tracker = static_cast(owner); +void on_initialize_done( + void *owner, const emel::text::generator::events::initialize_done &ev) { + auto *tracker = static_cast(owner); tracker->initialize_done_called = true; tracker->initialize_request = ev.request; } -void on_initialize_error(void * owner, const emel::text::generator::events::initialize_error & ev) { - auto * tracker = static_cast(owner); +void on_initialize_error( + void *owner, const emel::text::generator::events::initialize_error &ev) { + auto *tracker = static_cast(owner); tracker->initialize_error_called = true; tracker->initialize_request = ev.request; tracker->err = ev.err; } -void on_generate_done(void * owner, const emel::text::generator::events::generation_done & ev) { - auto * tracker = static_cast(owner); +void on_generate_done( + void *owner, const emel::text::generator::events::generation_done &ev) { + auto *tracker = static_cast(owner); tracker->generate_done_called = true; tracker->generate_request = ev.request; tracker->tokens_generated = ev.tokens_generated; tracker->output_length = ev.output_length; } -void on_generate_error(void * owner, const emel::text::generator::events::generation_error & ev) { - auto * tracker = static_cast(owner); +void on_generate_error( + void *owner, const emel::text::generator::events::generation_error &ev) { + auto *tracker = static_cast(owner); tracker->generate_error_called = true; tracker->generate_request = ev.request; tracker->tokens_generated = ev.tokens_generated; @@ -104,7 +108,7 @@ void on_generate_error(void * owner, const emel::text::generator::events::genera tracker->err = ev.err; } -int32_t add_token(emel::model::data::vocab & vocab, const char * text, +int32_t add_token(emel::model::data::vocab &vocab, const char *text, const int32_t type = 0) { const uint32_t length = static_cast(std::strlen(text)); const uint32_t offset = vocab.token_bytes_used; @@ -119,20 +123,22 @@ int32_t add_token(emel::model::data::vocab & vocab, const char * text, return static_cast(id); } -bool tokenizer_bind_dispatch(void * tokenizer_sm, - const emel::text::tokenizer::event::bind & ev) { - return static_cast(tokenizer_sm)->process_event(ev); +bool tokenizer_bind_dispatch(void *tokenizer_sm, + const emel::text::tokenizer::event::bind &ev) { + return static_cast(tokenizer_sm) + ->process_event(ev); } bool tokenizer_tokenize_dispatch( - void * tokenizer_sm, const emel::text::tokenizer::event::tokenize & ev) { - return static_cast(tokenizer_sm)->process_event(ev); + void *tokenizer_sm, const emel::text::tokenizer::event::tokenize &ev) { + return static_cast(tokenizer_sm) + ->process_event(ev); } -emel::error::type sampler_select_argmax(int32_t & candidate_ids, - float & candidate_scores, - int32_t & candidate_count, - int32_t & selected_token_out) { +emel::error::type sampler_select_argmax(int32_t &candidate_ids, + float &candidate_scores, + int32_t &candidate_count, + int32_t &selected_token_out) { int32_t best_index = 0; float best_score = (&candidate_scores)[0]; for (int32_t idx = 1; idx < candidate_count; ++idx) { @@ -146,7 +152,8 @@ emel::error::type sampler_select_argmax(int32_t & candidate_ids, } template -constexpr void for_each_type(stateforward::sml::aux::type_list, fn && visitor) { +constexpr void for_each_type(stateforward::sml::aux::type_list, + fn &&visitor) { (visitor.template operator()(), ...); } @@ -158,32 +165,41 @@ struct prepared_model { }; static_assert(std::is_reference_v< - decltype(std::declval() + decltype(std::declval< + const emel::text::generator::event::initialize &>() .dispatch_tokenizer_bind)>); static_assert(std::is_reference_v< - decltype(std::declval() + decltype(std::declval< + const emel::text::generator::event::initialize &>() .dispatch_tokenizer_tokenize)>); -static_assert(std::is_same_v< - std::remove_cvref_t< - decltype(std::declval() - .sampler_fns)>, - std::span>); -static_assert(std::is_same_v< - std::remove_cvref_t< - decltype(std::declval().messages)>, - std::span>); -static_assert(std::is_same_v< - std::remove_cvref_t< - decltype(std::declval().output)>, - std::span>); -static_assert(std::is_reference_v< - decltype(std::declval() - .output_length_out)>); - -emel::text::generator::diagnostics capture_generator_diagnostics( - emel::text::generator::sm & generator) { +static_assert( + std::is_same_v< + std::remove_cvref_t() + .sampler_fns)>, + std::span>); +static_assert( + std::is_same_v< + std::remove_cvref_t() + .messages)>, + std::span>); +static_assert( + std::is_same_v< + std::remove_cvref_t() + .output)>, + std::span>); +static_assert( + std::is_reference_v< + decltype(std::declval() + .output_length_out)>); + +emel::text::generator::diagnostics +capture_generator_diagnostics(emel::text::generator::sm &generator) { emel::text::generator::diagnostics diagnostics = {}; - CHECK(generator.process_event(emel::text::generator::event::capture_diagnostics{diagnostics})); + CHECK(generator.process_event( + emel::text::generator::event::capture_diagnostics{diagnostics})); return diagnostics; } @@ -195,11 +211,12 @@ struct checked_formatter_ctx { bool seen = false; }; -bool format_checked_messages(void * formatter_ctx, - const emel::text::formatter::format_request & request, - int32_t * error_out) { +bool format_checked_messages( + void *formatter_ctx, const emel::text::formatter::format_request &request, + int32_t *error_out) { if (error_out != nullptr) { - *error_out = emel::text::formatter::error_code(emel::text::formatter::error::none); + *error_out = + emel::text::formatter::error_code(emel::text::formatter::error::none); } if (request.output_length_out != nullptr) { *request.output_length_out = 0u; @@ -213,7 +230,7 @@ bool format_checked_messages(void * formatter_ctx, return false; } - auto & ctx = *static_cast(formatter_ctx); + auto &ctx = *static_cast(formatter_ctx); if (request.messages.size() != ctx.expected_messages.size() || request.add_generation_prompt != ctx.expected_add_generation_prompt || request.enable_thinking != ctx.expected_enable_thinking) { @@ -253,14 +270,17 @@ bool format_checked_messages(void * formatter_ctx, return true; } -void build_prepared_model(prepared_model & prepared) { +void build_prepared_model(prepared_model &prepared) { prepared.tensor_storage.reserve(12); - prepared.data.vocab_data.tokenizer_model_id = emel::model::data::tokenizer_model::BPE; - prepared.data.vocab_data.tokenizer_pre_id = emel::model::data::tokenizer_pre::GPT2; + prepared.data.vocab_data.tokenizer_model_id = + emel::model::data::tokenizer_model::BPE; + prepared.data.vocab_data.tokenizer_pre_id = + emel::model::data::tokenizer_pre::GPT2; prepared.data.vocab_data.ignore_merges = true; prepared.hello_id = add_token(prepared.data.vocab_data, "hello"); prepared.world_id = add_token(prepared.data.vocab_data, "world"); - prepared.data.params.n_vocab = static_cast(prepared.data.vocab_data.n_tokens); + prepared.data.params.n_vocab = + static_cast(prepared.data.vocab_data.n_tokens); prepared.data.params.n_embd = 4; prepared.data.params.n_head = 1; prepared.data.params.n_head_kv = 1; @@ -273,16 +293,18 @@ void build_prepared_model(prepared_model & prepared) { std::memcpy(prepared.data.architecture_name.data(), "llama", 5u); uint32_t tensor_index = 0u; - const auto add_name = [&](emel::model::data::tensor_record & tensor, const std::string_view name) { + const auto add_name = [&](emel::model::data::tensor_record &tensor, + const std::string_view name) { tensor.name_offset = prepared.data.name_bytes_used; tensor.name_length = static_cast(name.size()); - std::memcpy(prepared.data.name_storage.data() + prepared.data.name_bytes_used, - name.data(), - name.size()); + std::memcpy(prepared.data.name_storage.data() + + prepared.data.name_bytes_used, + name.data(), name.size()); prepared.data.name_bytes_used += static_cast(name.size()); }; - const auto add_vector = [&](const std::string_view name, const std::vector & values) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const auto add_vector = [&](const std::string_view name, + const std::vector &values) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.push_back(values); tensor.type = static_cast(emel::kernel::event::dtype::f32); @@ -291,11 +313,10 @@ void build_prepared_model(prepared_model & prepared) { tensor.data = prepared.tensor_storage.back().data(); tensor.data_size = static_cast(values.size() * sizeof(float)); }; - const auto add_matrix = [&](const std::string_view name, - const int32_t rows, + const auto add_matrix = [&](const std::string_view name, const int32_t rows, const int32_t cols, - const std::vector & values) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const std::vector &values) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.push_back(values); tensor.type = static_cast(emel::kernel::event::dtype::f32); @@ -306,11 +327,11 @@ void build_prepared_model(prepared_model & prepared) { tensor.data_size = static_cast(values.size() * sizeof(float)); }; - add_matrix("token_embd.weight", 2, 4, {1.0f, 0.0f, 0.0f, 0.0f, - 1.0f, 0.0f, 0.0f, 0.0f}); + add_matrix("token_embd.weight", 2, 4, + {1.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f}); add_vector("output_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); - add_matrix("output.weight", 2, 4, {0.0f, 0.0f, 0.0f, 0.0f, - 1.0f, 0.0f, 0.0f, 0.0f}); + add_matrix("output.weight", 2, 4, + {0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f}); add_vector("blk.0.attn_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); add_matrix("blk.0.attn_q.weight", 4, 4, std::vector(16, 0.0f)); add_matrix("blk.0.attn_k.weight", 4, 4, std::vector(16, 0.0f)); @@ -323,16 +344,19 @@ void build_prepared_model(prepared_model & prepared) { prepared.data.n_tensors = tensor_index; } -void build_quantized_contract_prepared_model(prepared_model & prepared) { +void build_quantized_contract_prepared_model(prepared_model &prepared) { constexpr int32_t k_n_embd = static_cast(emel::kernel::detail::quant::QK_K); prepared.tensor_storage.reserve(12); - prepared.data.vocab_data.tokenizer_model_id = emel::model::data::tokenizer_model::BPE; - prepared.data.vocab_data.tokenizer_pre_id = emel::model::data::tokenizer_pre::GPT2; + prepared.data.vocab_data.tokenizer_model_id = + emel::model::data::tokenizer_model::BPE; + prepared.data.vocab_data.tokenizer_pre_id = + emel::model::data::tokenizer_pre::GPT2; prepared.data.vocab_data.ignore_merges = true; prepared.hello_id = add_token(prepared.data.vocab_data, "hello"); prepared.world_id = add_token(prepared.data.vocab_data, "world"); - prepared.data.params.n_vocab = static_cast(prepared.data.vocab_data.n_tokens); + prepared.data.params.n_vocab = + static_cast(prepared.data.vocab_data.n_tokens); prepared.data.params.n_embd = k_n_embd; prepared.data.params.n_head = 1; prepared.data.params.n_head_kv = 1; @@ -347,35 +371,39 @@ void build_quantized_contract_prepared_model(prepared_model & prepared) { std::memcpy(prepared.data.architecture_name.data(), "llama", 5u); uint32_t tensor_index = 0u; - const auto add_name = [&](emel::model::data::tensor_record & tensor, const std::string_view name) { + const auto add_name = [&](emel::model::data::tensor_record &tensor, + const std::string_view name) { tensor.name_offset = prepared.data.name_bytes_used; tensor.name_length = static_cast(name.size()); - std::memcpy(prepared.data.name_storage.data() + prepared.data.name_bytes_used, - name.data(), - name.size()); + std::memcpy(prepared.data.name_storage.data() + + prepared.data.name_bytes_used, + name.data(), name.size()); prepared.data.name_bytes_used += static_cast(name.size()); }; const auto add_vector = [&](const std::string_view name, const int32_t cols) { - auto & tensor = prepared.data.tensors[tensor_index++]; + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.emplace_back(static_cast(cols), 1.0f); tensor.type = static_cast(emel::kernel::event::dtype::f32); tensor.n_dims = 1; tensor.dims[0] = cols; tensor.data = prepared.tensor_storage.back().data(); - tensor.data_size = static_cast(prepared.tensor_storage.back().size() * sizeof(float)); + tensor.data_size = static_cast( + prepared.tensor_storage.back().size() * sizeof(float)); }; - const auto add_matrix = [&](const std::string_view name, const int32_t rows, const int32_t cols) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const auto add_matrix = [&](const std::string_view name, const int32_t rows, + const int32_t cols) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); - prepared.tensor_storage.emplace_back(static_cast(rows) * static_cast(cols), - 0.0f); + prepared.tensor_storage.emplace_back( + static_cast(rows) * static_cast(cols), 0.0f); tensor.type = static_cast(emel::kernel::event::dtype::f32); tensor.n_dims = 2; tensor.dims[0] = cols; tensor.dims[1] = rows; tensor.data = prepared.tensor_storage.back().data(); - tensor.data_size = static_cast(prepared.tensor_storage.back().size() * sizeof(float)); + tensor.data_size = static_cast( + prepared.tensor_storage.back().size() * sizeof(float)); }; add_matrix("token_embd.weight", 2, k_n_embd); @@ -393,18 +421,21 @@ void build_quantized_contract_prepared_model(prepared_model & prepared) { prepared.data.n_tensors = tensor_index; } -void build_qwen3_quantized_contract_prepared_model(prepared_model & prepared, +void build_qwen3_quantized_contract_prepared_model(prepared_model &prepared, const bool include_q_norm, const bool include_k_norm) { constexpr int32_t k_n_embd = static_cast(emel::kernel::detail::quant::QK_K); prepared.tensor_storage.reserve(14); - prepared.data.vocab_data.tokenizer_model_id = emel::model::data::tokenizer_model::BPE; - prepared.data.vocab_data.tokenizer_pre_id = emel::model::data::tokenizer_pre::GPT2; + prepared.data.vocab_data.tokenizer_model_id = + emel::model::data::tokenizer_model::BPE; + prepared.data.vocab_data.tokenizer_pre_id = + emel::model::data::tokenizer_pre::GPT2; prepared.data.vocab_data.ignore_merges = true; prepared.hello_id = add_token(prepared.data.vocab_data, "hello"); prepared.world_id = add_token(prepared.data.vocab_data, "world"); - prepared.data.params.n_vocab = static_cast(prepared.data.vocab_data.n_tokens); + prepared.data.params.n_vocab = + static_cast(prepared.data.vocab_data.n_tokens); prepared.data.params.n_embd = k_n_embd; prepared.data.params.n_head = 1; prepared.data.params.n_head_kv = 1; @@ -417,35 +448,39 @@ void build_qwen3_quantized_contract_prepared_model(prepared_model & prepared, std::memcpy(prepared.data.architecture_name.data(), "qwen3", 5u); uint32_t tensor_index = 0u; - const auto add_name = [&](emel::model::data::tensor_record & tensor, const std::string_view name) { + const auto add_name = [&](emel::model::data::tensor_record &tensor, + const std::string_view name) { tensor.name_offset = prepared.data.name_bytes_used; tensor.name_length = static_cast(name.size()); - std::memcpy(prepared.data.name_storage.data() + prepared.data.name_bytes_used, - name.data(), - name.size()); + std::memcpy(prepared.data.name_storage.data() + + prepared.data.name_bytes_used, + name.data(), name.size()); prepared.data.name_bytes_used += static_cast(name.size()); }; const auto add_vector = [&](const std::string_view name, const int32_t cols) { - auto & tensor = prepared.data.tensors[tensor_index++]; + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.emplace_back(static_cast(cols), 1.0f); tensor.type = static_cast(emel::kernel::event::dtype::f32); tensor.n_dims = 1; tensor.dims[0] = cols; tensor.data = prepared.tensor_storage.back().data(); - tensor.data_size = static_cast(prepared.tensor_storage.back().size() * sizeof(float)); + tensor.data_size = static_cast( + prepared.tensor_storage.back().size() * sizeof(float)); }; - const auto add_matrix = [&](const std::string_view name, const int32_t rows, const int32_t cols) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const auto add_matrix = [&](const std::string_view name, const int32_t rows, + const int32_t cols) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); - prepared.tensor_storage.emplace_back(static_cast(rows) * static_cast(cols), - 0.0f); + prepared.tensor_storage.emplace_back( + static_cast(rows) * static_cast(cols), 0.0f); tensor.type = static_cast(emel::kernel::event::dtype::f32); tensor.n_dims = 2; tensor.dims[0] = cols; tensor.dims[1] = rows; tensor.data = prepared.tensor_storage.back().data(); - tensor.data_size = static_cast(prepared.tensor_storage.back().size() * sizeof(float)); + tensor.data_size = static_cast( + prepared.tensor_storage.back().size() * sizeof(float)); }; add_matrix("token_embd.weight", 2, k_n_embd); @@ -469,16 +504,19 @@ void build_qwen3_quantized_contract_prepared_model(prepared_model & prepared, prepared.data.n_tensors = tensor_index; } -void build_lfm2_quantized_contract_prepared_model(prepared_model & prepared) { +void build_lfm2_quantized_contract_prepared_model(prepared_model &prepared) { constexpr int32_t k_n_embd = static_cast(emel::kernel::detail::quant::QK_K); prepared.tensor_storage.reserve(30); - prepared.data.vocab_data.tokenizer_model_id = emel::model::data::tokenizer_model::BPE; - prepared.data.vocab_data.tokenizer_pre_id = emel::model::data::tokenizer_pre::GPT2; + prepared.data.vocab_data.tokenizer_model_id = + emel::model::data::tokenizer_model::BPE; + prepared.data.vocab_data.tokenizer_pre_id = + emel::model::data::tokenizer_pre::GPT2; prepared.data.vocab_data.ignore_merges = true; prepared.hello_id = add_token(prepared.data.vocab_data, "hello"); prepared.world_id = add_token(prepared.data.vocab_data, "world"); - prepared.data.params.n_vocab = static_cast(prepared.data.vocab_data.n_tokens); + prepared.data.params.n_vocab = + static_cast(prepared.data.vocab_data.n_tokens); prepared.data.params.n_embd = k_n_embd; prepared.data.params.n_head = 1; prepared.data.params.n_head_kv = 1; @@ -491,35 +529,39 @@ void build_lfm2_quantized_contract_prepared_model(prepared_model & prepared) { std::memcpy(prepared.data.architecture_name.data(), "lfm2", 4u); uint32_t tensor_index = 0u; - const auto add_name = [&](emel::model::data::tensor_record & tensor, const std::string_view name) { + const auto add_name = [&](emel::model::data::tensor_record &tensor, + const std::string_view name) { tensor.name_offset = prepared.data.name_bytes_used; tensor.name_length = static_cast(name.size()); - std::memcpy(prepared.data.name_storage.data() + prepared.data.name_bytes_used, - name.data(), - name.size()); + std::memcpy(prepared.data.name_storage.data() + + prepared.data.name_bytes_used, + name.data(), name.size()); prepared.data.name_bytes_used += static_cast(name.size()); }; const auto add_vector = [&](const std::string_view name, const int32_t cols) { - auto & tensor = prepared.data.tensors[tensor_index++]; + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.emplace_back(static_cast(cols), 1.0f); tensor.type = static_cast(emel::kernel::event::dtype::f32); tensor.n_dims = 1; tensor.dims[0] = cols; tensor.data = prepared.tensor_storage.back().data(); - tensor.data_size = static_cast(prepared.tensor_storage.back().size() * sizeof(float)); + tensor.data_size = static_cast( + prepared.tensor_storage.back().size() * sizeof(float)); }; - const auto add_matrix = [&](const std::string_view name, const int32_t rows, const int32_t cols) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const auto add_matrix = [&](const std::string_view name, const int32_t rows, + const int32_t cols) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); - prepared.tensor_storage.emplace_back(static_cast(rows) * static_cast(cols), - 0.0f); + prepared.tensor_storage.emplace_back( + static_cast(rows) * static_cast(cols), 0.0f); tensor.type = static_cast(emel::kernel::event::dtype::f32); tensor.n_dims = 2; tensor.dims[0] = cols; tensor.dims[1] = rows; tensor.data = prepared.tensor_storage.back().data(); - tensor.data_size = static_cast(prepared.tensor_storage.back().size() * sizeof(float)); + tensor.data_size = static_cast( + prepared.tensor_storage.back().size() * sizeof(float)); }; const auto add_common_block = [&](const int32_t block_index) { const std::string prefix = "blk." + std::to_string(block_index) + "."; @@ -550,14 +592,17 @@ void build_lfm2_quantized_contract_prepared_model(prepared_model & prepared) { prepared.data.n_tensors = tensor_index; } -void build_qwen3_prepared_model(prepared_model & prepared) { +void build_qwen3_prepared_model(prepared_model &prepared) { prepared.tensor_storage.reserve(14); - prepared.data.vocab_data.tokenizer_model_id = emel::model::data::tokenizer_model::BPE; - prepared.data.vocab_data.tokenizer_pre_id = emel::model::data::tokenizer_pre::GPT2; + prepared.data.vocab_data.tokenizer_model_id = + emel::model::data::tokenizer_model::BPE; + prepared.data.vocab_data.tokenizer_pre_id = + emel::model::data::tokenizer_pre::GPT2; prepared.data.vocab_data.ignore_merges = true; prepared.hello_id = add_token(prepared.data.vocab_data, "hello"); prepared.world_id = add_token(prepared.data.vocab_data, "world"); - prepared.data.params.n_vocab = static_cast(prepared.data.vocab_data.n_tokens); + prepared.data.params.n_vocab = + static_cast(prepared.data.vocab_data.n_tokens); prepared.data.params.n_embd = 4; prepared.data.params.n_head = 2; prepared.data.params.n_head_kv = 2; @@ -572,16 +617,18 @@ void build_qwen3_prepared_model(prepared_model & prepared) { std::memcpy(prepared.data.architecture_name.data(), "qwen3", 5u); uint32_t tensor_index = 0u; - const auto add_name = [&](emel::model::data::tensor_record & tensor, const std::string_view name) { + const auto add_name = [&](emel::model::data::tensor_record &tensor, + const std::string_view name) { tensor.name_offset = prepared.data.name_bytes_used; tensor.name_length = static_cast(name.size()); - std::memcpy(prepared.data.name_storage.data() + prepared.data.name_bytes_used, - name.data(), - name.size()); + std::memcpy(prepared.data.name_storage.data() + + prepared.data.name_bytes_used, + name.data(), name.size()); prepared.data.name_bytes_used += static_cast(name.size()); }; - const auto add_vector = [&](const std::string_view name, const std::vector & values) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const auto add_vector = [&](const std::string_view name, + const std::vector &values) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.push_back(values); tensor.type = static_cast(emel::kernel::event::dtype::f32); @@ -590,11 +637,10 @@ void build_qwen3_prepared_model(prepared_model & prepared) { tensor.data = prepared.tensor_storage.back().data(); tensor.data_size = static_cast(values.size() * sizeof(float)); }; - const auto add_matrix = [&](const std::string_view name, - const int32_t rows, + const auto add_matrix = [&](const std::string_view name, const int32_t rows, const int32_t cols, - const std::vector & values) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const std::vector &values) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.push_back(values); tensor.type = static_cast(emel::kernel::event::dtype::f32); @@ -605,11 +651,11 @@ void build_qwen3_prepared_model(prepared_model & prepared) { tensor.data_size = static_cast(values.size() * sizeof(float)); }; - add_matrix("token_embd.weight", 2, 4, {1.0f, 0.0f, 0.0f, 0.0f, - 1.0f, 0.0f, 0.0f, 0.0f}); + add_matrix("token_embd.weight", 2, 4, + {1.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f}); add_vector("output_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); - add_matrix("output.weight", 2, 4, {0.0f, 0.0f, 0.0f, 0.0f, - 1.0f, 0.0f, 0.0f, 0.0f}); + add_matrix("output.weight", 2, 4, + {0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f}); add_vector("blk.0.attn_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); add_matrix("blk.0.attn_q.weight", 4, 4, std::vector(16, 0.0f)); add_matrix("blk.0.attn_k.weight", 4, 4, std::vector(16, 0.0f)); @@ -624,14 +670,17 @@ void build_qwen3_prepared_model(prepared_model & prepared) { prepared.data.n_tensors = tensor_index; } -void build_gemma4_prepared_model(prepared_model & prepared) { +void build_gemma4_prepared_model(prepared_model &prepared) { prepared.tensor_storage.reserve(13); - prepared.data.vocab_data.tokenizer_model_id = emel::model::data::tokenizer_model::BPE; - prepared.data.vocab_data.tokenizer_pre_id = emel::model::data::tokenizer_pre::GPT2; + prepared.data.vocab_data.tokenizer_model_id = + emel::model::data::tokenizer_model::BPE; + prepared.data.vocab_data.tokenizer_pre_id = + emel::model::data::tokenizer_pre::GPT2; prepared.data.vocab_data.ignore_merges = true; prepared.hello_id = add_token(prepared.data.vocab_data, "hello"); prepared.world_id = add_token(prepared.data.vocab_data, "world"); - prepared.data.params.n_vocab = static_cast(prepared.data.vocab_data.n_tokens); + prepared.data.params.n_vocab = + static_cast(prepared.data.vocab_data.n_tokens); prepared.data.params.n_embd = 4; prepared.data.params.n_embd_out = 4; prepared.data.params.n_ff = 4; @@ -651,16 +700,18 @@ void build_gemma4_prepared_model(prepared_model & prepared) { std::memcpy(prepared.data.architecture_name.data(), "gemma4", 6u); uint32_t tensor_index = 0u; - const auto add_name = [&](emel::model::data::tensor_record & tensor, const std::string_view name) { + const auto add_name = [&](emel::model::data::tensor_record &tensor, + const std::string_view name) { tensor.name_offset = prepared.data.name_bytes_used; tensor.name_length = static_cast(name.size()); - std::memcpy(prepared.data.name_storage.data() + prepared.data.name_bytes_used, - name.data(), - name.size()); + std::memcpy(prepared.data.name_storage.data() + + prepared.data.name_bytes_used, + name.data(), name.size()); prepared.data.name_bytes_used += static_cast(name.size()); }; - const auto add_vector = [&](const std::string_view name, const std::vector & values) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const auto add_vector = [&](const std::string_view name, + const std::vector &values) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.push_back(values); tensor.type = static_cast(emel::kernel::event::dtype::f32); @@ -669,11 +720,10 @@ void build_gemma4_prepared_model(prepared_model & prepared) { tensor.data = prepared.tensor_storage.back().data(); tensor.data_size = static_cast(values.size() * sizeof(float)); }; - const auto add_matrix = [&](const std::string_view name, - const int32_t rows, + const auto add_matrix = [&](const std::string_view name, const int32_t rows, const int32_t cols, - const std::vector & values) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const std::vector &values) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.push_back(values); tensor.type = static_cast(emel::kernel::event::dtype::f32); @@ -684,8 +734,8 @@ void build_gemma4_prepared_model(prepared_model & prepared) { tensor.data_size = static_cast(values.size() * sizeof(float)); }; - add_matrix("token_embd.weight", 2, 4, {1.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 1.0f, 0.0f, 0.0f}); + add_matrix("token_embd.weight", 2, 4, + {1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f}); add_vector("output_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); add_vector("blk.0.attn_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); add_matrix("blk.0.attn_q.weight", 4, 4, std::vector(16, 0.0f)); @@ -701,14 +751,17 @@ void build_gemma4_prepared_model(prepared_model & prepared) { prepared.data.n_tensors = tensor_index; } -void build_gemma4_multiwidth_prepared_model(prepared_model & prepared) { +void build_gemma4_multiwidth_prepared_model(prepared_model &prepared) { prepared.tensor_storage.reserve(5 * 13); - prepared.data.vocab_data.tokenizer_model_id = emel::model::data::tokenizer_model::BPE; - prepared.data.vocab_data.tokenizer_pre_id = emel::model::data::tokenizer_pre::GPT2; + prepared.data.vocab_data.tokenizer_model_id = + emel::model::data::tokenizer_model::BPE; + prepared.data.vocab_data.tokenizer_pre_id = + emel::model::data::tokenizer_pre::GPT2; prepared.data.vocab_data.ignore_merges = true; prepared.hello_id = add_token(prepared.data.vocab_data, "hello"); prepared.world_id = add_token(prepared.data.vocab_data, "world"); - prepared.data.params.n_vocab = static_cast(prepared.data.vocab_data.n_tokens); + prepared.data.params.n_vocab = + static_cast(prepared.data.vocab_data.n_tokens); prepared.data.params.n_embd = 4; prepared.data.params.n_embd_out = 4; prepared.data.params.n_ff = 4; @@ -731,23 +784,26 @@ void build_gemma4_multiwidth_prepared_model(prepared_model & prepared) { prepared.data.params.rope_freq_base_swa = 10000.0f; prepared.data.params.tie_word_embeddings = true; prepared.data.params.attention_sliding_window_pattern_count = 5u; - prepared.data.params.attention_sliding_window_pattern_flags = {1u, 1u, 1u, 1u, 0u}; + prepared.data.params.attention_sliding_window_pattern_flags = {1u, 1u, 1u, 1u, + 0u}; prepared.data.n_layers = 5; prepared.data.weights_data = prepared.data.tensors.data(); prepared.data.weights_size = 1u; std::memcpy(prepared.data.architecture_name.data(), "gemma4", 6u); uint32_t tensor_index = 0u; - const auto add_name = [&](emel::model::data::tensor_record & tensor, const std::string_view name) { + const auto add_name = [&](emel::model::data::tensor_record &tensor, + const std::string_view name) { tensor.name_offset = prepared.data.name_bytes_used; tensor.name_length = static_cast(name.size()); - std::memcpy(prepared.data.name_storage.data() + prepared.data.name_bytes_used, - name.data(), - name.size()); + std::memcpy(prepared.data.name_storage.data() + + prepared.data.name_bytes_used, + name.data(), name.size()); prepared.data.name_bytes_used += static_cast(name.size()); }; - const auto add_vector = [&](const std::string_view name, const std::vector & values) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const auto add_vector = [&](const std::string_view name, + const std::vector &values) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.push_back(values); tensor.type = static_cast(emel::kernel::event::dtype::f32); @@ -756,11 +812,10 @@ void build_gemma4_multiwidth_prepared_model(prepared_model & prepared) { tensor.data = prepared.tensor_storage.back().data(); tensor.data_size = static_cast(values.size() * sizeof(float)); }; - const auto add_matrix = [&](const std::string_view name, - const int32_t rows, + const auto add_matrix = [&](const std::string_view name, const int32_t rows, const int32_t cols, - const std::vector & values) { - auto & tensor = prepared.data.tensors[tensor_index++]; + const std::vector &values) { + auto &tensor = prepared.data.tensors[tensor_index++]; add_name(tensor, name); prepared.tensor_storage.push_back(values); tensor.type = static_cast(emel::kernel::event::dtype::f32); @@ -770,23 +825,30 @@ void build_gemma4_multiwidth_prepared_model(prepared_model & prepared) { tensor.data = prepared.tensor_storage.back().data(); tensor.data_size = static_cast(values.size() * sizeof(float)); }; - const auto add_block = [&](const int32_t block, const int32_t q_rows, const int32_t kv_rows) { + const auto add_block = [&](const int32_t block, const int32_t q_rows, + const int32_t kv_rows) { const std::string prefix = "blk." + std::to_string(block) + "."; add_vector(prefix + "attn_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); - add_matrix(prefix + "attn_q.weight", q_rows, 4, std::vector(q_rows * 4, 0.0f)); - add_matrix(prefix + "attn_k.weight", kv_rows, 4, std::vector(kv_rows * 4, 0.0f)); - add_matrix(prefix + "attn_v.weight", kv_rows, 4, std::vector(kv_rows * 4, 0.0f)); - add_vector(prefix + "attn_q_norm.weight", std::vector(static_cast(q_rows / 2), 1.0f)); - add_vector(prefix + "attn_k_norm.weight", std::vector(static_cast(kv_rows), 1.0f)); - add_matrix(prefix + "attn_output.weight", 4, q_rows, std::vector(q_rows * 4, 0.0f)); + add_matrix(prefix + "attn_q.weight", q_rows, 4, + std::vector(q_rows * 4, 0.0f)); + add_matrix(prefix + "attn_k.weight", kv_rows, 4, + std::vector(kv_rows * 4, 0.0f)); + add_matrix(prefix + "attn_v.weight", kv_rows, 4, + std::vector(kv_rows * 4, 0.0f)); + add_vector(prefix + "attn_q_norm.weight", + std::vector(static_cast(q_rows / 2), 1.0f)); + add_vector(prefix + "attn_k_norm.weight", + std::vector(static_cast(kv_rows), 1.0f)); + add_matrix(prefix + "attn_output.weight", 4, q_rows, + std::vector(q_rows * 4, 0.0f)); add_vector(prefix + "ffn_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); add_matrix(prefix + "ffn_gate.weight", 4, 4, std::vector(16, 0.0f)); add_matrix(prefix + "ffn_down.weight", 4, 4, std::vector(16, 0.0f)); add_matrix(prefix + "ffn_up.weight", 4, 4, std::vector(16, 0.0f)); }; - add_matrix("token_embd.weight", 2, 4, {1.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 1.0f, 0.0f, 0.0f}); + add_matrix("token_embd.weight", 2, 4, + {1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f}); add_vector("output_norm.weight", {1.0f, 1.0f, 1.0f, 1.0f}); for (int32_t block = 0; block < 4; ++block) { add_block(block, 4, 2); @@ -795,21 +857,22 @@ void build_gemma4_multiwidth_prepared_model(prepared_model & prepared) { prepared.data.n_tensors = tensor_index; } -emel::model::data & stabilize_model(prepared_model & prepared) { +emel::model::data &stabilize_model(prepared_model &prepared) { prepared.data.weights_data = prepared.data.tensors.data(); return prepared.data; } -emel::error::type build_generation_contract(prepared_model & prepared, - emel::model::generation::contract & contract_out) { - auto & model = stabilize_model(prepared); +emel::error::type +build_generation_contract(prepared_model &prepared, + emel::model::generation::contract &contract_out) { + auto &model = stabilize_model(prepared); return emel::model::generation::build_contract(model, contract_out); } -emel::model::data::tensor_record * find_tensor(prepared_model & prepared, - const std::string_view name) { +emel::model::data::tensor_record *find_tensor(prepared_model &prepared, + const std::string_view name) { for (uint32_t idx = 0; idx < prepared.data.n_tensors; ++idx) { - auto & tensor = prepared.data.tensors[idx]; + auto &tensor = prepared.data.tensors[idx]; if (emel::model::tensor_name_view(prepared.data, tensor) == name) { return &tensor; } @@ -817,10 +880,10 @@ emel::model::data::tensor_record * find_tensor(prepared_model & prepared, return nullptr; } -const emel::model::generation::quantized_stage_audit & find_stage_audit( - const emel::model::generation::quantized_path_audit & audit, - const emel::model::generation::quantized_stage_family family) { - for (const auto & stage : audit.stages) { +const emel::model::generation::quantized_stage_audit & +find_stage_audit(const emel::model::generation::quantized_path_audit &audit, + const emel::model::generation::quantized_stage_family family) { + for (const auto &stage : audit.stages) { if (stage.family == family) { return stage; } @@ -828,14 +891,15 @@ const emel::model::generation::quantized_stage_audit & find_stage_audit( return audit.stages[0]; } -void apply_flash_kv_width_mismatch(prepared_model & prepared) { - const auto tensor_name = [&](const emel::model::data::tensor_record & tensor) { - return std::string_view(prepared.data.name_storage.data() + tensor.name_offset, +void apply_flash_kv_width_mismatch(prepared_model &prepared) { + const auto tensor_name = [&](const emel::model::data::tensor_record &tensor) { + return std::string_view(prepared.data.name_storage.data() + + tensor.name_offset, tensor.name_length); }; for (uint32_t idx = 0; idx < prepared.data.n_tensors; ++idx) { - auto & tensor = prepared.data.tensors[idx]; + auto &tensor = prepared.data.tensors[idx]; const std::string_view name = tensor_name(tensor); if (name == "blk.0.attn_k.weight" || name == "blk.0.attn_v.weight") { tensor.dims[1] = 2; @@ -844,7 +908,7 @@ void apply_flash_kv_width_mismatch(prepared_model & prepared) { } } -void apply_quantized_contract_tensor_types(prepared_model & prepared) { +void apply_quantized_contract_tensor_types(prepared_model &prepared) { find_tensor(prepared, "token_embd.weight")->type = static_cast(emel::kernel::event::dtype::q2_k); find_tensor(prepared, "output.weight")->type = @@ -865,7 +929,7 @@ void apply_quantized_contract_tensor_types(prepared_model & prepared) { static_cast(emel::kernel::event::dtype::q2_k); } -void apply_qwen3_quantized_contract_tensor_types(prepared_model & prepared) { +void apply_qwen3_quantized_contract_tensor_types(prepared_model &prepared) { apply_quantized_contract_tensor_types(prepared); find_tensor(prepared, "blk.0.attn_q_norm.weight")->type = static_cast(emel::kernel::event::dtype::f32); @@ -875,11 +939,12 @@ void apply_qwen3_quantized_contract_tensor_types(prepared_model & prepared) { struct generator_fixture { static constexpr std::string_view k_phase_4_user_content = "hello"; - static constexpr std::array k_phase_4_messages = { - emel::text::formatter::chat_message{ - .role = "user", - .content = k_phase_4_user_content, - }, + static constexpr std::array + k_phase_4_messages = { + emel::text::formatter::chat_message{ + .role = "user", + .content = k_phase_4_user_content, + }, }; static constexpr bool k_phase_4_add_generation_prompt = false; static constexpr bool k_phase_4_enable_thinking = false; @@ -888,7 +953,7 @@ struct generator_fixture { prepared_model prepared = {}; emel::text::tokenizer::sm tokenizer{}; emel::text::conditioner::sm conditioner{}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; emel::model::generation::contract generation_contract = {}; std::unique_ptr generator = {}; std::array samplers = { @@ -908,10 +973,11 @@ struct generator_fixture { explicit generator_fixture( const model_variant variant = model_variant::canonical, - void * formatter_ctx = nullptr, + void *formatter_ctx = nullptr, emel::text::formatter::format_fn format_prompt = emel::text::formatter::format_raw, - const emel::memory::hybrid::kv_binding & kv_cache = {}) + const emel::memory::hybrid::kv_binding &kv_cache = {}, + const size_t matmul_lanes = 8u) : prepared() { if (variant == model_variant::quantized_contract) { build_quantized_contract_prepared_model(prepared); @@ -928,12 +994,11 @@ struct generator_fixture { if (variant == model_variant::flash_kv_width_mismatch) { apply_flash_kv_width_mismatch(prepared); } - const emel::model::data & model = stabilize_model(prepared); + const emel::model::data &model = stabilize_model(prepared); REQUIRE(build_generation_contract(prepared, generation_contract) == emel::error::type{0}); - const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( - parallel_matmul_lanes); + const auto matmul_policy = emel::kernel::matmul::make_execution_policy( + parallel_matmul_lanes, emel::kernel::detect_host_kind(), matmul_lanes); generator = std::make_unique( emel::text::generator::dependencies{ .generation_contract = generation_contract, @@ -949,22 +1014,23 @@ struct generator_fixture { world_id = prepared.world_id; } - emel::text::generator::event::initialize make_initialize( - callback_tracker & tracker, - emel::error::type * error_out = nullptr, - const emel::text::generator::selection_mode selection_mode = - emel::text::generator::selection_mode::sample_logits) { + emel::text::generator::event::initialize + make_initialize(callback_tracker &tracker, + emel::error::type *error_out = nullptr, + const emel::text::generator::selection_mode selection_mode = + emel::text::generator::selection_mode::sample_logits) { const std::span sampler_span = selection_mode == emel::text::generator::selection_mode::sample_logits ? std::span{samplers} : std::span{}; emel::text::generator::event::initialize request{ - &tokenizer, - tokenizer_bind_dispatch, - tokenizer_tokenize_dispatch, - sampler_span, + &tokenizer, + tokenizer_bind_dispatch, + tokenizer_tokenize_dispatch, + sampler_span, }; - request.preprocessor_variant = emel::text::tokenizer::preprocessor::preprocessor_kind::bpe; + request.preprocessor_variant = + emel::text::tokenizer::preprocessor::preprocessor_kind::bpe; request.encoder_variant = emel::text::encoders::encoder_kind::bpe; request.add_special = false; request.parse_special = false; @@ -975,48 +1041,54 @@ struct generator_fixture { request.block_tokens = 4; request.strip_leading_space = false; request.error_out = error_out; - request.on_done = emel::callback( + request.on_done = emel::callback( &tracker, on_initialize_done); - request.on_error = emel::callback( + request.on_error = emel::callback( &tracker, on_initialize_error); return request; } - emel::text::generator::event::generate make_generate(callback_tracker & tracker, - char * output, - size_t output_capacity, - size_t & output_length_out, - emel::error::type * error_out = nullptr) { + emel::text::generator::event::generate + make_generate(callback_tracker &tracker, char *output, size_t output_capacity, + size_t &output_length_out, + emel::error::type *error_out = nullptr) { emel::text::generator::event::generate request{ - std::span{k_phase_4_messages}, - k_phase_4_max_tokens, - std::span{output, output_capacity}, - output_length_out, + std::span{ + k_phase_4_messages}, + k_phase_4_max_tokens, + std::span{output, output_capacity}, + output_length_out, }; request.add_generation_prompt = k_phase_4_add_generation_prompt; request.enable_thinking = k_phase_4_enable_thinking; request.error_out = error_out; - request.on_done = emel::callback( + request.on_done = emel::callback( &tracker, on_generate_done); - request.on_error = emel::callback( + request.on_error = emel::callback( &tracker, on_generate_error); return request; } - emel::model::data & model() noexcept { return prepared.data; } + emel::model::data &model() noexcept { return prepared.data; } }; -} // namespace +} // namespace TEST_CASE("generator_starts_uninitialized") { auto fixture = std::make_unique(); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); } TEST_CASE("generator_initialize_reserves_lifecycle_managed_graph_tensors") { auto fixture = std::make_unique(); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::none); + emel::error::type error = + emel::error::cast(emel::text::generator::error::none); auto initialize = fixture->make_initialize(tracker, &error); REQUIRE(fixture->generator->process_event(initialize)); @@ -1040,12 +1112,12 @@ TEST_CASE("generator_initialize_reserves_lifecycle_managed_graph_tensors") { TEST_CASE("generator_dependencies_accept_custom_kv_actor") { emel::memory::test::recording_kv_actor kv{}; auto fixture = std::make_unique( - generator_fixture::model_variant::canonical, - nullptr, + generator_fixture::model_variant::canonical, nullptr, emel::text::formatter::format_raw, emel::memory::hybrid::bind_kv_actor(kv)); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type error = + emel::error::cast(emel::text::generator::error::backend); const auto request = fixture->make_initialize(tracker, &error); REQUIRE(fixture->generator->process_event(request)); @@ -1057,15 +1129,17 @@ TEST_CASE("generator_dependencies_accept_custom_kv_actor") { callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 99; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); REQUIRE(fixture->generator->process_event(generate_request)); REQUIRE(generate_tracker.generate_done_called); REQUIRE_FALSE(generate_tracker.generate_error_called); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); CHECK(kv.allocate_sequence_count == 1); CHECK(kv.allocate_slots_count == 1); CHECK(kv.capture_view_count >= 1); @@ -1076,39 +1150,59 @@ TEST_CASE("generator_rejects_generate_before_initialize") { callback_tracker tracker{}; std::array output = {}; size_t output_length = 7; - emel::error::type error = emel::error::cast(emel::text::generator::error::none); - const auto request = - fixture->make_generate(tracker, output.data(), output.size(), output_length, &error); + emel::error::type error = + emel::error::cast(emel::text::generator::error::none); + const auto request = fixture->make_generate( + tracker, output.data(), output.size(), output_length, &error); CHECK_FALSE(fixture->generator->process_event(request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(tracker.generate_done_called); CHECK(tracker.generate_error_called); - CHECK(error == emel::error::cast(emel::text::generator::error::invalid_request)); - CHECK(tracker.err == emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(error == + emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(tracker.err == + emel::error::cast(emel::text::generator::error::invalid_request)); } TEST_CASE("generator_initialize_succeeds_and_enters_ready") { auto fixture = std::make_unique(); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type error = + emel::error::cast(emel::text::generator::error::backend); const auto request = fixture->make_initialize(tracker, &error); CHECK(fixture->generator->process_event(request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK(tracker.initialize_done_called); CHECK_FALSE(tracker.initialize_error_called); CHECK(error == emel::error::cast(emel::text::generator::error::none)); } -TEST_CASE("generator_reinitialize_reprepares_backend_when_block_geometry_changes") { +TEST_CASE("generator diagnostics are available before initialization") { + auto fixture = std::make_unique(); + + const auto diagnostics = capture_generator_diagnostics(*fixture->generator); + + CHECK(fixture->generator->is( + stateforward::sml::state)); + CHECK(diagnostics.kernel_dispatch_calls == 0u); + CHECK(diagnostics.optimized_q4_dispatch_calls == 0u); +} + +TEST_CASE( + "generator_reinitialize_reprepares_backend_when_block_geometry_changes") { auto fixture = std::make_unique(); callback_tracker first_tracker{}; emel::error::type first_error = emel::error::cast(emel::text::generator::error::backend); - const auto first_request = fixture->make_initialize(first_tracker, &first_error); + const auto first_request = + fixture->make_initialize(first_tracker, &first_error); REQUIRE(fixture->generator->process_event(first_request)); - REQUIRE(fixture->generator->is(stateforward::sml::state)); + REQUIRE(fixture->generator->is( + stateforward::sml::state)); REQUIRE(first_error == emel::error::cast(emel::text::generator::error::none)); callback_tracker second_tracker{}; @@ -1119,21 +1213,26 @@ TEST_CASE("generator_reinitialize_reprepares_backend_when_block_geometry_changes second_request.block_tokens = 2; CHECK(fixture->generator->process_event(second_request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK(second_tracker.initialize_done_called); CHECK_FALSE(second_tracker.initialize_error_called); CHECK(second_error == emel::error::cast(emel::text::generator::error::none)); } -TEST_CASE("generator_initialize_accepts_explicit_preselected_argmax_mode_without_sampler_chain") { +TEST_CASE("generator_initialize_accepts_explicit_preselected_argmax_mode_" + "without_sampler_chain") { auto fixture = std::make_unique(); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type error = + emel::error::cast(emel::text::generator::error::backend); const auto request = fixture->make_initialize( - tracker, &error, emel::text::generator::selection_mode::preselected_argmax); + tracker, &error, + emel::text::generator::selection_mode::preselected_argmax); CHECK(fixture->generator->process_event(request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK(tracker.initialize_done_called); CHECK_FALSE(tracker.initialize_error_called); CHECK(error == emel::error::cast(emel::text::generator::error::none)); @@ -1142,22 +1241,28 @@ TEST_CASE("generator_initialize_accepts_explicit_preselected_argmax_mode_without TEST_CASE("generator_rejects_invalid_initialize_request") { auto fixture = std::make_unique(); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::none); + emel::error::type error = + emel::error::cast(emel::text::generator::error::none); auto request = fixture->make_initialize(tracker, &error); request.tokenizer_sm = nullptr; CHECK_FALSE(fixture->generator->process_event(request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(tracker.initialize_done_called); CHECK(tracker.initialize_error_called); - CHECK(error == emel::error::cast(emel::text::generator::error::invalid_request)); - CHECK(tracker.err == emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(error == + emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(tracker.err == + emel::error::cast(emel::text::generator::error::invalid_request)); } -TEST_CASE("generator_initialize_rejects_block_pool_exceeding_physical_capacity") { +TEST_CASE( + "generator_initialize_rejects_block_pool_exceeding_physical_capacity") { auto fixture = std::make_unique(); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::none); + emel::error::type error = + emel::error::cast(emel::text::generator::error::none); auto request = fixture->make_initialize(tracker, &error); // Fixture model n_ctx is 8 (physical capacity 8 with 4-token blocks); a // 3-block pool could hand out ids beyond physical storage, so the geometry @@ -1166,17 +1271,21 @@ TEST_CASE("generator_initialize_rejects_block_pool_exceeding_physical_capacity") request.block_tokens = 4; CHECK_FALSE(fixture->generator->process_event(request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(tracker.initialize_done_called); CHECK(tracker.initialize_error_called); - CHECK(error == emel::error::cast(emel::text::generator::error::invalid_request)); - CHECK(tracker.err == emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(error == + emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(tracker.err == + emel::error::cast(emel::text::generator::error::invalid_request)); } TEST_CASE("generator_initialize_requires_explicit_block_geometry") { auto fixture = std::make_unique(); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::none); + emel::error::type error = + emel::error::cast(emel::text::generator::error::none); auto request = fixture->make_initialize(tracker, &error); // The public contract requires explicit positive geometry; zero fields are // rejected by valid_initialize before any backend or memory dispatch. @@ -1184,26 +1293,32 @@ TEST_CASE("generator_initialize_requires_explicit_block_geometry") { request.block_tokens = 0; CHECK_FALSE(fixture->generator->process_event(request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(tracker.initialize_done_called); CHECK(tracker.initialize_error_called); - CHECK(error == emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(error == + emel::error::cast(emel::text::generator::error::invalid_request)); } TEST_CASE("generator_initialize_rejects_block_tokens_larger_than_context") { auto fixture = std::make_unique(); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::none); + emel::error::type error = + emel::error::cast(emel::text::generator::error::none); auto request = fixture->make_initialize(tracker, &error); request.max_blocks = 1; request.block_tokens = INT32_MAX; CHECK_FALSE(fixture->generator->process_event(request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(tracker.initialize_done_called); CHECK(tracker.initialize_error_called); - CHECK(error == emel::error::cast(emel::text::generator::error::invalid_request)); - CHECK(tracker.err == emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(error == + emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(tracker.err == + emel::error::cast(emel::text::generator::error::invalid_request)); } TEST_CASE("generator_requires_construction_time_dependencies") { @@ -1213,19 +1328,16 @@ TEST_CASE("generator_requires_construction_time_dependencies") { TEST_CASE("generator_initialize_rejects_default_generation_contract") { emel::text::tokenizer::sm tokenizer{}; emel::text::conditioner::sm conditioner{}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> - parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( - parallel_matmul_lanes); + emel::kernel::matmul::make_auto_execution_policy(parallel_matmul_lanes); emel::model::generation::contract generation_contract{}; - emel::text::generator::sm generator{ - emel::text::generator::dependencies{ - .generation_contract = generation_contract, - .conditioner = conditioner, - .matmul_policy = matmul_policy, - .runtime_policy = {}, - }}; + emel::text::generator::sm generator{emel::text::generator::dependencies{ + .generation_contract = generation_contract, + .conditioner = conditioner, + .matmul_policy = matmul_policy, + .runtime_policy = {}, + }}; std::array samplers = { emel::logits::sampler::fn::from(), }; @@ -1243,29 +1355,35 @@ TEST_CASE("generator_initialize_rejects_default_generation_contract") { request.max_blocks = 2; request.block_tokens = 4; request.error_out = &error; - request.on_done = - emel::callback( - &tracker, on_initialize_done); - request.on_error = - emel::callback( - &tracker, on_initialize_error); + request.on_done = emel::callback( + &tracker, on_initialize_done); + request.on_error = emel::callback( + &tracker, on_initialize_error); CHECK_FALSE(generator.process_event(request)); - CHECK(generator.is(stateforward::sml::state)); + CHECK(generator.is( + stateforward::sml::state)); CHECK_FALSE(tracker.initialize_done_called); CHECK(tracker.initialize_error_called); - CHECK(error == emel::error::cast(emel::text::generator::error::invalid_request)); - CHECK(tracker.err == emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(error == + emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(tracker.err == + emel::error::cast(emel::text::generator::error::invalid_request)); } -TEST_CASE("generator_initialize_reports_original_request_without_generation_callbacks") { +TEST_CASE("generator_initialize_reports_original_request_without_generation_" + "callbacks") { auto fixture = std::make_unique(); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type error = + emel::error::cast(emel::text::generator::error::backend); const auto request = fixture->make_initialize(tracker, &error); REQUIRE(fixture->generator->process_event(request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK(tracker.initialize_done_called); CHECK_FALSE(tracker.initialize_error_called); CHECK(tracker.initialize_request == &request); @@ -1275,23 +1393,30 @@ TEST_CASE("generator_initialize_reports_original_request_without_generation_call CHECK(error == emel::error::cast(emel::text::generator::error::none)); } -TEST_CASE("generator_initialize_can_rebind_ready_session_without_re_reserving_graph") { +TEST_CASE("generator_initialize_can_rebind_ready_session_without_re_reserving_" + "graph") { auto fixture = std::make_unique(); callback_tracker first_tracker{}; - emel::error::type first_error = emel::error::cast(emel::text::generator::error::backend); - const auto first_request = fixture->make_initialize(first_tracker, &first_error); + emel::error::type first_error = + emel::error::cast(emel::text::generator::error::backend); + const auto first_request = + fixture->make_initialize(first_tracker, &first_error); REQUIRE(fixture->generator->process_event(first_request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK(first_tracker.initialize_done_called); CHECK(first_error == emel::error::cast(emel::text::generator::error::none)); callback_tracker second_tracker{}; - emel::error::type second_error = emel::error::cast(emel::text::generator::error::backend); - const auto second_request = fixture->make_initialize(second_tracker, &second_error); + emel::error::type second_error = + emel::error::cast(emel::text::generator::error::backend); + const auto second_request = + fixture->make_initialize(second_tracker, &second_error); CHECK(fixture->generator->process_event(second_request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK(second_tracker.initialize_done_called); CHECK_FALSE(second_tracker.initialize_error_called); CHECK(second_error == emel::error::cast(emel::text::generator::error::none)); @@ -1300,23 +1425,28 @@ TEST_CASE("generator_initialize_can_rebind_ready_session_without_re_reserving_gr TEST_CASE("generator_generate_runs_native_generator_contract") { auto fixture = std::make_unique(); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 99; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); CHECK(fixture->generator->process_event(generate_request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(generate_tracker.generate_error_called); CHECK(generate_tracker.generate_done_called); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); CHECK(generate_tracker.tokens_generated == 1); CHECK(output_length == 5); CHECK(generate_tracker.output_length == 5); @@ -1333,91 +1463,160 @@ TEST_CASE("generator_generate_runs_native_generator_contract") { } } +TEST_CASE("generator unsupported lane policy stays serial through prefill and " + "decode") { + auto fixture = std::make_unique( + generator_fixture::model_variant::canonical, nullptr, + emel::text::formatter::format_raw, emel::memory::hybrid::kv_binding{}, + 3u); + callback_tracker initialize_tracker{}; + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); + REQUIRE(fixture->generator->process_event(initialize_request)); + REQUIRE(initialize_tracker.initialize_done_called); + REQUIRE(initialize_error == + emel::error::cast(emel::text::generator::error::none)); + + constexpr std::array messages = { + emel::text::formatter::chat_message{ + .role = "user", + .content = "hello hello hello hello hello hello hello hello", + }, + }; + callback_tracker generate_tracker{}; + std::array output = {}; + size_t output_length = 0u; + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); + emel::text::generator::event::generate generate_request{ + std::span{messages}, 1, + std::span{output}, output_length}; + generate_request.error_out = &generate_error; + generate_request.on_done = emel::callback( + &generate_tracker, on_generate_done); + generate_request.on_error = emel::callback( + &generate_tracker, on_generate_error); + + CHECK(fixture->generator->process_event(generate_request)); + CHECK(generate_tracker.generate_done_called); + CHECK_FALSE(generate_tracker.generate_error_called); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_tracker.tokens_generated == 1); +} + TEST_CASE("generator_qwen3_generator_initializes_and_generates_one_token") { - auto fixture = std::make_unique(generator_fixture::model_variant::qwen3_canonical); + auto fixture = std::make_unique( + generator_fixture::model_variant::qwen3_canonical); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); REQUIRE(initialize_tracker.initialize_done_called); REQUIRE_FALSE(initialize_tracker.initialize_error_called); - REQUIRE(initialize_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(initialize_error == + emel::error::cast(emel::text::generator::error::none)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 0u; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); CHECK(fixture->generator->process_event(generate_request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(generate_tracker.generate_error_called); CHECK(generate_tracker.generate_done_called); CHECK(generate_tracker.tokens_generated == 1); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); CHECK(output_length > 0u); } TEST_CASE("generator_gemma4_generator_initializes_and_generates_one_token") { - auto fixture = std::make_unique(generator_fixture::model_variant::gemma4_canonical); + auto fixture = std::make_unique( + generator_fixture::model_variant::gemma4_canonical); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); REQUIRE(initialize_tracker.initialize_done_called); REQUIRE_FALSE(initialize_tracker.initialize_error_called); - REQUIRE(initialize_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(initialize_error == + emel::error::cast(emel::text::generator::error::none)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 0u; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); CHECK(fixture->generator->process_event(generate_request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(generate_tracker.generate_error_called); CHECK(generate_tracker.generate_done_called); CHECK(generate_tracker.tokens_generated == 1); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); CHECK(output_length > 0u); } -TEST_CASE("generator_gemma4_generator_initializes_with_mixed_sliding_and_full_attention_widths") { +TEST_CASE("generator_gemma4_generator_initializes_with_mixed_sliding_and_full_" + "attention_widths") { auto fixture = std::make_unique( generator_fixture::model_variant::gemma4_multiwidth); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); - CHECK(initialize_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(initialize_error == + emel::error::cast(emel::text::generator::error::none)); } -TEST_CASE("generator_generate_f32_fixture_does_not_claim_quantized_optimized_dispatch") { +TEST_CASE("generator_generate_f32_fixture_does_not_claim_quantized_optimized_" + "dispatch") { auto fixture = std::make_unique(); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 0; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); REQUIRE(fixture->generator->process_event(generate_request)); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); const auto diagnostics = capture_generator_diagnostics(*fixture->generator); CHECK(diagnostics.optimized_q2_dispatch_calls == 0u); CHECK(diagnostics.shared_q2_dispatch_calls == 0u); @@ -1436,7 +1635,8 @@ TEST_CASE("generator_generate_f32_fixture_does_not_claim_quantized_optimized_dis CHECK(diagnostics.shared_q6_dispatch_calls == 0u); } -TEST_CASE("generator_quantized_path_audit_classifies_canonical_stage_families") { +TEST_CASE( + "generator_quantized_path_audit_classifies_canonical_stage_families") { auto prepared = std::make_unique(); build_prepared_model(*prepared); @@ -1457,17 +1657,18 @@ TEST_CASE("generator_quantized_path_audit_classifies_canonical_stage_families") REQUIRE(build_generation_contract(*prepared, contract) == emel::error::cast(emel::model::loader::error::none)); - const auto & audit = contract.quantized_audit; - const auto & token_embedding = find_stage_audit( + const auto &audit = contract.quantized_audit; + const auto &token_embedding = find_stage_audit( audit, emel::model::generation::quantized_stage_family::token_embedding); - const auto & output_norm = find_stage_audit( + const auto &output_norm = find_stage_audit( audit, emel::model::generation::quantized_stage_family::output_norm); - const auto & output = find_stage_audit( + const auto &output = find_stage_audit( audit, emel::model::generation::quantized_stage_family::output); - const auto & attention_q = find_stage_audit( + const auto &attention_q = find_stage_audit( audit, emel::model::generation::quantized_stage_family::attention_q); - const auto & feed_forward_norm = find_stage_audit( - audit, emel::model::generation::quantized_stage_family::feed_forward_norm); + const auto &feed_forward_norm = find_stage_audit( + audit, + emel::model::generation::quantized_stage_family::feed_forward_norm); CHECK(token_embedding.contract == emel::model::generation::quantized_contract_kind:: @@ -1484,7 +1685,8 @@ TEST_CASE("generator_quantized_path_audit_classifies_canonical_stage_families") approved_dense_f32_by_contract); } -TEST_CASE("generator_qwen3_quantized_path_audit_publishes_explicit_norm_stage_families") { +TEST_CASE("generator_qwen3_quantized_path_audit_publishes_explicit_norm_stage_" + "families") { auto prepared = std::make_unique(); build_qwen3_quantized_contract_prepared_model(*prepared, true, true); apply_qwen3_quantized_contract_tensor_types(*prepared); @@ -1493,12 +1695,12 @@ TEST_CASE("generator_qwen3_quantized_path_audit_publishes_explicit_norm_stage_fa REQUIRE(build_generation_contract(*prepared, contract) == emel::error::cast(emel::model::loader::error::none)); - const auto & audit = contract.quantized_audit; - const auto & attention_q_norm = find_stage_audit( + const auto &audit = contract.quantized_audit; + const auto &attention_q_norm = find_stage_audit( audit, emel::model::generation::quantized_stage_family::attention_q_norm); - const auto & attention_k_norm = find_stage_audit( + const auto &attention_k_norm = find_stage_audit( audit, emel::model::generation::quantized_stage_family::attention_k_norm); - const auto & attention_q = find_stage_audit( + const auto &attention_q = find_stage_audit( audit, emel::model::generation::quantized_stage_family::attention_q); CHECK(attention_q_norm.contract == @@ -1511,7 +1713,8 @@ TEST_CASE("generator_qwen3_quantized_path_audit_publishes_explicit_norm_stage_fa emel::model::generation::quantized_contract_kind::native_quantized); } -TEST_CASE("generator_qwen3_quantized_path_audit_requires_attention_q_norm_tensor") { +TEST_CASE( + "generator_qwen3_quantized_path_audit_requires_attention_q_norm_tensor") { auto prepared = std::make_unique(); build_qwen3_quantized_contract_prepared_model(*prepared, false, true); @@ -1520,7 +1723,8 @@ TEST_CASE("generator_qwen3_quantized_path_audit_requires_attention_q_norm_tensor emel::error::cast(emel::model::loader::error::model_invalid)); } -TEST_CASE("generator_qwen3_quantized_path_audit_requires_attention_k_norm_tensor") { +TEST_CASE( + "generator_qwen3_quantized_path_audit_requires_attention_k_norm_tensor") { auto prepared = std::make_unique(); build_qwen3_quantized_contract_prepared_model(*prepared, true, false); @@ -1529,19 +1733,21 @@ TEST_CASE("generator_qwen3_quantized_path_audit_requires_attention_k_norm_tensor emel::error::cast(emel::model::loader::error::model_invalid)); } -TEST_CASE("generator_quantized_path_audit_marks_unsupported_quantized_stage_no_claim") { +TEST_CASE("generator_quantized_path_audit_marks_unsupported_quantized_stage_no_" + "claim") { auto prepared = std::make_unique(); build_prepared_model(*prepared); REQUIRE(find_tensor(*prepared, "blk.0.attn_q.weight") != nullptr); - find_tensor(*prepared, "blk.0.attn_q.weight")->type = emel::kernel::detail::dtype_q5_1; + find_tensor(*prepared, "blk.0.attn_q.weight")->type = + emel::kernel::detail::dtype_q5_1; emel::model::generation::contract contract{}; REQUIRE(build_generation_contract(*prepared, contract) == emel::error::cast(emel::model::loader::error::none)); - const auto & audit = contract.quantized_audit; - const auto & attention_q = find_stage_audit( + const auto &audit = contract.quantized_audit; + const auto &attention_q = find_stage_audit( audit, emel::model::generation::quantized_stage_family::attention_q); CHECK(attention_q.tensor_type == emel::kernel::detail::dtype_q5_1); @@ -1550,7 +1756,8 @@ TEST_CASE("generator_quantized_path_audit_marks_unsupported_quantized_stage_no_c CHECK(attention_q.consistent_across_layers); } -TEST_CASE("generator_quantized_path_audit_marks_nonvector_f32_stage_disallowed_fallback") { +TEST_CASE("generator_quantized_path_audit_marks_nonvector_f32_stage_disallowed_" + "fallback") { auto prepared = std::make_unique(); build_prepared_model(*prepared); @@ -1558,20 +1765,23 @@ TEST_CASE("generator_quantized_path_audit_marks_nonvector_f32_stage_disallowed_f REQUIRE(find_tensor(*prepared, "output.weight") != nullptr); REQUIRE(find_tensor(*prepared, "blk.0.attn_q.weight") != nullptr); - find_tensor(*prepared, "token_embd.weight")->type = emel::kernel::detail::dtype_f32; - find_tensor(*prepared, "output.weight")->type = emel::kernel::detail::dtype_f32; - find_tensor(*prepared, "blk.0.attn_q.weight")->type = emel::kernel::detail::dtype_f32; + find_tensor(*prepared, "token_embd.weight")->type = + emel::kernel::detail::dtype_f32; + find_tensor(*prepared, "output.weight")->type = + emel::kernel::detail::dtype_f32; + find_tensor(*prepared, "blk.0.attn_q.weight")->type = + emel::kernel::detail::dtype_f32; emel::model::generation::contract contract{}; REQUIRE(build_generation_contract(*prepared, contract) == emel::error::cast(emel::model::loader::error::none)); - const auto & audit = contract.quantized_audit; - const auto & token_embedding = find_stage_audit( + const auto &audit = contract.quantized_audit; + const auto &token_embedding = find_stage_audit( audit, emel::model::generation::quantized_stage_family::token_embedding); - const auto & output = find_stage_audit( + const auto &output = find_stage_audit( audit, emel::model::generation::quantized_stage_family::output); - const auto & attention_q = find_stage_audit( + const auto &attention_q = find_stage_audit( audit, emel::model::generation::quantized_stage_family::attention_q); CHECK(token_embedding.contract == @@ -1583,7 +1793,8 @@ TEST_CASE("generator_quantized_path_audit_marks_nonvector_f32_stage_disallowed_f emel::model::generation::quantized_contract_kind::disallowed_fallback); } -TEST_CASE("generator_lfm2_quantized_path_audit_accepts_hybrid_native_quantized_mix") { +TEST_CASE( + "generator_lfm2_quantized_path_audit_accepts_hybrid_native_quantized_mix") { auto prepared = std::make_unique(); build_lfm2_quantized_contract_prepared_model(*prepared); @@ -1620,15 +1831,16 @@ TEST_CASE("generator_lfm2_quantized_path_audit_accepts_hybrid_native_quantized_m REQUIRE(build_generation_contract(*prepared, contract) == emel::error::cast(emel::model::loader::error::none)); - const auto & audit = contract.quantized_audit; - const auto & token_embedding = find_stage_audit( + const auto &audit = contract.quantized_audit; + const auto &token_embedding = find_stage_audit( audit, emel::model::generation::quantized_stage_family::token_embedding); - const auto & attention_q = find_stage_audit( + const auto &attention_q = find_stage_audit( audit, emel::model::generation::quantized_stage_family::attention_q); - const auto & attention_q_norm = find_stage_audit( + const auto &attention_q_norm = find_stage_audit( audit, emel::model::generation::quantized_stage_family::attention_q_norm); - const auto & feed_forward_down = find_stage_audit( - audit, emel::model::generation::quantized_stage_family::feed_forward_down); + const auto &feed_forward_down = find_stage_audit( + audit, + emel::model::generation::quantized_stage_family::feed_forward_down); CHECK(token_embedding.contract == emel::model::generation::quantized_contract_kind:: @@ -1644,11 +1856,13 @@ TEST_CASE("generator_lfm2_quantized_path_audit_accepts_hybrid_native_quantized_m CHECK_FALSE(feed_forward_down.consistent_across_layers); } -TEST_CASE("generator_initialize_quantized_contract_fixture_reports_zero_disallowed_fallback_stages") { +TEST_CASE("generator_initialize_quantized_contract_fixture_reports_zero_" + "disallowed_fallback_stages") { auto fixture = std::make_unique( generator_fixture::model_variant::quantized_contract); callback_tracker tracker{}; - emel::error::type error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type error = + emel::error::cast(emel::text::generator::error::backend); const auto initialize = fixture->make_initialize(tracker, &error); REQUIRE(fixture->generator->process_event(initialize)); @@ -1660,28 +1874,34 @@ TEST_CASE("generator_initialize_quantized_contract_fixture_reports_zero_disallow CHECK(diagnostics.explicit_no_claim_stage_count == 0u); } -TEST_CASE("generator_generate_quantized_contract_fixture_preserves_zero_disallowed_fallback") { +TEST_CASE("generator_generate_quantized_contract_fixture_preserves_zero_" + "disallowed_fallback") { auto fixture = std::make_unique( generator_fixture::model_variant::quantized_contract); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); - REQUIRE(initialize_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(initialize_error == + emel::error::cast(emel::text::generator::error::none)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 0; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); REQUIRE(fixture->generator->process_event(generate_request)); CHECK_FALSE(generate_tracker.generate_error_called); CHECK(generate_tracker.generate_done_called); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); const auto diagnostics = capture_generator_diagnostics(*fixture->generator); CHECK(diagnostics.native_quantized_stage_count == 8u); CHECK(diagnostics.approved_dense_f32_stage_count == 4u); @@ -1691,11 +1911,14 @@ TEST_CASE("generator_generate_quantized_contract_fixture_preserves_zero_disallow CHECK(diagnostics.optimized_q6_vector_dispatch_calls > 0u); CHECK(diagnostics.optimized_q6_vector_packed_dispatch_calls > 0u); CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_dispatch_calls > 0u); -#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#if defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_MATMUL_INT8) CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_dispatch_calls > 0u); - CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls > 0u); + CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls > + 0u); #else - CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls == 0u); + CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls == + 0u); CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_dispatch_calls == 0u); #endif } else if (host_is_x86_64()) { @@ -1709,89 +1932,120 @@ TEST_CASE("generator_generate_quantized_contract_fixture_preserves_zero_disallow CHECK(diagnostics.optimized_q6_vector_packed_dispatch_calls == 0u); CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_dispatch_calls == 0u); CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls == 0u); + CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls == + 0u); } else { CHECK(diagnostics.optimized_q6_vector_dispatch_calls == 0u); CHECK(diagnostics.optimized_q6_vector_packed_dispatch_calls == 0u); CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_dispatch_calls == 0u); CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls == 0u); + CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_i8mm_dispatch_calls == + 0u); } } -TEST_CASE("generator_generate_quantized_contract_fixture_supports_explicit_preselected_argmax_mode") { +TEST_CASE("generator_generate_quantized_contract_fixture_supports_explicit_" + "preselected_argmax_mode") { auto fixture = std::make_unique( generator_fixture::model_variant::quantized_contract); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); const auto initialize_request = fixture->make_initialize( - initialize_tracker, - &initialize_error, + initialize_tracker, &initialize_error, emel::text::generator::selection_mode::preselected_argmax); auto long_initialize_request = initialize_request; long_initialize_request.max_generated_tokens = 16; long_initialize_request.max_blocks = 2; REQUIRE(fixture->generator->process_event(long_initialize_request)); - REQUIRE(initialize_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(initialize_error == + emel::error::cast(emel::text::generator::error::none)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 0; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); REQUIRE(fixture->generator->process_event(generate_request)); CHECK_FALSE(generate_tracker.generate_error_called); CHECK(generate_tracker.generate_done_called); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); CHECK(output_length > 0u); const auto diagnostics = capture_generator_diagnostics(*fixture->generator); if (host_is_aarch64()) { CHECK(diagnostics.optimized_q6_vector_argmax_dispatch_calls > 0u); -#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) - CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls > 0u); -#elif defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) - CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls > 0u); - CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls == 0u); +#if defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_MATMUL_INT8) + CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls == + 0u); + CHECK(diagnostics + .optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls == + 0u); + CHECK( + diagnostics.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls > + 0u); +#elif defined(__aarch64__) && defined(__ARM_NEON) && \ + defined(__ARM_FEATURE_DOTPROD) + CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls > + 0u); + CHECK(diagnostics + .optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls == + 0u); + CHECK(diagnostics + .optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls == + 0u); #else - CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls == 0u); + CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls == + 0u); + CHECK(diagnostics + .optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls == + 0u); + CHECK(diagnostics + .optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls == + 0u); #endif } else { CHECK(diagnostics.optimized_q6_vector_argmax_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls == 0u); - CHECK(diagnostics.optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls == 0u); + CHECK(diagnostics.optimized_q6_vector_packed_q8_rhs_argmax_dispatch_calls == + 0u); + CHECK(diagnostics + .optimized_q6_vector_prepared_q8_rhs_argmax_i8mm_dispatch_calls == + 0u); + CHECK(diagnostics + .optimized_q6_vector_q8_argmax_prepared_i8mm_dispatch_calls == + 0u); } } -TEST_CASE( - "generator_generate_quantized_contract_fixture_allows_repeated_explicit_preselected_argmax_generates") { +TEST_CASE("generator_generate_quantized_contract_fixture_allows_repeated_" + "explicit_preselected_argmax_generates") { auto fixture = std::make_unique( generator_fixture::model_variant::quantized_contract); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); const auto initialize_request = fixture->make_initialize( - initialize_tracker, - &initialize_error, + initialize_tracker, &initialize_error, emel::text::generator::selection_mode::preselected_argmax); REQUIRE(fixture->generator->process_event(initialize_request)); - REQUIRE(initialize_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(initialize_error == + emel::error::cast(emel::text::generator::error::none)); callback_tracker first_tracker{}; std::array first_output = {}; size_t first_output_length = 0; - emel::error::type first_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type first_error = + emel::error::cast(emel::text::generator::error::backend); const auto first_request = fixture->make_generate( - first_tracker, first_output.data(), first_output.size(), first_output_length, &first_error); + first_tracker, first_output.data(), first_output.size(), + first_output_length, &first_error); REQUIRE(fixture->generator->process_event(first_request)); REQUIRE_FALSE(first_tracker.generate_error_called); @@ -1802,13 +2056,11 @@ TEST_CASE( callback_tracker second_tracker{}; std::array second_output = {}; size_t second_output_length = 0; - emel::error::type second_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type second_error = + emel::error::cast(emel::text::generator::error::backend); const auto second_request = fixture->make_generate( - second_tracker, - second_output.data(), - second_output.size(), - second_output_length, - &second_error); + second_tracker, second_output.data(), second_output.size(), + second_output_length, &second_error); CHECK(fixture->generator->process_event(second_request)); CHECK_FALSE(second_tracker.generate_error_called); @@ -1817,38 +2069,40 @@ TEST_CASE( CHECK(second_output_length > 0u); } -TEST_CASE( - "generator_generate_quantized_contract_fixture_allows_repeated_multi_token_explicit_preselected_argmax_generates") { +TEST_CASE("generator_generate_quantized_contract_fixture_allows_repeated_multi_" + "token_explicit_preselected_argmax_generates") { auto fixture = std::make_unique( generator_fixture::model_variant::quantized_contract); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); const auto initialize_request = fixture->make_initialize( - initialize_tracker, - &initialize_error, + initialize_tracker, &initialize_error, emel::text::generator::selection_mode::preselected_argmax); REQUIRE(fixture->generator->process_event(initialize_request)); - REQUIRE(initialize_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(initialize_error == + emel::error::cast(emel::text::generator::error::none)); const auto make_long_generate = - [](callback_tracker & tracker, - std::array & output, - size_t & output_length, - emel::error::type * error_out) { + [](callback_tracker &tracker, std::array &output, + size_t &output_length, emel::error::type *error_out) { emel::text::generator::event::generate request{ - std::span{ - generator_fixture::k_phase_4_messages}, - 2, - std::span{output.data(), output.size()}, - output_length, + std::span{ + generator_fixture::k_phase_4_messages}, + 2, + std::span{output.data(), output.size()}, + output_length, }; - request.add_generation_prompt = generator_fixture::k_phase_4_add_generation_prompt; + request.add_generation_prompt = + generator_fixture::k_phase_4_add_generation_prompt; request.enable_thinking = generator_fixture::k_phase_4_enable_thinking; request.error_out = error_out; - request.on_done = emel::callback( + request.on_done = emel::callback( &tracker, on_generate_done); - request.on_error = emel::callback( + request.on_error = emel::callback( &tracker, on_generate_error); return request; }; @@ -1856,9 +2110,10 @@ TEST_CASE( callback_tracker first_tracker{}; std::array first_output = {}; size_t first_output_length = 0; - emel::error::type first_error = emel::error::cast(emel::text::generator::error::backend); - const auto first_request = - make_long_generate(first_tracker, first_output, first_output_length, &first_error); + emel::error::type first_error = + emel::error::cast(emel::text::generator::error::backend); + const auto first_request = make_long_generate( + first_tracker, first_output, first_output_length, &first_error); REQUIRE(fixture->generator->process_event(first_request)); REQUIRE_FALSE(first_tracker.generate_error_called); @@ -1869,9 +2124,10 @@ TEST_CASE( callback_tracker second_tracker{}; std::array second_output = {}; size_t second_output_length = 0; - emel::error::type second_error = emel::error::cast(emel::text::generator::error::backend); - const auto second_request = - make_long_generate(second_tracker, second_output, second_output_length, &second_error); + emel::error::type second_error = + emel::error::cast(emel::text::generator::error::backend); + const auto second_request = make_long_generate( + second_tracker, second_output, second_output_length, &second_error); CHECK(fixture->generator->process_event(second_request)); CHECK_FALSE(second_tracker.generate_error_called); @@ -1883,20 +2139,25 @@ TEST_CASE( TEST_CASE("generator structured message request pins the phase 4 contract") { auto fixture = std::make_unique(); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 0; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); - CHECK(generate_request.messages.size() == generator_fixture::k_phase_4_messages.size()); - CHECK(generate_request.messages[0].role == generator_fixture::k_phase_4_messages[0].role); + CHECK(generate_request.messages.size() == + generator_fixture::k_phase_4_messages.size()); + CHECK(generate_request.messages[0].role == + generator_fixture::k_phase_4_messages[0].role); CHECK(generate_request.messages[0].content == generator_fixture::k_phase_4_messages[0].content); CHECK(generate_request.add_generation_prompt == @@ -1910,43 +2171,51 @@ TEST_CASE("generator structured message request pins the phase 4 contract") { CHECK_FALSE(generate_tracker.generate_error_called); CHECK(generate_tracker.generate_done_called); CHECK(generate_tracker.generate_request == &generate_request); - CHECK(generate_tracker.tokens_generated == generator_fixture::k_phase_4_max_tokens); + CHECK(generate_tracker.tokens_generated == + generator_fixture::k_phase_4_max_tokens); CHECK(generate_tracker.output_length == 5); CHECK(output_length == generate_tracker.output_length); CHECK(std::string_view(output.data(), output_length) == "world"); } -TEST_CASE("generator structured message request reaches the conditioner formatter") { - static constexpr std::array k_messages = { - emel::text::formatter::chat_message{ - .role = "system", - .content = "policy", - }, - emel::text::formatter::chat_message{ - .role = "user", - .content = "hello", - }, - }; +TEST_CASE( + "generator structured message request reaches the conditioner formatter") { + static constexpr std::array + k_messages = { + emel::text::formatter::chat_message{ + .role = "system", + .content = "policy", + }, + emel::text::formatter::chat_message{ + .role = "user", + .content = "hello", + }, + }; checked_formatter_ctx formatter_ctx{ - .expected_messages = std::span{k_messages}, + .expected_messages = + std::span{k_messages}, .expected_add_generation_prompt = true, .expected_enable_thinking = false, .formatted = "hello", .seen = false, }; - auto fixture = std::make_unique(generator_fixture::model_variant::canonical, - &formatter_ctx, - format_checked_messages); + auto fixture = std::make_unique( + generator_fixture::model_variant::canonical, &formatter_ctx, + format_checked_messages); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); - REQUIRE(initialize_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(initialize_error == + emel::error::cast(emel::text::generator::error::none)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 0; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::backend); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::backend); emel::text::generator::event::generate generate_request{ std::span{k_messages}, 1, @@ -1956,17 +2225,19 @@ TEST_CASE("generator structured message request reaches the conditioner formatte generate_request.add_generation_prompt = true; generate_request.enable_thinking = false; generate_request.error_out = &generate_error; - generate_request.on_done = emel::callback( + generate_request.on_done = emel::callback( &generate_tracker, on_generate_done); - generate_request.on_error = - emel::callback( - &generate_tracker, on_generate_error); + generate_request.on_error = emel::callback( + &generate_tracker, on_generate_error); REQUIRE(fixture->generator->process_event(generate_request)); CHECK(formatter_ctx.seen); CHECK_FALSE(generate_tracker.generate_error_called); CHECK(generate_tracker.generate_done_called); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); CHECK(generate_tracker.tokens_generated == 1); CHECK(std::string_view(output.data(), output_length) == "world"); } @@ -1974,51 +2245,62 @@ TEST_CASE("generator structured message request reaches the conditioner formatte TEST_CASE("generator_generate_reports_bounded_output_buffer_errors") { auto fixture = std::make_unique(); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 17; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::none); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::none); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); CHECK_FALSE(fixture->generator->process_event(generate_request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK_FALSE(generate_tracker.generate_done_called); CHECK(generate_tracker.generate_error_called); CHECK(generate_tracker.generate_request == &generate_request); CHECK(generate_tracker.tokens_generated == 0); CHECK(generate_tracker.output_length == 0); CHECK(output_length == 0); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::invalid_request)); - CHECK(generate_tracker.err == emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::invalid_request)); + CHECK(generate_tracker.err == + emel::error::cast(emel::text::generator::error::invalid_request)); } TEST_CASE("generator_generate_uses_nonflash_runtime_without_claiming_flash") { auto fixture = std::make_unique( generator_fixture::model_variant::flash_kv_width_mismatch); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); callback_tracker generate_tracker{}; std::array output = {}; size_t output_length = 0; - emel::error::type generate_error = emel::error::cast(emel::text::generator::error::none); + emel::error::type generate_error = + emel::error::cast(emel::text::generator::error::none); const auto generate_request = - fixture->make_generate(generate_tracker, output.data(), output.size(), output_length, - &generate_error); + fixture->make_generate(generate_tracker, output.data(), output.size(), + output_length, &generate_error); CHECK(fixture->generator->process_event(generate_request)); - CHECK(fixture->generator->is(stateforward::sml::state)); + CHECK(fixture->generator->is( + stateforward::sml::state)); CHECK(generate_tracker.generate_done_called); CHECK_FALSE(generate_tracker.generate_error_called); - CHECK(generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(generate_error == + emel::error::cast(emel::text::generator::error::none)); CHECK(generate_tracker.tokens_generated == 1); CHECK(output_length == 5); CHECK(generate_tracker.output_length == 5); @@ -2032,43 +2314,51 @@ TEST_CASE("generator_generate_uses_nonflash_runtime_without_claiming_flash") { TEST_CASE("generator_generate_multiple_tokens_and_resets_sequence_on_reuse") { auto fixture = std::make_unique(); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + const auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); REQUIRE(fixture->generator->process_event(initialize_request)); callback_tracker first_tracker{}; std::array first_output = {}; size_t first_output_length = 0; - emel::error::type first_error = emel::error::cast(emel::text::generator::error::backend); - auto first_request = - fixture->make_generate(first_tracker, first_output.data(), first_output.size(), - first_output_length, &first_error); + emel::error::type first_error = + emel::error::cast(emel::text::generator::error::backend); + auto first_request = fixture->make_generate( + first_tracker, first_output.data(), first_output.size(), + first_output_length, &first_error); first_request.max_tokens = 2; CHECK(fixture->generator->process_event(first_request)); CHECK(first_error == emel::error::cast(emel::text::generator::error::none)); CHECK(first_tracker.tokens_generated == 2); - CHECK(std::string_view(first_output.data(), first_output_length) == "worldworld"); + CHECK(std::string_view(first_output.data(), first_output_length) == + "worldworld"); callback_tracker second_tracker{}; std::array second_output = {}; size_t second_output_length = 0; - emel::error::type second_error = emel::error::cast(emel::text::generator::error::backend); - const auto second_request = - fixture->make_generate(second_tracker, second_output.data(), second_output.size(), - second_output_length, &second_error); + emel::error::type second_error = + emel::error::cast(emel::text::generator::error::backend); + const auto second_request = fixture->make_generate( + second_tracker, second_output.data(), second_output.size(), + second_output_length, &second_error); CHECK(fixture->generator->process_event(second_request)); CHECK(second_error == emel::error::cast(emel::text::generator::error::none)); CHECK(second_tracker.tokens_generated == 1); - CHECK(std::string_view(second_output.data(), second_output_length) == "world"); + CHECK(std::string_view(second_output.data(), second_output_length) == + "world"); } TEST_CASE("generator_resets_renderer_stop_state_when_reusing_a_sequence") { auto fixture = std::make_unique(); callback_tracker initialize_tracker{}; - emel::error::type initialize_error = emel::error::cast(emel::text::generator::error::backend); - auto initialize_request = fixture->make_initialize(initialize_tracker, &initialize_error); + emel::error::type initialize_error = + emel::error::cast(emel::text::generator::error::backend); + auto initialize_request = + fixture->make_initialize(initialize_tracker, &initialize_error); const std::array stops = {"worldworld"}; initialize_request.stop_sequences = stops; REQUIRE(fixture->generator->process_event(initialize_request)); @@ -2076,10 +2366,11 @@ TEST_CASE("generator_resets_renderer_stop_state_when_reusing_a_sequence") { callback_tracker first_tracker{}; std::array first_output = {}; size_t first_output_length = 0; - emel::error::type first_error = emel::error::cast(emel::text::generator::error::backend); - auto first_request = - fixture->make_generate(first_tracker, first_output.data(), first_output.size(), - first_output_length, &first_error); + emel::error::type first_error = + emel::error::cast(emel::text::generator::error::backend); + auto first_request = fixture->make_generate( + first_tracker, first_output.data(), first_output.size(), + first_output_length, &first_error); first_request.max_tokens = 2; REQUIRE(fixture->generator->process_event(first_request)); @@ -2090,10 +2381,11 @@ TEST_CASE("generator_resets_renderer_stop_state_when_reusing_a_sequence") { callback_tracker second_tracker{}; std::array second_output = {}; size_t second_output_length = 0; - emel::error::type second_error = emel::error::cast(emel::text::generator::error::backend); - auto second_request = - fixture->make_generate(second_tracker, second_output.data(), second_output.size(), - second_output_length, &second_error); + emel::error::type second_error = + emel::error::cast(emel::text::generator::error::backend); + auto second_request = fixture->make_generate( + second_tracker, second_output.data(), second_output.size(), + second_output_length, &second_error); second_request.max_tokens = 2; CHECK(fixture->generator->process_event(second_request)); @@ -2102,45 +2394,52 @@ TEST_CASE("generator_resets_renderer_stop_state_when_reusing_a_sequence") { CHECK(second_output_length == 0); } -TEST_CASE("generator_reinitialize_clears_lifecycle_publish_state_before_next_generate") { +TEST_CASE("generator_reinitialize_clears_lifecycle_publish_state_before_next_" + "generate") { auto fixture = std::make_unique(); callback_tracker first_initialize_tracker{}; emel::error::type first_initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto first_initialize = - fixture->make_initialize(first_initialize_tracker, &first_initialize_error); + const auto first_initialize = fixture->make_initialize( + first_initialize_tracker, &first_initialize_error); REQUIRE(fixture->generator->process_event(first_initialize)); callback_tracker first_generate_tracker{}; std::array first_output = {}; size_t first_output_length = 0; - emel::error::type first_generate_error = emel::error::cast(emel::text::generator::error::backend); - const auto first_generate = - fixture->make_generate(first_generate_tracker, first_output.data(), first_output.size(), - first_output_length, &first_generate_error); + emel::error::type first_generate_error = + emel::error::cast(emel::text::generator::error::backend); + const auto first_generate = fixture->make_generate( + first_generate_tracker, first_output.data(), first_output.size(), + first_output_length, &first_generate_error); REQUIRE(fixture->generator->process_event(first_generate)); - REQUIRE(first_generate_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(first_generate_error == + emel::error::cast(emel::text::generator::error::none)); callback_tracker second_initialize_tracker{}; emel::error::type second_initialize_error = emel::error::cast(emel::text::generator::error::backend); - const auto second_initialize = - fixture->make_initialize(second_initialize_tracker, &second_initialize_error); + const auto second_initialize = fixture->make_initialize( + second_initialize_tracker, &second_initialize_error); REQUIRE(fixture->generator->process_event(second_initialize)); - REQUIRE(second_initialize_error == emel::error::cast(emel::text::generator::error::none)); + REQUIRE(second_initialize_error == + emel::error::cast(emel::text::generator::error::none)); callback_tracker second_generate_tracker{}; std::array second_output = {}; size_t second_output_length = 0; - emel::error::type second_generate_error = emel::error::cast(emel::text::generator::error::backend); - const auto second_generate = - fixture->make_generate(second_generate_tracker, second_output.data(), second_output.size(), - second_output_length, &second_generate_error); + emel::error::type second_generate_error = + emel::error::cast(emel::text::generator::error::backend); + const auto second_generate = fixture->make_generate( + second_generate_tracker, second_output.data(), second_output.size(), + second_output_length, &second_generate_error); CHECK(fixture->generator->process_event(second_generate)); - CHECK(second_generate_error == emel::error::cast(emel::text::generator::error::none)); + CHECK(second_generate_error == + emel::error::cast(emel::text::generator::error::none)); CHECK(second_generate_tracker.tokens_generated == 1); - CHECK(std::string_view(second_output.data(), second_output_length) == "world"); + CHECK(std::string_view(second_output.data(), second_output_length) == + "world"); } TEST_CASE("generator_docs_table_uses_typed_completion_event_names") { @@ -2152,7 +2451,8 @@ TEST_CASE("generator_docs_table_uses_typed_completion_event_names") { for_each_type(transitions{}, [&]() { using event = typename transition_t::event; - const std::string event_name = emel::docs::detail::table_event_name(); + const std::string event_name = + emel::docs::detail::table_event_name(); if (event_name == "completion") { has_initialize_completion = true; } @@ -2165,16 +2465,15 @@ TEST_CASE("generator_docs_table_uses_typed_completion_event_names") { CHECK(has_generate_completion); } -TEST_CASE("generator_sm_models_explicit_prefill_boundary_and_decode_compute_states") { +TEST_CASE( + "generator_sm_models_explicit_prefill_boundary_and_decode_compute_states") { using machine_t = stateforward::sml::sm; using states = typename machine_t::states; + CHECK(emel::detail::type_list_contains::value); CHECK(emel::detail::type_list_contains< - emel::text::generator::prefill_running, - states>::value); - CHECK(emel::detail::type_list_contains< - emel::text::generator::prefill_result_decision, - states>::value); + emel::text::generator::prefill_result_decision, states>::value); CHECK_FALSE(emel::detail::type_list_contains< emel::text::generator::prefill::contract_runtime_decision, states>::value); @@ -2187,20 +2486,20 @@ TEST_CASE("generator_sm_models_explicit_prefill_boundary_and_decode_compute_stat CHECK_FALSE(emel::detail::type_list_contains< emel::text::generator::prefill::compute_result_decision, states>::value); - CHECK(emel::detail::type_list_contains::value); CHECK(emel::detail::type_list_contains< - emel::text::generator::decode_compute_nonflash, - states>::value); + emel::text::generator::decode_compute_flash, states>::value); + CHECK(emel::detail::type_list_contains< + emel::text::generator::decode_compute_nonflash, states>::value); } TEST_CASE("generator_sm_models_explicit_initializer_boundary") { using machine_t = stateforward::sml::sm; using states = typename machine_t::states; - CHECK(emel::detail::type_list_contains::value); + CHECK(emel::detail::type_list_contains::value); CHECK(emel::detail::type_list_contains< - emel::text::generator::initializer_result_decision, - states>::value); + emel::text::generator::initializer_result_decision, states>::value); CHECK_FALSE(emel::detail::type_list_contains< emel::text::generator::initializer::binding_conditioner, states>::value); @@ -2210,7 +2509,8 @@ TEST_CASE("generator_sm_models_explicit_initializer_boundary") { } TEST_CASE("generator_initialize_public_wrapper_has_no_runtime_branch") { - std::ifstream stream(repo_root() / "src" / "emel" / "text" / "generator" / "sm.hpp"); + std::ifstream stream(repo_root() / "src" / "emel" / "text" / "generator" / + "sm.hpp"); REQUIRE(stream.good()); const std::string source((std::istreambuf_iterator(stream)), @@ -2218,7 +2518,8 @@ TEST_CASE("generator_initialize_public_wrapper_has_no_runtime_branch") { const std::string marker = "bool process_event(const event::initialize &"; const auto start = source.find(marker); REQUIRE(start != std::string::npos); - const auto end = source.find("bool process_event(const event::generate &", start); + const auto end = + source.find("bool process_event(const event::generate &", start); REQUIRE(end != std::string::npos); const auto wrapper = source.substr(start, end - start); @@ -2227,26 +2528,28 @@ TEST_CASE("generator_initialize_public_wrapper_has_no_runtime_branch") { CHECK(wrapper.find("switch") == std::string::npos); } -TEST_CASE("generator_route_guards_do_not_delegate_behavior_selection_to_detail_helpers") { - const auto read_source = [](const std::filesystem::path & path) { +TEST_CASE("generator_route_guards_do_not_delegate_behavior_selection_to_detail_" + "helpers") { + const auto read_source = [](const std::filesystem::path &path) { std::ifstream stream(path); REQUIRE(stream.good()); - return std::string( - (std::istreambuf_iterator(stream)), std::istreambuf_iterator()); + return std::string((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); }; - const std::string parent_guards = - read_source(repo_root() / "src" / "emel" / "text" / "generator" / "guards.hpp"); + const std::string parent_guards = read_source( + repo_root() / "src" / "emel" / "text" / "generator" / "guards.hpp"); const std::string prefill_guards = - read_source(repo_root() / "src" / "emel" / "text" / "generator" / "prefill" / "guards.hpp"); + read_source(repo_root() / "src" / "emel" / "text" / "generator" / + "prefill" / "guards.hpp"); const std::array forbidden_parent_patterns{ - "emel::text::generator::detail::preselected_argmax_direct_supported(", - "emel::text::generator::detail::prefill_chunk4_q8_gemm_backend_ready(", - "emel::text::generator::detail::prefill_chunk8_q8_k_backend_ready(", - "emel::text::generator::detail::prefill_chunk4_backend_ready<", - "emel::text::generator::detail::prefill_chunk8_backend_ready<", - "emel::text::generator::detail::flash_attention_supported(", - "emel::text::generator::detail::block_uses_attention(", + "emel::text::generator::detail::preselected_argmax_direct_supported(", + "emel::text::generator::detail::prefill_chunk4_q8_gemm_backend_ready(", + "emel::text::generator::detail::prefill_chunk8_q8_k_backend_ready(", + "emel::text::generator::detail::prefill_chunk4_backend_ready<", + "emel::text::generator::detail::prefill_chunk8_backend_ready<", + "emel::text::generator::detail::flash_attention_supported(", + "emel::text::generator::detail::block_uses_attention(", }; for (const auto pattern : forbidden_parent_patterns) { CAPTURE(pattern); @@ -2256,48 +2559,54 @@ TEST_CASE("generator_route_guards_do_not_delegate_behavior_selection_to_detail_h } TEST_CASE("generator_scalar_kernel_route_choice_stays_in_state_machines") { - const auto read_source = [](const std::filesystem::path & path) { + const auto read_source = [](const std::filesystem::path &path) { std::ifstream stream(path); REQUIRE(stream.good()); - return std::string( - (std::istreambuf_iterator(stream)), std::istreambuf_iterator()); + return std::string((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); }; - const std::string detail_source = - read_source(repo_root() / "src" / "emel" / "text" / "generator" / "detail.hpp"); + const std::string detail_source = read_source( + repo_root() / "src" / "emel" / "text" / "generator" / "detail.hpp"); CHECK(detail_source.find("block_uses_attention(") == std::string::npos); CHECK(detail_source.find("prefill_chunk4_q8_gemm_backend_ready(") == std::string::npos); CHECK(detail_source.find("prefill_chunk8_q8_k_backend_ready(") == std::string::npos); - CHECK(detail_source.find("prefill_chunk4_backend_ready<") == std::string::npos); - CHECK(detail_source.find("chunk4_matmul_backend_ready(") == std::string::npos); + CHECK(detail_source.find("prefill_chunk4_backend_ready<") == + std::string::npos); + CHECK(detail_source.find("chunk4_matmul_backend_ready(") == + std::string::npos); CHECK(detail_source.find("run_kernel_mode(") == std::string::npos); - CHECK(detail_source.find("run_kernel_mode_preselected_argmax(") == std::string::npos); - CHECK(detail_source.find("plan->kind == step_kind::prefill ?") == std::string::npos); + CHECK(detail_source.find("run_kernel_mode_preselected_argmax(") == + std::string::npos); + CHECK(detail_source.find("plan->kind == step_kind::prefill ?") == + std::string::npos); CHECK(detail_source.find("phase_lifecycle(") == std::string::npos); - const auto guarded_validate_sig = detail_source.find("validate_guarded_compute("); + const auto guarded_validate_sig = + detail_source.find("validate_guarded_compute("); REQUIRE(guarded_validate_sig != std::string::npos); - const auto guarded_validate_start = detail_source.find('{', guarded_validate_sig); + const auto guarded_validate_start = + detail_source.find('{', guarded_validate_sig); REQUIRE(guarded_validate_start != std::string::npos); - const auto guarded_validate_end = - detail_source.find("validate_guarded_preselected_argmax(", guarded_validate_start); + const auto guarded_validate_end = detail_source.find( + "validate_guarded_preselected_argmax(", guarded_validate_start); REQUIRE(guarded_validate_end != std::string::npos); const auto guarded_validate_callbacks = detail_source.substr(guarded_validate_start + 1, guarded_validate_end - guarded_validate_start - 1); const std::array forbidden_validate_callback_patterns{ - "request_plan(", - "check_backend(", - "expected_outputs", - "selected_token_out == nullptr", - "selected_score_out == nullptr", - "positions_count !=", - "token_count <=", - "*err_out =", - "k_error_invalid", - "return false", + "request_plan(", + "check_backend(", + "expected_outputs", + "selected_token_out == nullptr", + "selected_score_out == nullptr", + "positions_count !=", + "token_count <=", + "*err_out =", + "k_error_invalid", + "return false", }; for (const auto pattern : forbidden_validate_callback_patterns) { CAPTURE(pattern); @@ -2308,46 +2617,41 @@ TEST_CASE("generator_scalar_kernel_route_choice_stays_in_state_machines") { REQUIRE(guarded_bind_sig != std::string::npos); const auto guarded_bind_start = detail_source.find('{', guarded_bind_sig); REQUIRE(guarded_bind_start != std::string::npos); - const auto guarded_bind_end = - detail_source.find("template forbidden_bind_callback_patterns{ - "request_plan(", - "check_backend(", - "== nullptr", - "expected_outputs", - "*err_out =", - "k_error_invalid", - "return false", + "request_plan(", "check_backend(", "== nullptr", "expected_outputs", + "*err_out =", "k_error_invalid", "return false", }; for (const auto pattern : forbidden_bind_callback_patterns) { CAPTURE(pattern); CHECK(guarded_bind_callback.find(pattern) == std::string::npos); } - const auto guarded_extract_sig = detail_source.find("extract_guarded_outputs("); + const auto guarded_extract_sig = + detail_source.find("extract_guarded_outputs("); REQUIRE(guarded_extract_sig != std::string::npos); - const auto guarded_extract_start = detail_source.find('{', guarded_extract_sig); + const auto guarded_extract_start = + detail_source.find('{', guarded_extract_sig); REQUIRE(guarded_extract_start != std::string::npos); - const auto guarded_extract_end = - detail_source.find("extract_guarded_preselected_argmax(", guarded_extract_start); + const auto guarded_extract_end = detail_source.find( + "extract_guarded_preselected_argmax(", guarded_extract_start); REQUIRE(guarded_extract_end != std::string::npos); const auto guarded_extract_callbacks = detail_source.substr(guarded_extract_start + 1, guarded_extract_end - guarded_extract_start - 1); const std::array forbidden_extract_callback_patterns{ - "check_backend(", - "== nullptr", - "selected_token_out == nullptr", - "selected_score_out == nullptr", - "*err_out =", - "k_error_invalid", - "return false", - "request_plan(", + "check_backend(", + "== nullptr", + "selected_token_out == nullptr", + "selected_score_out == nullptr", + "*err_out =", + "k_error_invalid", + "return false", + "request_plan(", }; for (const auto pattern : forbidden_extract_callback_patterns) { CAPTURE(pattern); @@ -2359,32 +2663,38 @@ TEST_CASE("generator_scalar_kernel_route_choice_stays_in_state_machines") { const auto audited_end = detail_source.find("extract_guarded_outputs(", audited_start); REQUIRE(audited_end != std::string::npos); - const auto audited_wrappers = detail_source.substr(audited_start, audited_end - audited_start); + const auto audited_wrappers = + detail_source.substr(audited_start, audited_end - audited_start); const std::array forbidden_wrapper_patterns{ - "request_plan(", - "plan->kind", - "check_backend(", - "bound_ready", - "selected_token_out == nullptr", - "selected_score_out == nullptr", - "prefill_chunk4_backend_ready", - "prefill_chunk8_q8_k_backend_ready", - "*err_out =", - "k_error_invalid", + "request_plan(", + "plan->kind", + "check_backend(", + "bound_ready", + "selected_token_out == nullptr", + "selected_score_out == nullptr", + "prefill_chunk4_backend_ready", + "prefill_chunk8_q8_k_backend_ready", + "*err_out =", + "k_error_invalid", }; for (const auto pattern : forbidden_wrapper_patterns) { CAPTURE(pattern); CHECK(audited_wrappers.find(pattern) == std::string::npos); } - auto matmul_start = detail_source.find("inline bool matmul_vector(native_backend"); + auto matmul_start = + detail_source.find("inline bool matmul_vector(native_backend"); REQUIRE(matmul_start != std::string::npos); - matmul_start = detail_source.find("inline bool matmul_vector(native_backend", matmul_start + 1u); + matmul_start = detail_source.find("inline bool matmul_vector(native_backend", + matmul_start + 1u); REQUIRE(matmul_start != std::string::npos); - const auto matmul_end = detail_source.find("inline bool matmul_vector_argmax", matmul_start); + const auto matmul_end = + detail_source.find("inline bool matmul_vector_argmax", matmul_start); REQUIRE(matmul_end != std::string::npos); - const auto matmul_body = detail_source.substr(matmul_start, matmul_end - matmul_start); - CHECK(matmul_body.find("packed_q8_0_input_path_supported") == std::string::npos); + const auto matmul_body = + detail_source.substr(matmul_start, matmul_end - matmul_start); + CHECK(matmul_body.find("packed_q8_0_input_path_supported") == + std::string::npos); CHECK(matmul_body.find("q8_input_path_supported") == std::string::npos); const auto native_matmul_start = @@ -2393,25 +2703,33 @@ TEST_CASE("generator_scalar_kernel_route_choice_stays_in_state_machines") { const auto native_matmul_end = detail_source.find("inline bool matmul_vector(", native_matmul_start); REQUIRE(native_matmul_end != std::string::npos); - const auto native_matmul_body = - detail_source.substr(native_matmul_start, native_matmul_end - native_matmul_start); - CHECK(native_matmul_body.find("packed_q8_0_input_path_supported") == std::string::npos); - CHECK(native_matmul_body.find("q8_input_path_supported") == std::string::npos); - CHECK(native_matmul_body.find("matmul_vector_packed_q8_0(") == std::string::npos); + const auto native_matmul_body = detail_source.substr( + native_matmul_start, native_matmul_end - native_matmul_start); + CHECK(native_matmul_body.find("packed_q8_0_input_path_supported") == + std::string::npos); + CHECK(native_matmul_body.find("q8_input_path_supported") == + std::string::npos); + CHECK(native_matmul_body.find("matmul_vector_packed_q8_0(") == + std::string::npos); CHECK(native_matmul_body.find("matmul_vector_q8_k(") == std::string::npos); - const std::string parent_actions = - read_source(repo_root() / "src" / "emel" / "text" / "generator" / "actions.hpp"); + const std::string parent_actions = read_source( + repo_root() / "src" / "emel" / "text" / "generator" / "actions.hpp"); CHECK(parent_actions.find("detail::validate,") == std::string::npos); - CHECK(parent_actions.find("detail::validate_preselected_argmax") == std::string::npos); + CHECK(parent_actions.find("detail::validate_preselected_argmax") == + std::string::npos); CHECK(parent_actions.find("detail::bind_inputs") == std::string::npos); CHECK(parent_actions.find("detail::extract_outputs") == std::string::npos); - CHECK(parent_actions.find("detail::extract_preselected_argmax") == std::string::npos); - CHECK(parent_actions.find("detail::validate_guarded_compute") != std::string::npos); + CHECK(parent_actions.find("detail::extract_preselected_argmax") == + std::string::npos); + CHECK(parent_actions.find("detail::validate_guarded_compute") != + std::string::npos); CHECK(parent_actions.find("detail::validate_guarded_preselected_argmax") != std::string::npos); - CHECK(parent_actions.find("detail::bind_guarded_inputs") != std::string::npos); - CHECK(parent_actions.find("detail::extract_guarded_outputs") != std::string::npos); + CHECK(parent_actions.find("detail::bind_guarded_inputs") != + std::string::npos); + CHECK(parent_actions.find("detail::extract_guarded_outputs") != + std::string::npos); CHECK(parent_actions.find("detail::extract_guarded_preselected_argmax") != std::string::npos); CHECK(parent_actions.find("io.selected_token_out = &ev.ctx.selected_token") != @@ -2419,50 +2737,60 @@ TEST_CASE("generator_scalar_kernel_route_choice_stays_in_state_machines") { CHECK(parent_actions.find("io.selected_score_out = &ev.ctx.selected_score") != std::string::npos); CHECK(parent_actions.find("detail::run_kernel_flash>") == std::string::npos); - CHECK(parent_actions.find("detail::run_kernel_nonflash>") == std::string::npos); - CHECK(parent_actions.find("detail::run_kernel_flash_preselected_argmax>") == + CHECK(parent_actions.find("detail::run_kernel_nonflash>") == std::string::npos); - CHECK(parent_actions.find("detail::run_kernel_nonflash_preselected_argmax>") == + CHECK(parent_actions.find("detail::run_kernel_flash_preselected_argmax>") == std::string::npos); + CHECK( + parent_actions.find("detail::run_kernel_nonflash_preselected_argmax>") == + std::string::npos); - const std::string parent_sm = - read_source(repo_root() / "src" / "emel" / "text" / "generator" / "sm.hpp"); + const std::string parent_sm = read_source(repo_root() / "src" / "emel" / + "text" / "generator" / "sm.hpp"); CHECK(parent_sm.find("guard::guard_decode_compute_invalid_request{}") != std::string::npos); CHECK(parent_sm.find("guard::guard_decode_compute_backend_unavailable{}") != std::string::npos); - CHECK(parent_sm.find("guard::guard_decode_materialized_scalar_kernel_ready{}") != - std::string::npos); - CHECK(parent_sm.find("guard::guard_decode_preselected_argmax_kernel_ready{}") != + CHECK(parent_sm.find( + "guard::guard_decode_materialized_scalar_kernel_ready{}") != std::string::npos); + CHECK( + parent_sm.find("guard::guard_decode_preselected_argmax_kernel_ready{}") != + std::string::npos); const std::string prefill_actions = - read_source(repo_root() / "src" / "emel" / "text" / "generator" / "prefill" / - "actions.hpp"); + read_source(repo_root() / "src" / "emel" / "text" / "generator" / + "prefill" / "actions.hpp"); CHECK(prefill_actions.find("detail::run_kernel_flash>") == std::string::npos); - CHECK(prefill_actions.find("detail::run_kernel_nonflash>") == std::string::npos); - CHECK(prefill_actions.find("detail::run_kernel_flash_preselected_argmax>") == + CHECK(prefill_actions.find("detail::run_kernel_nonflash>") == std::string::npos); - CHECK(prefill_actions.find("detail::run_kernel_nonflash_preselected_argmax>") == + CHECK(prefill_actions.find("detail::run_kernel_flash_preselected_argmax>") == std::string::npos); + CHECK( + prefill_actions.find("detail::run_kernel_nonflash_preselected_argmax>") == + std::string::npos); const std::string prefill_sm = - read_source(repo_root() / "src" / "emel" / "text" / "generator" / "prefill" / - "sm.hpp"); - CHECK(prefill_sm.find("guard::guard_compute_invalid_request{}") != std::string::npos); - CHECK(prefill_sm.find("guard::guard_compute_backend_unavailable{}") != std::string::npos); - CHECK(prefill_sm.find("guard::guard_materialized_logits_with_scalar_kernel_ready{}") != + read_source(repo_root() / "src" / "emel" / "text" / "generator" / + "prefill" / "sm.hpp"); + CHECK(prefill_sm.find("guard::guard_compute_invalid_request{}") != + std::string::npos); + CHECK(prefill_sm.find("guard::guard_compute_backend_unavailable{}") != std::string::npos); - CHECK(prefill_sm.find("guard::guard_preselected_argmax_with_scalar_kernel_ready{}") != + CHECK(prefill_sm.find( + "guard::guard_materialized_logits_with_scalar_kernel_ready{}") != + std::string::npos); + CHECK(prefill_sm.find( + "guard::guard_preselected_argmax_with_scalar_kernel_ready{}") != std::string::npos); } TEST_CASE("generator_layer_route_actor_keeps_residual_choice_explicit") { - const auto read_source = [](const std::filesystem::path & path) { + const auto read_source = [](const std::filesystem::path &path) { std::ifstream stream(path); REQUIRE(stream.good()); - return std::string( - (std::istreambuf_iterator(stream)), std::istreambuf_iterator()); + return std::string((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); }; const auto layer_root = @@ -2470,9 +2798,8 @@ TEST_CASE("generator_layer_route_actor_keeps_residual_choice_explicit") { const std::string layer_sm = read_source(layer_root / "sm.hpp"); const std::string layer_actions = read_source(layer_root / "actions.hpp"); const std::string layer_events = read_source(layer_root / "events.hpp"); - const std::string detail_source = - read_source(repo_root() / "src" / "emel" / "text" / "generator" / - "detail.hpp"); + const std::string detail_source = read_source( + repo_root() / "src" / "emel" / "text" / "generator" / "detail.hpp"); CHECK(layer_sm.find("action::effect_prepare_scalar") != std::string::npos); CHECK(layer_sm.find("action::effect_normalize_scalar") != std::string::npos); @@ -2481,8 +2808,7 @@ TEST_CASE("generator_layer_route_actor_keeps_residual_choice_explicit") { CHECK(layer_sm.find("guard::guard_stream_ready{}") != std::string::npos); CHECK(layer_sm.find("guard::guard_stream_failed{}") != std::string::npos); CHECK(layer_sm.find("guard::guard_normalized_ok{}") != std::string::npos); - CHECK(layer_sm.find("guard::guard_normalized_failed{}") != - std::string::npos); + CHECK(layer_sm.find("guard::guard_normalized_failed{}") != std::string::npos); CHECK(layer_sm.find("guard::guard_scalar_normalized_attention_route<") != std::string::npos); CHECK(layer_sm.find("guard::guard_scalar_normalized_shortconv_route{}") != @@ -2495,16 +2821,21 @@ TEST_CASE("generator_layer_route_actor_keeps_residual_choice_explicit") { std::string::npos); CHECK(layer_sm.find("guard::guard_chunk8_normalized_shortconv_route{}") != std::string::npos); - CHECK(layer_sm.find("sml::state <= *sml::state") != + CHECK(layer_sm.find( + "sml::state <= *sml::state") != std::string::npos); - CHECK(layer_sm.find("sml::state <= *sml::state") != + CHECK(layer_sm.find( + "sml::state <= *sml::state") != std::string::npos); CHECK(layer_sm.find("+ sml::event") != std::string::npos); CHECK(layer_sm.find("+ sml::event") != std::string::npos); CHECK(layer_sm.find("+ sml::event") != std::string::npos); - CHECK(layer_sm.find("sml::completion") != std::string::npos); - CHECK(layer_sm.find("sml::completion") != std::string::npos); - CHECK(layer_sm.find("sml::completion") != std::string::npos); + CHECK(layer_sm.find("sml::completion") != + std::string::npos); + CHECK(layer_sm.find("sml::completion") != + std::string::npos); + CHECK(layer_sm.find("sml::completion") != + std::string::npos); CHECK(layer_sm.find("guard::guard_residual_ok{}") != std::string::npos); CHECK(layer_sm.find("guard::guard_residual_failed{}") != std::string::npos); CHECK(layer_sm.find("guard::guard_feed_forward_ok{}") != std::string::npos); @@ -2518,7 +2849,8 @@ TEST_CASE("generator_layer_route_actor_keeps_residual_choice_explicit") { CHECK(layer_actions.find(" ? ") == std::string::npos); CHECK(layer_actions.find("&&") == std::string::npos); CHECK(layer_actions.find("ev.ok") == std::string::npos); - CHECK(layer_actions.find("detail::block_uses_attention") == std::string::npos); + CHECK(layer_actions.find("detail::block_uses_attention") == + std::string::npos); CHECK(layer_events.find("emel::model::generation_residual_route") != std::string::npos); @@ -2543,30 +2875,26 @@ TEST_CASE("generator_layer_route_actor_keeps_residual_choice_explicit") { CHECK(detail_source.find("layer::process_chunk8<") == std::string::npos); } -TEST_CASE("generator_generic_headers_do_not_own_model_family_detail_contracts") { - const auto read_source = [](const std::filesystem::path & path) { +TEST_CASE( + "generator_generic_headers_do_not_own_model_family_detail_contracts") { + const auto read_source = [](const std::filesystem::path &path) { std::ifstream stream(path); REQUIRE(stream.good()); - return std::string( - (std::istreambuf_iterator(stream)), std::istreambuf_iterator()); + return std::string((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); }; const std::array forbidden_patterns{ - "emel/model/generation/detail", - "emel/model/llama/detail", - "emel/model/qwen3/detail", - "emel/model/gemma4/detail", - "emel/model/lfm2/detail", - "model::generation::detail", - "model::llama::detail", - "model::qwen3::detail", - "model::gemma4::detail", - "model::lfm2::detail", + "emel/model/generation/detail", "emel/model/llama/detail", + "emel/model/qwen3/detail", "emel/model/gemma4/detail", + "emel/model/lfm2/detail", "model::generation::detail", + "model::llama::detail", "model::qwen3::detail", + "model::gemma4::detail", "model::lfm2::detail", }; const auto generator_root = repo_root() / "src" / "emel" / "text" / "generator"; - for (const auto & entry : + for (const auto &entry : std::filesystem::recursive_directory_iterator(generator_root)) { if (!entry.is_regular_file() || entry.path().extension() != ".hpp") { continue; @@ -2581,16 +2909,11 @@ TEST_CASE("generator_generic_headers_do_not_own_model_family_detail_contracts") } const std::array neutral_generation_patterns{ - "llama", - "qwen3", - "gemma4", - "lfm2", - "whisper", - "moshi", + "llama", "qwen3", "gemma4", "lfm2", "whisper", "moshi", }; const auto generation_root = repo_root() / "src" / "emel" / "model" / "generation"; - for (const auto & entry : + for (const auto &entry : std::filesystem::recursive_directory_iterator(generation_root)) { if (!entry.is_regular_file()) { continue; @@ -2608,13 +2931,15 @@ TEST_CASE("generator_generic_headers_do_not_own_model_family_detail_contracts") TEST_CASE("docs_detail_shortens_lambda_type_names_for_mermaid") { using emel::docs::detail::shorten_type_name; - CHECK(shorten_type_name("lambda at /tmp/path/my_action.cpp:42:7>") == "lambda_my_action_42_7"); - CHECK(shorten_type_name("lambda at my_action.cpp:42>") == "lambda_my_action_42"); + CHECK(shorten_type_name("lambda at /tmp/path/my_action.cpp:42:7>") == + "lambda_my_action_42_7"); + CHECK(shorten_type_name("lambda at my_action.cpp:42>") == + "lambda_my_action_42"); CHECK(shorten_type_name("lambda at my_action.cpp>") == "lambda_my_action"); } TEST_CASE("docs_detail_table_event_name_supports_non_completion_event") { - const auto event_name = - emel::docs::detail::table_event_name(); + const auto event_name = emel::docs::detail::table_event_name< + emel::text::generator::event::generate_run>(); CHECK(event_name == "generate_run"); } diff --git a/tests/text/generator/parallel_matmul_tests.cpp b/tests/text/generator/parallel_matmul_tests.cpp deleted file mode 100644 index fd56b2fb..00000000 --- a/tests/text/generator/parallel_matmul_tests.cpp +++ /dev/null @@ -1,480 +0,0 @@ -#include -#include -#include -#include - -#include - -// View-sliced parallel matmul behavior: pack-group-aligned slice arithmetic -// and bit-exact parity between serial and parallel lane dispatch. Route-level -// proof against maintained model fixtures lives in the generator lifecycle -// tests; this file covers the slicing and fork/join detail surface. - -#include "emel/text/generator/actions.hpp" -#include "emel/text/generator/detail.hpp" -#include "emel/text/generator/guards.hpp" -#include "emel/text/generator/matmul/actions.hpp" -#include "emel/text/generator/matmul/guards.hpp" -#include "generator_test_policies.hpp" - -namespace { - -namespace gen_detail = emel::text::generator::detail; -using emel::kernel::event::dtype; -using gen_detail::matmul_lane_mode; -using gen_detail::matmul_row_slice; - -inline constexpr size_t k_test_matmul_lanes = 8u; - -emel::model::data::tensor_record make_tensor_record(void *data, - const int32_t type, - const int32_t cols, - const int32_t rows) { - emel::model::data::tensor_record tensor = {}; - tensor.data = data; - tensor.type = type; - tensor.n_dims = 2; - tensor.dims[0] = static_cast(cols); - tensor.dims[1] = static_cast(rows); - tensor.data_size = - gen_detail::row_storage_bytes(tensor, cols) * static_cast(rows); - return tensor; -} - -void check_slices_tile_rows(const uint64_t rows, const uint64_t group_rows) { - CAPTURE(rows); - CAPTURE(group_rows); - std::array slices = {}; - const size_t count = gen_detail::compute_matmul_row_slices( - rows, group_rows, k_test_matmul_lanes, slices); - REQUIRE(count >= 1u); - REQUIRE(count <= k_test_matmul_lanes); - uint64_t expected_begin = 0u; - for (size_t lane = 0; lane < count; ++lane) { - CHECK(static_cast(slices[lane].row_begin) == expected_begin); - CHECK(slices[lane].row_count > 0); - CHECK(static_cast(slices[lane].row_begin) % group_rows == 0u); - expected_begin += static_cast(slices[lane].row_count); - } - CHECK(expected_begin == rows); -} - -TEST_CASE("parallel matmul slice group rows match pack formats") { - namespace matmul = emel::text::generator::matmul; - - CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::f32)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::f32)); - CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::f16)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::f16)); - CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q8_0)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q8_0)); - CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q4_k)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q4_k)); - CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q6_k)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q6_k)); - - CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q8_0_x4_bl4)); - CHECK(matmul::guard::guard_uses_x4_row_groups(dtype::q8_0_x4_bl4)); - CHECK_FALSE(matmul::guard::guard_uses_x8_row_groups(dtype::q8_0_x4_bl8)); - CHECK(matmul::guard::guard_uses_x4_row_groups(dtype::q8_0_x4_bl8)); - - CHECK(matmul::guard::guard_uses_x8_row_groups(dtype::q4_k_x8_bl4)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q4_k_x8_bl4)); - CHECK(matmul::guard::guard_uses_x8_row_groups(dtype::q4_k_x8_bl8)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q4_k_x8_bl8)); - CHECK(matmul::guard::guard_uses_x8_row_groups(dtype::q6_k_x8)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups(dtype::q6_k_x8)); - CHECK(matmul::guard::guard_uses_x8_row_groups(dtype::q6_k_x8_q8_prepared)); - CHECK_FALSE( - matmul::guard::guard_uses_x4_row_groups(dtype::q6_k_x8_q8_prepared)); - CHECK(matmul::guard::guard_uses_x8_row_groups( - dtype::q6_k_x8_q8_argmax_prepared)); - CHECK_FALSE(matmul::guard::guard_uses_x4_row_groups( - dtype::q6_k_x8_q8_argmax_prepared)); -} - -TEST_CASE("parallel matmul slices tile rows contiguously and group aligned") { - check_slices_tile_rows(13u, 1u); - check_slices_tile_rows(3u, 1u); - check_slices_tile_rows(1u, 1u); - check_slices_tile_rows(256u, 1u); - check_slices_tile_rows(64u, 4u); - check_slices_tile_rows(20u, 4u); - check_slices_tile_rows(100u, 8u); - check_slices_tile_rows(8u, 8u); - check_slices_tile_rows(4096u, 8u); -} - -TEST_CASE("parallel matmul ragged tail lands in final slice") { - std::array slices = {}; - const size_t count = gen_detail::compute_matmul_row_slices( - 100u, 8u, k_test_matmul_lanes, slices); - REQUIRE(count == k_test_matmul_lanes); - const auto &tail = slices[count - 1u]; - CHECK(static_cast(tail.row_begin) % 8u == 0u); - CHECK(static_cast(tail.row_begin) + - static_cast(tail.row_count) == - 100u); -} - -TEST_CASE("parallel matmul sliced event offsets views by storage groups") { - std::array src0_storage = {}; - std::array dst_storage = {}; - emel::kernel::event::op_mul_mat ev = {}; - ev.src0.data = src0_storage.data(); - ev.src0.type = dtype::q4_k_x8_bl8; - ev.src0.ne = {256u, 64u, 1u, 1u}; - ev.src0.nb = {1u, 128u, 128u * 8u, 128u * 8u}; - ev.dst.data = dst_storage.data(); - ev.dst.type = dtype::f32; - ev.dst.ne = {1u, 64u, 1u, 1u}; - ev.dst.nb = {sizeof(float), sizeof(float), sizeof(float) * 64u, - sizeof(float) * 64u}; - - const matmul_row_slice slice{16, 24}; - const auto sliced = gen_detail::compute_sliced_mul_mat_event(ev, 8u, slice); - CHECK(sliced.src0.data == src0_storage.data() + (16u / 8u) * 128u); - CHECK(sliced.src0.ne[1] == 24u); - CHECK(sliced.src0.nb[1] == ev.src0.nb[1]); - CHECK(sliced.src0.nb[2] == 128u * 3u); - CHECK(sliced.dst.data == - reinterpret_cast(dst_storage.data()) + 16u * sizeof(float)); - CHECK(sliced.dst.ne[1] == 24u); - CHECK(sliced.src1.data == ev.src1.data); -} - -TEST_CASE("matmul actor guards and actions expose explicit rejection paths") { - namespace matmul = emel::text::generator::matmul; - - matmul::action::context ctx = {}; - emel::kernel::event::op_mul_mat request = {}; - request.src0.ne[1] = 0u; - matmul::event::dispatch_result serial_result = {}; - matmul::event::dispatch_result parallel_result = {}; - bool serial_accepted = true; - bool parallel_accepted = true; - const matmul::event::execute_serial serial_event{request, serial_result, - serial_accepted}; - const matmul::event::execute_parallel parallel_event{request, parallel_result, - parallel_accepted}; - - CHECK(matmul::guard::guard_parallel_unavailable{}(parallel_event, ctx)); - CHECK_FALSE(matmul::guard::guard_parallel_ready{}(parallel_event, ctx)); - request.src0.ne[1] = 4u; - CHECK(matmul::guard::guard_parallel_unavailable{}(parallel_event, ctx)); - - matmul::lane_pool<7u, 128u, 1048576u> parallel_lanes = {}; - ctx.parallel_matmul_lanes = matmul::make_lane_pool_ref(parallel_lanes); - REQUIRE(matmul::action::reserve_lane_storage( - ctx, ctx.parallel_matmul_lanes.lane_capacity)); - ctx.active_lanes = ctx.lane_capacity; - CHECK(matmul::guard::guard_parallel_ready{}(parallel_event, ctx)); - CHECK_FALSE(matmul::guard::guard_parallel_unavailable{}(parallel_event, ctx)); - - matmul::lane_pool<8u, 128u, 1048576u> over_cap_lanes = {}; - matmul::action::context over_cap_ctx = {}; - over_cap_ctx.parallel_matmul_lanes = - matmul::make_lane_pool_ref(over_cap_lanes); - REQUIRE(matmul::action::reserve_lane_storage( - over_cap_ctx, over_cap_ctx.parallel_matmul_lanes.lane_capacity)); - over_cap_ctx.active_lanes = over_cap_ctx.lane_capacity; - CHECK(matmul::guard::guard_parallel_ready{}(parallel_event, over_cap_ctx)); - CHECK_FALSE(matmul::guard::guard_parallel_unavailable{}(parallel_event, - over_cap_ctx)); - - serial_result.all_lanes_accepted = false; - CHECK(matmul::guard::guard_serial_rejected{}(serial_event, ctx)); - CHECK_FALSE(matmul::guard::guard_serial_accepted{}(serial_event, ctx)); - matmul::action::effect_reject_serial_execution(serial_event, ctx); - CHECK_FALSE(serial_accepted); - - parallel_result.all_submitted = false; - parallel_result.joined = true; - CHECK(matmul::guard::guard_parallel_submission_failed{}(parallel_event, ctx)); - CHECK_FALSE(matmul::guard::guard_parallel_join_failed{}(parallel_event, ctx)); - - parallel_result.all_submitted = true; - parallel_result.joined = false; - CHECK(matmul::guard::guard_parallel_join_failed{}(parallel_event, ctx)); - CHECK_FALSE( - matmul::guard::guard_parallel_all_lanes_accepted{}(parallel_event, ctx)); - - parallel_result.joined = true; - parallel_result.lane_count = 0u; - parallel_result.all_lanes_accepted = false; - CHECK(matmul::guard::guard_parallel_lane_rejected{}(parallel_event, ctx)); - CHECK_FALSE( - matmul::guard::guard_parallel_all_lanes_accepted{}(parallel_event, ctx)); - - parallel_result.all_lanes_accepted = true; - CHECK( - matmul::guard::guard_parallel_all_lanes_accepted{}(parallel_event, ctx)); - matmul::action::effect_reject_parallel_execution(parallel_event, ctx); - CHECK_FALSE(parallel_accepted); - - struct event_with_accepted { - bool &accepted; - }; - bool unexpected_accepted = true; - const event_with_accepted unexpected{unexpected_accepted}; - matmul::action::effect_on_unexpected(unexpected, ctx); - CHECK_FALSE(unexpected_accepted); - - struct event_without_accepted {}; - matmul::action::effect_on_unexpected(event_without_accepted{}, ctx); -} - -TEST_CASE( - "parallel matmul lane policy handles empty and invalid injected lanes") { - namespace matmul = emel::text::generator::matmul; - - bool ran = false; - matmul::lane_task{}(); - matmul::lane_task{ - [](void *ctx) noexcept { *static_cast(ctx) = true; }, &ran}(); - CHECK(ran); - - matmul::lane_join_group empty_group = {}; - CHECK(empty_group.wait()); - empty_group.reset(); - - using join_group = typename matmul::lane_pool<1u, 128u, 1048576u>::join_group; - auto &typed_group = empty_group.get(); - auto &same_group = empty_group.get(); - CHECK(&same_group == &typed_group); - CHECK(empty_group.wait()); - empty_group.reset(); - CHECK(empty_group.wait()); - - matmul::lane_pool_ref missing_pool = {}; - CHECK_FALSE(missing_pool.valid()); - CHECK_FALSE(missing_pool.try_submit(empty_group, matmul::lane_task{})); - - matmul::lane_pool<1u, 128u, 1048576u> one_worker_pool = {}; - auto ref = matmul::make_lane_pool_ref(one_worker_pool); - CHECK(ref.valid()); - - auto missing_submit = ref; - missing_submit.submit = nullptr; - CHECK_FALSE(missing_submit.valid()); - CHECK_FALSE(missing_submit.try_submit(empty_group, matmul::lane_task{})); - - auto mismatched_capacity = ref; - mismatched_capacity.lane_capacity = mismatched_capacity.worker_lanes; - CHECK_FALSE(mismatched_capacity.valid()); - - auto zero_workers = ref; - zero_workers.worker_lanes = 0u; - zero_workers.lane_capacity = 1u; - CHECK_FALSE(zero_workers.valid()); -} - -TEST_CASE("parallel matmul lane dispatch rejects missing request pieces") { - namespace matmul = emel::text::generator::matmul; - - matmul::action::lane_dispatch dispatch = {}; - matmul::action::run_lane_dispatch(&dispatch); - CHECK_FALSE(dispatch.accepted); - - emel::kernel::sm kernel = {}; - dispatch.kernel = &kernel; - matmul::action::run_lane_dispatch(&dispatch); - CHECK_FALSE(dispatch.accepted); - - emel::kernel::event::op_mul_mat request = {}; - dispatch.kernel = nullptr; - dispatch.request = &request; - matmul::action::run_lane_dispatch(&dispatch); - CHECK_FALSE(dispatch.accepted); -} - -struct parallel_backend_fixture { - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> - parallel_matmul_lanes = {}; - emel::text::generator::matmul::execution_policy policy = - emel::text::generator::matmul::make_auto_execution_policy( - parallel_matmul_lanes); - emel::text::generator::matmul::sm matmul_actor{policy}; - gen_detail::native_backend backend = {}; - - parallel_backend_fixture() { - backend.kernel_kind = policy.kernel_kind; - backend.routes = emel::text::generator::test::k_generation_route_policy; - backend.kernel.set_kind(backend.kernel_kind); - backend.matmul_actor = &matmul_actor; - matmul_actor.process_event( - emel::text::generator::matmul::event::configure_kernel_kind{ - backend.kernel_kind}); - } -}; - -TEST_CASE("parallel matmul f32 gemv matches serial dispatch bit exact") { - constexpr int32_t rows = 61; - constexpr int32_t cols = 32; - std::vector weights(static_cast(rows) * - static_cast(cols)); - for (size_t idx = 0; idx < weights.size(); ++idx) { - weights[idx] = 0.25f * static_cast((idx * 31u + 7u) % 17u) - 2.0f; - } - std::vector input(static_cast(cols)); - for (size_t idx = 0; idx < input.size(); ++idx) { - input[idx] = 0.5f * static_cast((idx * 13u + 3u) % 11u) - 2.5f; - } - - auto record = make_tensor_record(weights.data(), - emel::kernel::detail::dtype_f32, cols, rows); - gen_detail::tensor_matrix matrix{&record, rows, cols}; - - parallel_backend_fixture fixture; - std::vector out_serial(static_cast(rows), -1.0f); - std::vector out_parallel(static_cast(rows), -2.0f); - - emel::kernel::event::op_mul_mat serial_ev{ - .src0 = gen_detail::make_src_view(matrix), - .src1 = gen_detail::make_src_view(input.data(), static_cast(1u), - static_cast(input.size())), - .dst = gen_detail::make_dst_view( - out_serial.data(), static_cast(1u), - static_cast(out_serial.size())), - }; - emel::kernel::event::op_mul_mat parallel_ev = serial_ev; - parallel_ev.dst = - gen_detail::make_dst_view(out_parallel.data(), static_cast(1u), - static_cast(out_parallel.size())); - - CHECK(gen_detail::compute_mul_mat(fixture.backend, - serial_ev)); - CHECK(gen_detail::compute_mul_mat(fixture.backend, - parallel_ev)); - CHECK(std::memcmp(out_serial.data(), out_parallel.data(), - out_serial.size() * sizeof(float)) == 0); -} - -TEST_CASE("parallel matmul q8_0 gemv matches serial dispatch bit exact") { - constexpr int32_t rows = 61; - constexpr int32_t cols = 64; - constexpr size_t blocks_per_row = - static_cast(cols) / static_cast(gen_detail::quant::QK8_0); - std::vector weights(static_cast(rows) * - blocks_per_row); - for (size_t block = 0; block < weights.size(); ++block) { - weights[block].d = gen_detail::quant::fp32_to_fp16( - 0.01f + 0.001f * static_cast(block % 7u)); - for (size_t idx = 0; idx < weights[block].qs.size(); ++idx) { - weights[block].qs[idx] = static_cast( - static_cast((block * 37u + idx * 5u) % 255u) - 127); - } - } - std::vector input(static_cast(cols)); - for (size_t idx = 0; idx < input.size(); ++idx) { - input[idx] = 0.125f * static_cast((idx * 7u + 1u) % 19u) - 1.0f; - } - - auto record = make_tensor_record( - weights.data(), emel::kernel::detail::dtype_q8_0, cols, rows); - gen_detail::tensor_matrix matrix{&record, rows, cols}; - - parallel_backend_fixture fixture; - std::vector out_serial(static_cast(rows), -1.0f); - std::vector out_parallel(static_cast(rows), -2.0f); - - emel::kernel::event::op_mul_mat serial_ev{ - .src0 = gen_detail::make_src_view(matrix), - .src1 = gen_detail::make_src_view(input.data(), static_cast(1u), - static_cast(input.size())), - .dst = gen_detail::make_dst_view( - out_serial.data(), static_cast(1u), - static_cast(out_serial.size())), - }; - emel::kernel::event::op_mul_mat parallel_ev = serial_ev; - parallel_ev.dst = - gen_detail::make_dst_view(out_parallel.data(), static_cast(1u), - static_cast(out_parallel.size())); - - CHECK(gen_detail::compute_mul_mat(fixture.backend, - serial_ev)); - CHECK(gen_detail::compute_mul_mat(fixture.backend, - parallel_ev)); - CHECK(std::memcmp(out_serial.data(), out_parallel.data(), - out_serial.size() * sizeof(float)) == 0); -} - -TEST_CASE("parallel matmul route guard rejects disengaged lane pool") { - emel::text::generator::action::context ctx = {}; - ctx.compute.backend.n_embd = - emel::text::generator::test::k_generation_route_policy - .parallel_min_gemv_dim; - emel::text::generator::event::generate_ctx run_ctx = {}; - std::array messages = {}; - std::array output = {}; - size_t output_length = 0u; - const emel::text::generator::event::generate request{ - std::span(messages), 1, - std::span(output), output_length}; - const emel::text::generator::event::generate_run run{request, run_ctx}; - - CHECK_FALSE(emel::text::generator::guard::guard_decode_parallel_lanes_ready{}( - run, ctx)); - - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> - serial_matmul_lanes = {}; - emel::text::generator::matmul::execution_policy serial_policy{ - .parallel_matmul_lanes = - emel::text::generator::matmul::make_lane_pool_ref( - serial_matmul_lanes), - .kernel_kind = emel::kernel::detect_host_kind(), - .active_lanes = 1u, - }; - emel::text::generator::matmul::sm serial_matmul_actor{serial_policy}; - ctx.compute.backend.matmul_actor = &serial_matmul_actor; - CHECK_FALSE(emel::text::generator::guard::guard_decode_parallel_lanes_ready{}( - run, ctx)); - - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> - parallel_matmul_lanes = {}; - auto parallel_policy = - emel::text::generator::matmul::make_auto_execution_policy( - parallel_matmul_lanes); - emel::text::generator::matmul::sm parallel_matmul_actor{parallel_policy}; - ctx.compute.backend.matmul_actor = ¶llel_matmul_actor; - CHECK(emel::text::generator::guard::guard_decode_parallel_lanes_ready{}(run, - ctx)); - ctx.compute.backend.parallel_lanes_enabled = false; - CHECK_FALSE(emel::text::generator::guard::guard_decode_parallel_lanes_ready{}( - run, ctx)); -} - -TEST_CASE( - "benchmark lane configuration guards and actions toggle parallel lanes") { - emel::text::generator::action::context ctx = {}; - const emel::text::generator::event::configure_benchmark_lane single{ - emel::text::generator::benchmark_lane::single}; - const emel::text::generator::event::configure_benchmark_lane multithreaded{ - emel::text::generator::benchmark_lane::multithreaded}; - - CHECK( - emel::text::generator::guard::guard_benchmark_lane_single{}(single, ctx)); - CHECK_FALSE(emel::text::generator::guard::guard_benchmark_lane_single{}( - multithreaded, ctx)); - CHECK(emel::text::generator::guard::guard_benchmark_lane_multithreaded{}( - multithreaded, ctx)); - CHECK_FALSE( - emel::text::generator::guard::guard_benchmark_lane_multithreaded{}(single, - ctx)); - - ctx.compute.backend.parallel_lanes_enabled = true; - ctx.benchmark_parallel_lanes_enabled = true; - emel::text::generator::action::effect_disable_parallel_benchmark_lanes(single, - ctx); - CHECK_FALSE(ctx.compute.backend.parallel_lanes_enabled); - CHECK_FALSE(ctx.benchmark_parallel_lanes_enabled); - ctx.compute.backend.parallel_lanes_enabled = true; - emel::text::generator::action::apply_benchmark_lane_policy(ctx); - CHECK_FALSE(ctx.compute.backend.parallel_lanes_enabled); - emel::text::generator::action::effect_enable_parallel_benchmark_lanes( - multithreaded, ctx); - CHECK(ctx.compute.backend.parallel_lanes_enabled); - CHECK(ctx.benchmark_parallel_lanes_enabled); -} - -} // namespace diff --git a/tests/text/generator/stream_window_tests.cpp b/tests/text/generator/stream_window_tests.cpp index ae804690..aa9ca936 100644 --- a/tests/text/generator/stream_window_tests.cpp +++ b/tests/text/generator/stream_window_tests.cpp @@ -550,7 +550,7 @@ struct generator_rig { prepared_model prepared{}; emel::text::tokenizer::sm tokenizer{}; emel::text::conditioner::sm conditioner{}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; emel::model::generation::contract generation_contract = {}; std::array samplers = { emel::logits::sampler::fn::from(), @@ -570,7 +570,7 @@ struct generator_rig_q8 { prepared_model prepared{}; emel::text::tokenizer::sm tokenizer{}; emel::text::conditioner::sm conditioner{}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; emel::model::generation::contract generation_contract = {}; std::array samplers = { emel::logits::sampler::fn::from(), @@ -589,7 +589,7 @@ make_generator(rig_type & rig, emel::model::tensor::window::sm * stream_window = nullptr, const bool stream_active = false) { const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::make_auto_execution_policy( rig.parallel_matmul_lanes); return std::make_unique( emel::text::generator::dependencies{ diff --git a/tools/bench/CMakeLists.txt b/tools/bench/CMakeLists.txt index 4266ca9b..f6bad8a5 100644 --- a/tools/bench/CMakeLists.txt +++ b/tools/bench/CMakeLists.txt @@ -284,9 +284,7 @@ if(EMEL_BENCH_NEEDS_LLAMA) set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) - if(EMEL_BENCH_SUITE_FILTER STREQUAL "speech_dialogue_moshi") - set(GGML_METAL OFF CACHE BOOL "CPU-only PersonaPlex reference lane" FORCE) - endif() + set(GGML_METAL OFF CACHE BOOL "CPU-only benchmark reference lanes" FORCE) set(LLAMA_BUILD_NUMBER 0 CACHE STRING "" FORCE) set(LLAMA_INSTALL_VERSION 0.0.0 CACHE STRING "" FORCE) add_subdirectory(${reference_impl_SOURCE_DIR}/ggml ggml) diff --git a/tools/bench/__pycache__/moshi_gguf_convert.cpython-314.pyc b/tools/bench/__pycache__/moshi_gguf_convert.cpython-314.pyc deleted file mode 100644 index 087136e78db7418ce69d14fd9e2f4ca1a7b28bd5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 100099 zcmd4433MCRnJ!q^cW~b~a3{n~q)3WdNQsiDU6cgLpcjcWM1mA7ZYqGX1lutjJ24f< zEx6;f;Evsf&vQ=bo@7E#CljVS^IF|AeI~P_@AlvS{`dd?``>1}&CKC?@*hq<_g$-s`zLykuUs*pdhk;n z$Avf-7vM%Xm+FY>usWcwqOkgiCZJ)z+JKh*>H<3Us}JbeuOVPyzs7)({h9(M{A!Mv z4_g8j219$qde|1QRdFs|z&@f0WQ{li*(2IO&WJ9MJE9NdjTi#?BgR0%h$&Dw!Uc*( zRDt3Vb)W=sOYyx7-^=m6V#FM%9I*tdMyvtHh%Hb(Vh_}eWCdzRwgl<`vwox|(BRSs z8b`u`Cd7BT41q1Kw*t+sKMk~u>l=YiS7BhA>+L|7>zjdYSCOmORpKgjmAT4Y6$n?ls@ipd z9+xAq{k)}`b5*-)wyVT;E#UB6Tg18Qy42#EE~(`My&2=yBW}ZnxH}k}Mtp0MzXf)> zoOs?+#7$)fcDb4nZeiiwu2zKGSh&yCj__6%?ss({+{wawT-y-tV&T26ZiIVSc)+zC z;a(Qr=h}hrP8Qzp+J*3L7CzwWL%5%X2VHv*-pj%VT>}X3W8p)t{Rkgm;lr*$gb%Xt z5!WGv53}%5*Aaw|vhXq2F@%q^@Nw4(gojx8gzL{-C-HQOo~BfRA=ha!m~ zGk8ABo=>};K-kT~F4r)^PqOeC*HZ{T&BA9{J3fQwbBT6*!gU_cBka4|HHz@FEIjP; zAbf#^pLC5OJkG*Txx5HZu<+BaNrW%5@G~wS!q2hrx$}loe=T*i;s>~JsY}9)O=C#(Hwp z6-4+27Jk&png%EIHW*AV^+3wvE(Mfi0Vo^UN7 ze4T|ST{m4{!_yn|1Ps6E`Z}K8WZ!(QKSB5g3qN=Mk}KT%B*%^0Ij)>LKUK}0f3BL7 zsfk?riz=78mUBitZ)iDg|Ln{aFCTF5p64CYv;K>|wz0W6$Du<{9CVC%#x8pO4&O{* z*5RG@PPKYxcpqMkixF8Qf1BCt+B9jCw{5b`v43{l>+d=^Khr+v<^8iWp1CRS)zlyd*Z5JKS zir_~H{IkwB)qb(cvdRg?DX91jCUr0k5khxPWz^PFJ77Tjd`CH;fS%O zrrWkW>u8q~QLK~~ECs!hn%2=Y!Ou>!l1`%=rvS@AWt{L$I*y(>Ipml?13x=8jdCO+ z&iQ6$WF%qOG=`L@wb$~D#W$)t^eKIvW zGs!x}&tS-9*vZOy25sP-8S|d{YJlop`mZgabHFYh~GCLi9 z?-Z~WnV#+%J?5D91X{0n#~jaleUldhRMF>oUjRj>nhu^AJl;sPayaHZW0$C)g!Vw) z{imLAOnI)&&Iei?-m7!7eth%Ih$9u1k=kxNaBa?;7(YNgaR|(Lu2F%3?E#e4!+TMA z-V4M<^Iqg<=O-^Z=4UWCW+w4wegcGnAHH+1w0VLN6=apG@oNr~q%VZWxW(_822B~|Jpr%p z)_J@!Pk=6nFDAvg)wwsW1%(;+aYlUCi~|EXa9eS!-wX5$_{K(OMlZ(;1H5O(4}6-Y zGR{wrqQyb8;sxpNFM2)W{&=49HBf<&EfvPJ=jtfx>zf_-Wj9iJ4aieJi5kvK#scK^KArEb6% zmj)p%3+WKjCqf1mG6r-m6Q&n)TtdDhGZV9M4e$+xLZ>{;3e2Okp24)z;%IAYJ0I7g zGyHK4Uc_|_f8+Yui3t#kkGP?a^l{@TBeA2Saq}o)%RG8|G;SRoy*%%kB8Yg(8XcYR z@&3S+4-{}#e4#)kyeJqQ#dxQF_5=dF@4`F=!ssYpiwZr!KNqi7qA4Smqmz^K6QdH{ z28Qxwh{tX>5bzh__B!`VqvciSnz1ORE?P5LU){21w!PlWH_I_IH*gO=D;oIbV>GY< z&1yumn&LU)m_8*A=~Gjl8CIsa-ak9fk9qlW0FP^FSn&=D8sijiUJ&XPI7BM=-OsSK!Vo^PC;ftb9e-Y5QRNze1j5E=? zGze>1Sm)9qtY=}p%Yd+vg$*tf!e$mWx-0>c%ZmEi;(8axnwK9Vo>=0GIW59KSJeXt z%hk<|sC!=n--CDM+-Vg!9Pl_UjZ523oR03-R7&6(h)G<>Xkc8=d%;J{#I-{hD6Bo$ zs35Ww@KZK5JT(pBnS@A$XL({-+LSNfM>KyaZXxb7POZtmPq#?-J^G2BWlyZ_#-ye~ zOpZtoRlp=wU^sh*y{6j;=jLYQielW=av!OGO&^9nQuBm4d<||+HOPl|YJ$D-6S#&3 zvBVfJz8PPD20@~<;tN(%u!bn^sCtv5#*y4rQls@f}dE?% z#m#Qd)Vz0)=Vy7Rh9AHyevsa4z+d>Otr{8}B06coCyd7}5-QOH>$73R0U8-BufOoh z3*ob?hLU&pN44KEePH_5(28Npy0uU!8i-l<3Hp6&X8R52b!Yg%&0}vJLmbNomc{W^ zN2g%k7TUk2HNJlA%g4f9tJQTNbRhC0 z5%4b&=|LU?`x&RwY`ssn@HY07EC{}{OOHER$MoPF0wIp>;&qCOn08?$Pq+!0S^71i zl`%?|FpO|5Gzz4PdiSAv_mPh@py~mBek>5z&(9D;$R9vQjm@6`{6{wM z_bAd3-Dpx|R2;wT`j=`=<72hvY5p{F<EkkhVoVL&GIOcFeX zh`RZ}L~AeVM7YO4gIiob?xjWDhQ=gpB~H~QVWRlRDp!MaAQ+ac8#}J=2%o+A*i{PM(A^5C60zKL%8=*CL&1cj(LyoYXE z=vGTN#(goyW*a@y?Jw}}ci{F_?zd`8%)d2ag88i-WBs>!+&;riY;!R&CHZ3*4rxZ5 z!-9pU&U#EtplR_3_)pG=s&dSMs`BtSW<*ta#Lu2KA_UD<=xo)41|Wh*^JvN>qrn`_ zXc;I}5(RdtMpg6bdCh1-jx?)D&W&!(Ht<^D*^Dl=Ov|#lgZUiIYpPQwlq>CWtL4#J z#T_dIE#)e)q@cep30gr{1-i)-91>aOnKXOPeGb6mHn zmlm5>Dtknf?rs9{)wGxiLF1SaB^DhTvr(9pf=m%z@GM#!V1ujr%4cM#1zy zj+7X+>@ltzyEs2{iASM1ww!KoIQELAv?$8P2k`v|6z39dD9*Yi_eTHM`eT-|PymMuTQ@+9Ui=+FTJnfa{E5haK@s0dd{4E z*Hm)bR1!0l3F@-X40!w1mp8nB=%>D4at?Ip{;tEaFWdA74Ma8L{Iwz-t^h!`E)vIK zEgm;xN@)oG1pod@+=!z$sRmT{**!dVkNrkKmg()rVJJ;#x(7sVJSva+(F1Xl;g>)$ z8P#U7pqa&z<`fO}w9Bn>tGB7y^2&HFG2})P(_(gF=*ZB3xH?ZU(9;DuD8!rr$cz%g z--Luylqs>*HCjs4z!#54VlMk6fU^ENqvzWpRIn(pLz(tml5=u!fcp>e8!0MuJf}|!SV(M)n zxf~NXkqmWe_IbyQ@4^KzzYyPgA+|QtA`e1X{)+Dk3A`W>7DZoOZEf3S8Sl2v)(gm! zq`o8(^GrSOfxwRh&~0XZ6m1;O-rCW*t*g6dd+!d5pS?(YZY)*AXb4ER4#4Oy`5BnpRqaE+ffAdR=d8>sjcXL`l$!YzotG{RYyxQERa_ZvRAdw** z$^iaT4}lW6g1Gj%S>H@t2T|Ij7iI&wct8 zt41v-Ux}(k@ixS@adVn;a>7C$9Ls|UpkMT(PwViVYf1!%dC8Q zRIj9^wfqrjt(nsv1DzxeE{SvJ669_K(~`^+$|Xfz{iv%Ub6tUTHn#zoY3cl;*ku0$ z(`B)A*5k2d?J}r3_OHc|TLY4k7c__(g|@h=?O`zq@JJp#)8?BY`TTPd zEeVz+zd#_vKDCKt=3yQKFzN@}=pF57W51~{JEfOJqI#f>gyJ3^LO}^cusHPta*%=B z#kd`5&z*8geVb=$?xN?RL4KiYyWsQqgL%pFN;UDf1@n_HBM~U70|x8-ZKZ5cRa3vh6R**0q^+hbo&~8)iG8ju48Q#*F*O>;k(M` z&?l?dj%bc??^QpM&j!f4=n`4ra-~j3Z0j-b4?g0l{rr1~&*=77L1aVRFAIu7_3K&r z;ivCrIYM4YRTr*=ha*o!4n;czTYc!@n%4CC@mG$A3x(3I)ts&sZMWi0_i9e}ind4b zrhPT1eMP(VbKh)R&Dpl1?NY?q2zkqD&XyHzvm%c4=I7Zp3)(fk<@Kporouhn?2GoU z<~6VATa>SMtmd_>=v$Sqwy)-Gxub8sZ{TujKQ(eCmG6#4tE11vDm(5JcP>~HQH9Em zrIw}2<>!U+L#ug*R`iDzpE{RzFSXq=2<3-Y^A4}*k0d@tCZki)5n6Ux+yH?e7PeZ@0 zefQ~CoZ3nFCSPU*7}v5BZ$n%=TT+97J_Q1*oa8k!aXptv-J2^bLynbjT#_Wa6LIIX z!~1q1gK^OHa&EXz$$z>vz0c!IRk|2FHP%iwu@G(z`&GNO>{ruikV|t>HQecDdctGR zh$IlqiO!9M^1+YUbf-T>qc826GbE=HcLLUkk(sri<`B$NyRd7Q>a^x4H(pL}VKxZ$ zlG~76>m0Ol2_b=~7&?Z9C4>#S6FE%ipg&=Qh&C;iTzXl&AhyCVzXRia90VjqF<@{T zSgnnniFQS;8elAN64PQBxEr*l8Zv83rVfux)5Ur$NU0U&tTz0G- z{z{s3bUWUCgc`F$5d|4=uH( z=luc4xDUGb3%<#DT0&wiHtmp>YogkeK$G1D*}RN-z2j_{%Ec!(maU$M2TrV=utK_= zfqU6BkBOd1lcHs){L2~Rvf`Lv90_tlbZHW z&BcjBYO?WJEJg{=7%h8YGvc)Hhp4EwLx)ZrA3ZVfVKxUubT_PzL?nojgGaru?^@;OIMsOjhSywJELISw75jJsaF79M!*OYhfytM)c^ zTdd;1`?c@a{(0i*Zs~zfN)N1=Y%8Ynb$juey?bf=<0p}0|1R8YyB=uNS-YX4HSLD{ z$zJg0d1RoqzJBtRlW!l51itgs2Tv_&e`NmHynNtqkNx;qEdTHw?UDOATvqW%ub8 z(cYsUp}LPfCfCM@$1ajcuyQ|%yT`VfHX$hx&a_-~B`EcrN-^6cmY=h>Gqz0Wl8Uf% zG-Ck4-Qnn!%sx?%jzas%z&P<{LU0KT%Ts*lFdtY|iwEI+`ERCl<`VMN&l+Kj**~iP!M8 z;r}ed4EpzCsR3P~n3dDaS%?2eCwC5nJX~KOPFCoe0Rx_dBQ$`y6jdZrtAWss_Kk7 zldeQYRcR_mN3f}!6GfUIsbJ*CRCrj{iSzg}+uU?X5mlY0`53y9vES`)iM0Fxq5!gjg z>tiB5bY#Cc0h5w3Ze?%XgRV12P7aAGM_Pn3y<=R*h_+&Wrdc^|XGsQ*A2~F1VsPlJ z(=MrCM1zdDLF(YRX=YRuv-@ElH+t!cIGO*DpjQwa{mdwt$E8jBaWmO6kwqIH;Qvd+ zV=5zY87t1`vPQ?2(q=rcnq%|C6*SfF6#CdV_T05p-L_N-j;%3EN2mc-1L1}@`l0F3 zSzdqUm1n{^ua3MOT+vmpoAV>u@D-tAt5DDpGj|Hw&NXZ9jic9(zHvO%xTdqdZC%lo zCt~$37cHBFf_*XbenGo`&7Obbx$DopF%@#InX+z_UN3#49Fkm}Y2oy1PazE^`Gp1{ zyE@xO=?2h5!!n9qo3o2i9_MEZK!OUb_BLXv><~ z8ZP_LzM9>(s&2!h$u5{m1$Ak13Pu;OON%?3f*lAvF6+RUz90_7h}ql|Pz!H`S<=BU4S&U|O_j+QMyBV| zO71k9AF1N(l2&pJ;tWFyWvGDF^4H|=W32AfC(0O!(VQs4>h3Wh$t6py_)yPs?$|W9 zNf|O};9S}fl`QH^HRZXaKK7>~(VJ7xsa#l}9)xyxPBUD)RYhysNijF@DsIg@l-RYR zI5okixOC?dd`?nzj+M7w$+u9-dBFG4OcO+VWv1I-Y@x+}2tZ7W!G9Z}jSH;WaW8Gh zfQhWM^&kUDb`Z)DC&89jC4)cSvE&h|I_{Kq{LPNv^NaX!@GSo(eu6#H8jFDP(~wz8 z+aeOsr5y&6MLQGe(R8?5IRQ>hNtW<_%&C|~FgJR}0;FdIJSYdT|Dm02dT1ASIyg0P z_0%-XWnm8<*RgjH2GN>KR9pQeLUElRO40zE56H@vW@)I@*#P`LeK(5oMXw*rF1aaW z#k#bxg3)$rQ}O>3QnK;*V+2Cn&$@R$;MDfRs*j&oJiA_45~+LN`Hu4&GmEN~Lgz0E zOTIC7yJT>=>u-1b^^RD{;LWj{V}F|R=5FEOr#ddnxiEgicinem_WEpOEZPyX*GD~f z?M;Hc>EkCtO(3Dj;enpZZw~p^3|Zk_cMO$Sl6|5tysyK{PxYKBZ_U{MlPk9_#`c^P z`cHxWo>uJ#?Nu4}t3ZcM2Z#=94yb*V3;H*eC_YmTVy8No&3iA;``E4* z5zT`R;?<^oerD2{*kU7^dNWqhsTMJn#0B7@SbMRiBGeR_DA|oJZV<=v;<$U=Pa3U+8;RG5iI{v|vLhA`3ZzyIwDX33woW_U;Min1R z?7B%Ge+wWKee9seNA>YmWgn|l`_RX9XL42wEbRyO;&*w!_+euoYvqEsA>dKLXagX{ z%dNsvhohwq^}Ni@%2NgydsPahB}h!9HHCnJLw1r-D-*tCS`?p2EOX^4Ntz_;VKktU zIO%4IlU5~+1E|bH5;nxKDF>{fpX@0l)|w}L;HiN#aMsmhv-2~gXCgg`cRT|f@1RyD z&Q@fttpo3Lhr7OWw0h3^)46Qt=oI!^w&;*tNZs%uJme6m+CLgT`E{ZX|O{I zp;vjAgBj6;vLquBHlG2K)gvN z>tIQ{58b}Y$h7;W&FM=O;P91nV}|TIDD)^SB$V)@Q&51tdU#I1P|4Wu_CI)t|X<& zoC=Ht6N0!>1otj7f6zTNh$|DvL$d=0XOQ(yW4g7l?Pub8GuR@Dh7}tE6EaS=|BFeE zB?E~eU?R~SaV^8>l{AYodc|6u5iJ@;q?8je#46E5aUJw;@QtXoO$bt}3;T1l95!gX&IM@pjy2uq{87B4N=Ez_c6uL^rJRHklPRrJuJq6bE5 z`l5|^Ma5v9;b2*9j#o;F9vfwBZhCy&=_A}BDT9bR4TL)lk>?0!8WypG1V72rAsV|3 zxU|!W~#%GQ>eUa*{!Oo{TjUFlP&lvVCRzzr}af*as1SI9t!@&96WG z%G2-KziStYwy)^6%b0W1`IhsoipZtKy6l`}Dh!CYj~>{I9G&nQ2lI(K79>f!I%Tir9m+OGX`ub`3C;OfG#ATvF!LgnoZq zzJQ=%BH)DZ5=hKwA46jPe@1zOYDZJB!6Em7vxSkg%&1J5OB4?430Qu|wY1ZF6KeO* zh$bQ}4a_99lR3aqIYa(`!*|w`&mjP#I3s7ylW5+$&h+|IUw&#KaO2wbYcX9(SB;9fQKwgTUrPs;z5Vdl!TEG%6FRL@cBd(d{P0R!|%n zv8kYsO%FfPWzulaUeB?>BPY|-gt#;T6G0qPsj~00dnErJ`;FF#&&jcesO+-f&WgAX zSo8-Q-$HC4`(}|bbPl3=SI0>e@%<2eV5#0lq7R6AFK=_Jh@eV58cN{SKX zueCBCm##o&Y7H}~1(fRmS0{u7gG@+}r&J}MOOR)`M){uReJOVkLxpWVS^8Pr8r-eJ zDUcXZw`~TGeYJNg#smpe+#v7DT%lNan}b#1IQY79CG&^kJ|?q9fdayPV2*Z)sj zCr8;ja*s~}50uj^>+w9LA}jM=2UmUq^Ym`L_y1r}fNkRC)}MnANxWpS)92<`pv+OB zw9(7Cr1vi4CF?Sjy)aX1U|3N)1mLuSH8__rW=J#ID2Av{pIljOYP3-z8Eup@qm8sw zaxFf$4a!p506GitawpmVc(tA5o(FHuUuB9jlzB3n+F(33OiJ9DGb!5GB-ag))XG>RDf?Sle_^WMfNPFSuu!t<&xe=Vd#-M@ z4c4#_Tc*$vNx2A$3xO$GE3GAJ811P|O_K_lD9gEodJbhWD(i@6z4BRUN-wvIScY*d zO*hoNE*&1~U7rp!wLI8-okr*8)w|TBLLFx#8DCR+fhkw9E-&X!sH)QI-;fRq`8TFR zz`Lf-1P)?{aas=NAZ-#a!n*N>4E1!Trvc0@$#lRJ;^Cw#wZ2H1(!zK)q_nt#YeP$8 zrpegS&FOGZf|hhxsX4Qj)+zJa(9&AvS2Agv+^nTrkyi>V0kDy^k6%xL4& z)WWWw|Ckt$l4U9DhUaYMvr?V>Ij!EF4hJ>dnhqvwbihsjvp2tsS(rQ$v1IbnrNW9}_t|8HHkcJYMX*ij5D50CgtbNj@h=ed9dw z^H6hQqlD-*#d^^P4?HuX1vBj=g^{g4SfiA4CZwRSA|p>jOG&q{r z`Y~vy^hteo+?Mo-0=Y#zFZq}(Nn!Hk^!!x7H!4~`2WypONyMQQtD-QiJK^a@cEqAg z-Zy~_eO>~g(CWvbE0H5h;t=u+0}U=3lB~tZtx2#;33JnEqUGHM>{xP;rBHCoz75^K$J)qmCbo(vc zh<5S+fo|97_8D$*{g`Lo@0p4l2tml@q zQiE}shzx~tvUym`q&12eLy))N2E}Qj8oZ6(+Ub@>w`}@u$Id|b|HHNn3~MM$nQ00& zp{QETJM+Y;Qzu<#2M>%MKQZbaIR3=onYd=s8{o(Fvteu8?|8%lz{eL82;+5 z3x~ck8os=$D@NJ!^`YY^TUB$+zW1G}uz$JmcH5pC{ssR#Q=vnGeeW*}_Jq~VzFXUG zIfbI*G3yCIf8ys^y$c!`|GZhYQrx__f3SterPSL;70IzP$=CK%i0?{4Au81hO%{AQN;LR-CbMbC$>hp1Va8{!FuF@MwNAp zwBDv;s$ZF`3s1gL5ow4Pz1O~CawcQ;uV>|kd*66zVOFTIz+{h!$SWuVBifDEH;MO4XRs$(%Uuw`}8 z!*@zsueU7Jufg)E%+_q||jvid>{`QGg zOGixCx#W>V@~jY9QGN7FtBxL_YCB;@?OLsKvHshA(RM1rHY!3F6`_kf3Rr+k!*9;s zF||bt7j@sRK!D_;4305U7&W|CB@{aa=YUk3h4x4efhi|26$GY&v;)?K!LOWudsf*L z`tX&A_nR+;Y2-B6*X1vn-;zV=4hAMh--J zqPfxi(Qct>&n^8wSe9QBN>0S=LxOSWz6mKX-i-Ez*;PYjv`}d3TdnS2HT2)AUNP*K ziHwk26YW`SU#;CK)R4vf?z*+QEkg6&)w%(pb|30!IY9Vm8zg)*461O3Qfu9Vam)!G&STo# z^N1_4qc$PyV$v5(ghkkiNlv)C^ zc)3+rSsIn{GnOVHo5HSgZN@a_4QVX0+=gVu8Pixdq_HK^Ko*)Yja`{Wu>nDo+*TmB zL89F<)iFy6#|@A6OuoZnx;1CeO3GC_Go>$Q@|e_?%4Tw}ob>o;!(65OK~YZK=H(m* z_XX2^c++Q_`ZAR|MV_pHs(&O@_~zdfYQC~96i~HEy}@I(X{zj*aGhQ5#5M@DQ=zh5 zH|ew3+47Nlgzn6P8MHX*H8er=^nd zR>nC|hN$pGNaHF+E)A&|wvZ@WBULJfx|J)xrb)$8Tc{!(7ILdhhk$n}QnBR5M7P%E zQ09eG5URIJbPLk4U9zlvgga|Ylro2~6?xPsQ!*Sw*&ro`hKfl_Q%hRQWaf2DW?rA3 znw2nJ8%3kB0m4w_CZPlt;9DbobL$XlVj(?3P6{z2g)PAYL*(j{JP_0N3DL{(@e_x< z@J=Hy6C9UjW-lcCpOcGbX>*3`p1h6E0qH+D6gMT(q|8UuR#*l;+HE0Lgx!eC)*Yhx zTP+?pF>H))``lGcQb?k7Qu?oHj-n+g%~4uSCCnW)mEkk&XQLXGI)@dUY)W>kL(GvO z1`aHY-Pq_sv*I`wJLqUQ`BHW?CLv9xZ6NN13(ZQXyz7SQ|0vKybfO){wq(?t`~RFZQXhg(TsQyyT=c zbql}ly z|AG~pCVgh>`99u=qVQVW{+eG!friLiy-wVZMdGqx>E_9K4U$0;&!o&iS%sjLR4IM& ztYn!P*@zbp@4UMh&u>#G(`d4MDrIj@nW@=sCeIu7w9yuAyZ3(rm>2g2OmC7q>irq}Hn<+N- zMyYizh@Dd^lx~YX$`b~5V951;(-%w3c76Ns2(vED#(=bSD-qFRB8*XZJvrOEHTyi(CG0~E4rqI-mz zZGhQTuvSnOIrPm@A-_I)nnVdrl!=ovabk^)Opbk+ch^+^iK%|EZ0YiMt7ITl=UggP zE+xpL1bJeD;~!#ZHGE=fSZrC&S=qAl(J4-R*mc*`^ogk{QIA2P;9$&rNYEZ4Ia5VU zR|E+|{d;|j9jj$6E4mgLrjXtCUH?*FtbIUe+lSKZSCy}o*F?MDJFz&hTHY!^W`#O! zr6P7v5j!AS)0Vkt^@n?^+rJOG8pd(z5u(;-RI^ zCGXNO;P(}-6_!Wb-<+Y{7Swqb>O32Dp1}!$UD0j`?5f&A4cJF`TV1x^uw}6?*06(~ zYi_G+*S7AWV9RaDP&)c3*cwx}C7C;nEp`z*h2f=o;6UK>%;gnk&2ft;BZpo$aPgsq z4&rdU=wPn%nVB_5jLHrJcZ2 zIX9AB`$+HGFj}TbF72X|0XACq!48yJ@kuK*gPe%jYiUzf!leFS7c}iD_5(@BG@Go8 zQ=#gVP+o6RRm&2JnQu>RSpD>xYjWujAA_`Qh7kmt_;3!FQnzCW|rWaY`FyM@iS3!9gQ zg~H}oVPDL`wk?~(4UljW39^J=66%f!1xI7%V}kbBx+zZ-b3;jNYW&31n257mDCmos z`vq-3?Rth`xRBkl=)J4!`b5{YuC0zXkhd<9wbrg$>I5xZP-ca7Z&nEE!X)CNtnB(j zhM73IGC}0y533v+R zlS>%mC5S)DkszaUy5ZBQM3Nb`l`wox5NMalt%KOfjM}9bBtiy~5HitWATV~;OS|A< zkb8OJ6!HFP7j}Tdh?G`H*rBbTwYn@2En2ay&`R41A%Hiel+-5IV9uxzaH?E(WxLCQ zMTp_{V8v!ih2S=!>{&1=mwFdM-X|fk!NN&tBI(Lz@p4ed6xc|c%LH+Goe8A?>S}9r#_OhIm zcvmUIoU;snoSsH$mdQkRfXKE6FPWf!HGZ4rq~W z{m9xWMWjomW0KJ(1U`>>CgPsH*YsORBZ_zboI(!V;uc!{kBUA~2D(~?&c*W~P zt0I^Vj$ML@TxPLvBTWioQ9t8yHBI;F7QxOh{48qN&oWlSc)HcazKOF!f})~?i5kBK zy1rXQbp}piZor)!NNeE3O6#%#t7<`sbi>^(NF-lp^op%9FyM7-Gw~Si#72Rn*K@Z9 z^NI`xzape@2aOPXOK}e@eFJ=a%b0^JLA7D*s!s46Z4$H;*V1V@rMQuPjGPiQ6FDU} z{?xFMooH8=k*NrA%ygzDN>Vk3Hnu3YQLrP?>_KCjXcpGSJZJEKg7$e>mh`rpaKW1x z6I~lVN}h&a-0;Dl7)ktFfEz54{Pk{?ee)*$2zN-I(j5%HxOtL%dF+wLec7!+T=l|3 zb&Dg|^pd>bVCS-uYyZSPJDigNZ&a|jh;1S}T41y@ptJaF*uAd;Wv8qpJQNVMF7=OiY@AH6_%$Llv6Zz<8(fcq&x zLBnv*$Ym8QJQ;p6;*AVP2e9&oHz9j==+MveI~R=bP55S2RCmWzzh=z3VZUw#PH9D6HR*XGsMyrt3vTAHu8eB2%Skv2HpLqpBf@3TCbeee}X&{`X5hDp}edbM6wF zu(_11h%Zb5`ab32u(1-p-ipBj<8S` zKQZ%daZGroeehl*TOZD1o1*W7R%OzILm0EP(_i!# z@G}Y>rW;sTj`{y)t2n1kbSlA?OUx=Y!9l<}bozp{;QKx$8Uk*`Eey4!*F+BDrLyn5 z(PXitiYfODB!MwnLHw-bG@N-`6J^+Odn$?G&mvE@Ec-J|!=!SA(bW8#Hk@B!&ksKX zt1;#UEv$dDGE}#&#y6pG%i@`3&ktLLmc1+L0qI@ql16CXv!dQBJvT1aE_uFpb+Kwi zy<=T%Sr`=Z8dlVezp|G{o`@ca+1mhCXO9$ovvozsh+0FOt36nJ# zpRQMLU)IN}_XyfbG2@E%m|?5Xg;NKIC`0qnH5;sB;apo~*+Rq&0!FogTpJk+!`{gG zn4wvcpd0!TUnn{hvmO@ohd+f)8zKl5-7!PYGHls);p;uM%2Gx^-~d2vC(}`%VM|CIT$NHDCkSpOy$wZ_g-GgS*_|^F?Etbr@frK}wv}$e<-0D?!trm6P$>~|pt(%G>I-y!_)~f!P ziJH}PfPqdlt7up^uv)iIsNIiB!=w%c&SHk1CK!6^IYU0oY6q9*@9O&=Sh(En_uDyr z$?tyyIf$jnPtuCNFU&vO@h@abN6JW%vn8+lIK)@9t9yJohL>2@20$S>#ttUVqB}9I z^t*^X;PktMJ>c}aQH()^0F0LE!B&VdV7gES))XAyoO(DnZHc-wv5OO|V^SF5(veR9 zmp(Oq>Jo=7dT~^1a%Z3-y@6O{L^}p*(ff|K`g`z%wbx5l*W`)#HN-6 zaZ^NQNk<er9R1a~vA>eG zY?Pm5gN5A;iaf=Bv!(Z&%brY&`YHBWQZFi;45x{^&{8S8^jZ?OYoZGp>aC>ApNSf< z)JkcEY`{nnp`qp^a7lrScezP8cuEnxD1|4f>Es1H4g-MjNxV)$)Rcaa_YFC2V(Eir z9a~7{GYm#Mrj)y2V;R%%IR%H@9NU{J3Ja1rkO|ic+1N-d$JlUbAr4Th&@I(D6!}qJZxb^4$dLy#u#p4=w(DOp4dvBhB9;-iAYWpV}0XgCytyr!j1?y>lzq3 zbMU0=#Gq>wd!68wI9`(Y=yIPPJ#*^#k+Y-62i=3m&tOxOoNn;Rv+zVLPR_aP?Lik# z8yGz^aN^W)Y_-xcS4sS@QGmG7=NDTPo4VMP?gqMx_KAcH;-+35+tABc2$35W7tXn& zOp01~^x~WcJ?!T_NX};VuVc7|xCdsneNSkN@l(ltB5T&-NFI2eimoO151&~&A(;1p zO|_BBNtFT4YE4K46}aBCF!0)wP}&JaJxl&2uTZc%X6_TTeE_Go3f8(+UHwYa{uSMR z(&U?47bk@76RT}QLhDHYI;E;xuiGQk?gg)8+DE*Sh5TEpb-1$~A499UvX#n&GpJf@ zNaVOJ)Ops_duhyfZNjQ^1&M?SpFDT{w?b`m#^Y$ z72)7rQ_Ux)n&@*&`&Sx7=UMwG3+zRy63h;2CG0&>)7}#`_MRl#H{xH>)vyw!HE-5h zR_=nA(iKoT_-j+^3X#r~C9Ryy*+Nd!s@WN;U(*@C@=O?8Y6>E?IBcSNwWN9R@=D28 z2DUV^Ekgdf959F}4IdK91_|aN*4>9$cSBFA;YvG1{*h(}`lpM=DB~aN5O|b`SLv#h ztuxDzC~koS4uot5WI*Ax>tU97yaR~KW1i^n0rA4stQenh`E(XAZc+0+`dNmJ4t{P` zv6ma!;Dk1Bf{nNlrH5uNlLJASMM#)(GZtY)!?r+xA(LEOB(xhi`-eM4jg@vW_d)6; zTZUMQDO!#!=a$NFDwwf^#W3Wk86l*a$_91%;X2D={ZXT?MG<<@&Uw>5EO2B>h`@NAn&3@90K2i}{X? zW77J>wUiwk{9|6nes8vOeU^_~m3eQN{R(9B6>5ahN$)In3-H(YOMu6Qa|r^>^`P1E z^|5drIU0Op=8ma4ip_WATtEDx=&|yHz9sWdwk-$#_S%oH{Y}5H^C$$qG4pXjdwgA8 z5FTGqmqCJjqvU!?xZ_Ru&|f#(Z#ci^3?I08J)v^hly#W1Wh`yNXd z_IHwL1Nno@d8i|HwhaegPGfa7B`V(Ns1pB-H}J5SsfgM-+Gxy+H&KWzf7!O-OAzWM zO6y=r9l`wMnu+%Y=J^?R^jb}V4uZ77$-(4BhsSZLgGA|9u<~HUP|t{AA2UX0z6`De z_+UQ82=FXh{79wAPA?xn0ldMb}@ z;igDqv_4w2s1ht&poKG77Ot*fZ+bpVt@FZzLe1{gGFn6|U&Vr>;E3v#OCj&C@?gK5 z@-cvq`Gcxgo(p+@X|ji_zg`;lzx^^3$CgD7ywfYxZeKPo1wrxlsT$WBTcVST?xp=p z-ODOaQ019pB$Um9VfxaOE3JLE^PFqNTYjriSzAH}ewAJSp>IhSYuLSfF_wMkl_RXM zue~I#1@hl9Ms?Bgs9R{>f9v3_Ucr2vR!Bo^g){^abMBW9->>H?8|ZAzjK`GyA@o$A zk)Dd{LI?F8TO9s-gyQ*^=RI(867;e&YvCmt{!aY-VRX&o5!O{AtUCaSq`&iO)z16u z&YafL;}U2q{oc}wzi_2}56(DMJJ)vaz0U&T6rv=rB*9_iH5-5ZTHc6BcYBjPm7Hr&`i6!dc4*Cke3G1=rje z^90F)+uGFhseE+ibg3M(#G8u5`f>8qDxH)*P7XLP4Tm>D#DuFN`2f?3Dgn}|_nF*6p@nbH_)rWF6 zpq$yReuTg&&E+IZjM=KpolPUaor8OMr*8O0=K3D15_3J)B>pDp7hqJ#B|lXxw`216 zTw)aVg8Q4xr7?T(zeULKIHUF8e~y?N#;R)%V7$lbq8NSd%%d;QoePl#yT)PfjFFZ1 zh;o9SjH8~Aawx}0-lk)uGBZT9Xq8e$bnF?>lFWHy7p!=lqrDjH5>Hx?=;PHC`j`tS zn~m4a%i>U$g|O<_lGLaFU4}lb{sMiP&#>$}=noOg{vx$!He#6!doItN|DUXHYLxA9 z?A#yRH`o57^vy?7pMWBY7h19Q9A;DT!ijMbmpWJAE)eH|aTp?`;B1|{0617rbXvLd z;G7a>!?`*cCg_y{rnod82iKbdvys7UQgU-Cw7{9@+ro2l*K+Pczy@o5ua0#xQP;wL#(CU5O$bqD1UuTxv(Q1Ta&=!|gX5@-IJTFO*jhRwU6 zKPj!=ni&pSy=?MoTLWL#|F1TC0+jj>kCks%v3j79$I~&9&jR{Y*r@I2ZlLg_P#a-#z#>8OY9)O2sr%t{w$9kDl2S%J$5;9-ETpBFuD**C)fPy{Mx; zNnMgFCCnR--=pYYnNMZxZ0iN+9B^?f@p8AXj;%F3oiIJ{>tUi z2_Nqdv|YI@o4ampgVpG`7yC0V(N+=fIQhb6O8Q`tY?Z;R04Ro}lHmxs zcyI(y!pfE>pCM@;dgV;WVn94JX&Q?@p!#N|w;)X0xZn}?7M^>6ar~&VNt9t}3&_^CWHuW%e(?WD@SaZ8N|7Fh^|Q~3 z0y@Xol*bQeMSjQS-qEcY&gx2F9G#*%KjR}adofY4KSOj#nQ`PFnkwaJucld^!FMWh z_CMT`-U5li$Mk@#)#ROBu0%ax<)VP{N0dKqkeWGIAq5;~1_y>9mxnAHhDcuOU0BA$ zFSab)Rv;Z24nZ&IWgQi)kOG@PD98k%Lo};u8=rlCMl^|1XbPCLUK~Hcf@BO)`j5PU zi_J2Pk+hDaHSkDs`&Q5d-*js~ZY)XA$=Z5;PLWYkMIb2*|i<{ zxcIyePVYroKQbn0j#+pUpoD!=C8ZXYR47{&HI;JhO>{7w1;=2BsF^@O9RBX0Z>b32 z=jVt#T&5dOH$UA1xW%;ug};Je@Wmu_Hm2ncXZc_diDw$J!Jm?Q1`OX&9@1+7-N|4K z7GUIboeedRXV7iR0480(!B~t#H!e;ij8N%=Hsx4vd&tcF+N6@D41LjLO^GN$duo45 zi6tydO)O?3eQHBtUZ)mXz|E--TE(qYQYV-9q=IzaBZxPk7YmwTCE0(6dO zVLUuf`D%$O#7%S`Wwr`N7DQvm)$?<4BVpbs#!K7|loT<7p`@s^tEP;u5*z~#q9F52 zzXRE(Y2Qc#mvV{fxNPNS7eYk-13+T>n}-I{v;0b(A3n39hD}4>n%*MV4nb38Iid>H z{|YXE3T~8MFMV}XbZPX&`%k~~^rGs$^NSNp?(fa4RP1Ip3Dwd1?`-*COSD{QJw!%j z4R7`f=29|7Nb}OnTy=z1UpXK4tmukT(?S{7#?G%cvGc3pc9=}caEC2gBHi1gD>_(E zz*sPmC#9=euWA!2nS1FDvPSEqq@A)c+|X)4^PQ{~`1rAHT`E{I2-}YdmgAuV>)Pya zL1g~@;5)&^>i1p}>m*e6t!Vq#wMFlmqV=)jwmaJP`vxw%^;0`p)}4vAt(JAH7IfUn z>LhsWOS;A9g`T5=<=7X4x9x#ZowFx2A?j-4nyt3})I=JYP;*0Kv1t_nri?XuXolq1-- zg~r$QICoWm-{;c)TY0xUKdt?T$y@dn{mJ$0l1RPK)Vo^U8_V7inp`*A7A9{@U7w1X z%OaOo%#P@x?;QW&_)^ZdPcHda>UQFo)*CI?Tf&#$Y>%8?F;_BWD)!0ToO)~Oc0qmg z=_SLDtRGt!zl8qi&tA*U!zQVGvgNZeTfQu2%V)w}&*c=~w7zAHoc?B3XcCX+EK<;} z6_j5$uUT?eERJ;0qf5g}M+mHQ&ADySicmM4Z>w#vtJGw{u2N%iU{j8vC_FFJ?_aIl zzgATG&1;`(REC;g=?m#)_iAPL((^xjX+=K>KiUN?3k_&?Q%R%%p7m{u{uSdkdM@>GX=LD=6`_`&XElYqKhJJm z(5_Q1(JVrRCC{=J{^f^-%0ZN3waED&Pz*RH<8sg(egoduNl5MNRVR7wS-Q^khLa@NG*ht7Sp^3rj8z54iF4 z^{3z6vtrw_o?E__E1gmQPDS+6Qr(ZXe7t3;90R1k6}OyLG-6&G8qr22v^T?JxILGW ze`x=KU8ve~$F_$Y*Jl?+wC|hVF}?Lt^w^U2N9K>si!Y)QyUDx13ZB1Fmz;exVlA-L zEkem2=&0pT@nfn_Yd9O*uyrJ6J^BkIMhJKwEZu{3R79QAJ4DXFt6QKg(JRLN3>0$81N=s-40KJ(ofp`!1OrB6b!@teN4 zd~cnPHZP8U|GAHzTRex#?A(c)Z6_*}1v_Fjj@!&j^j;HZuZ?yuI#;?6|Fk-0J05CU z)7ip#uZ=R2)VOGj28E6zKdt>~fnYpIRP;18kM6YUS2p-Y6VuhkY(rwYoY&4P(;XCy z66K{dM{t}0joeb4ro9hVW5unD<4gNv#l5%H1tI6clTyYJQ*_{68v|1n)qcnHfhqC= zk~iZ(0+p$RXkjTTp(`ailM^lwiucD12SPPunRa1CU9t{Cq9VJC?@HF+;T*?)y|(teE$0cy|ZyY&l`~pNyn@+Og``amT##fsLzezOO^x4|1`q6I6x< z2)t*8#>t!zlk(eN!j^Oyk>7eU@(ebLmwr@A1XA5im9+H|f!sj^atG$HoGQ#=X8Xd# z;_(&J4ne(RqYWR4q%~jA{aZ&Enq4_d@G zV;&eT!96JBZ1F>uQ?x#jPpZyQGEyuF{~fPSP~znj$P&`+YEpK%ryj$~o?FwF1iDVit# zF2Ycqp2c!QgOIq8UM%-A0wHXwoP*`{oTiK$PSJeBC{hJIbizIWl0_B#LcQD~*B_~I zFfKupt`>!c4Zf?iBOztngk)MvGS=bhZRkh1{5PcJfWr_*_9!+S9)~0{ksq)?3SU%V zNd?U$7i_rV;UTbNYhUcAfgRn9;|%svVM*sb<@1ig>jfDQ0uK%&I?jt`jZf{)tO;=p z9o)!X^i2fd4CY$E>(}86GlmnnZUbCj6MX(Jkuz&~$feO}@G}`|STkE-_G`3)c)htB zhGSaug6Av8LI)8DkH7gsWPa6Nx2mnf3rkkG?#8aK?}GEQg%Lqtwx)%R@K=t82G2Qehs@7=zwa4 z@2%l3w{Dw8I_eLDmMCDi#-~DevoTPmIQWpnbb;9m^AixU!sScg+ML%hgotc~Pe?3| z1A}55Xab%|T2bQ!ySSE8iAQV=qEAFGE#hia9uO>|pI-tjhCTlS;*o!7aYz{LZ|8;2 zexqc?Sh`LeYwZ7P?@hqtx~?<9T3!`uLt)>5Eo*wMBuqOa>kKVJ`n}vB94A z&rHDJb9;C1eaJ{mwz8NK&3(*c<2?^vR$7E!vzH?^uKeF2(V9E?pV8tYUJu&IHyAhH zC2tOT1M!%L+CV(+UxPAt<Y5^sWrJQ@Y&z&R$bYK=+fVX?$-jl00LB&Nf#?|4%+iLpG&%1)!ycg+YUd< z%4Q7x;w|P0gg7-EcBc#mFs9oNpAqlZOnNF~yDz}FkKAhTAAJkuh&zf{^7jxTj;0v$ z=MbGXdo3p_yw($y-h>mIytWfnUVEJd?DqPjiUW)8FZq8MPwxIZ;@1^Rf((=|Nz-b8 zFuvsO-Qbg=nI$>`7Y>(*_l5sO>JJ8(Inii3ae_0)G>&Oa37I=(ZY(Jk_8|*+D(G;6 z!XV(U#=&t&m$f&2bCsQgd5)6TuPc_L-SNR}m@MtCs` zs~qr5{}>M=jpM?|j0-!vWt_J5m~~vRM#h`!vn?1n3X37GYM2|fF{LVll@Z=rQ z)a;jPUZ|N%2&Gnhsg#U}oO}LB=;GITBNwe8R3(K4C|1iFJzqgxOfbt8=XUb=XUaW#p7^o*D^>+&m1)DyufW)9tSKRKHpI2W=M&K(b1%7_X;cJqMgresC4HoSD{g-dh0LRp)G znOhc^(YyvC2tWhA$>jS($X?09)R;Wa#TZ!{9N1Wi-aZ;uSAO2mDkCFc4wAxt`{sW`W%nfdApg7yve+ zggj!2xl*nkxpE|s6>=3q?$Bg;cGKle&-<=E`K2cV>DT>p+rIkgP+~RN_!FXLy5Bct z2;+?WvJ=M~P(6aC4WBYhK#9r(b2qB z&fXKe4!lF(F4EXY->x^2z1zKRyeF}Dhc_ATDeT?pO~rc}dv|%$Clb9GKw>guw!`PU zso(tY-(oa0U>G$PVe7Z?)Qk*k+7*mtNHx}~tvLN}s7c0-Oy)EYG|XuM%vU^=g(eN8 z*fjgSdNz&Cq+ZXbKh?`MHRYmr=bVr%X9e#P=DgNsEk z52KtK>QbczyplM(o+401Z&9ZVTqbEbU8VCihVBHpfHc91wh$Bc` zDau^@r1NXaysN!^z4PnQ_KvQt^=y9evHRu}P%N-t$a89PYdwv+e>WR7WsJ}SASuJ61?{9hrUGY@Gd5q_~@e}H{t=i;NGtbJh) z6#C!nVYI?5QpB%`QnThh^I!F5Q9w9_=DLJgG_ckz`Wc&r|2-T=5>;sc-GbpL<5A-Y z!*BM0+Pmt^aw{`y@3CV?Rydww38ecUx(#B<@)0G5uoW{oYkeb=gZ^LJ$rk04xSIOZ18i6vr`oWr1oyxTY+a(qyq!H3FxBwu{tb(W ztoWq%do;LpQmJjqYXSKXD*+)Wz7cV?|SB|d4HHU*EQc5%GvV1wVv2FvhV!Xw`LjvXql%~ z!xS$*$hvW@wr;=%h6S_8{r~GmndfU_(!Z5;!!Oe;|08&-tsDNIDAh;g$!mrG7(WHe z*UHiWsBz!Yki*!N7+e3+2=6-j-}lXjnFZn7eEP33Ay5vd^+3&cifb zPKNg}dCC^giqTM}*Go^8Nm(NH5Hp=Q&q|B8M!6>PuoE39P_M!U?;#jU7Yh~qEwAhzcq7YBS+4P?a-?u7di4)%#je+ zB>eQ|BS*ml<=B868y_f#2UnSPow^IXMba2ZPd0i^owUU&RWL?xiKc#WMXJisudh_8 zk2YCy0|55HXrsT9%h1zuO-<@=j4d_Zc`J~k@_}+}LXN5xbLfpkwPqx0GXm zU7Kf&>QUvj(rc5V?T0~u8>22eYQ#zg+ahEY-Z@q{{x!&->RwMzWLv!KUWT}F#p@77VMNw zQ45S`8^JcNS`lq9F^a#+L;S~B7)-$cByGpQSWTAP`k{hB(B0sY!^qC~KO!xJYNce% z4jQvJJ(Vg4GTUsdfL~JTLOEja0x)^pQVQ1Iz~)67BEag!FH<_DBFIBerOCMz6KMPu zN~csrG0eUK|4$T=EN3CcG4OBj4h*KMq_ce&dZBz^cx-@oQcSj-0X6mx!NeP+_Mb-2 zp>#m%pm%`Z_!c zE(PMXClh%1q>WVDCb>2x8p4Mj5K~GK~2rbaOOQ4si90lS^I2 z*x7O989Orw9jHx|Y=fMPzF?p5fotVY9m`M!GZQhw4EP@9RVIzXV#orKbCO`-(Kd{Syt&D&&3lcQ6J zI8QUE#d?)scG9%u!D&x6d8ALFsswTs!@YvUH8Ud*wI}zd5(MD=w2Uwn2gjM!`HrdT zbtLWeL&#@fa%6C-OeZzuKwuvS9HH{s@yDPoLNZsW6AfG}2csb%mVM}!fKw9hrGQvw z;SJ0=W}M;-Pm3kREpw5fLgL%%8y|-*3m6LS1$JjgE{3R007wEUH9ZF$?i(Vxn-`!L z{`5b zsW}`r9ic!lv;`BX^a~Ns=CEmt99BM`8Qk0wsoWbj?UMtmzza%3uR(}eLddR>y1b3L zTz$DZnB;lAWbXX@=}7suP*F2%IYuo^AU&8|JwJHM(s0w#5aq1Twq0%uI!ouy2WuaS zls_EiTI9fOU_uS1Hb$IHDA&?_M@rnx!kZ$_Z4|zp!jD`&5=_~8J0)XwC|J-G$=nu7 zX{N{>;`j&cmG~|s{ufNInm-=0*HKfq+_6BA5c>V8%J$3cuv$3hn?DjM-5%z4ptUmx zp6Q)E2b+v?lPLaJq;yx9+bxGx&Nl_?4@9aChPie*aMOHSaBF*{`cRlVEC-TxnqVr- z)6`PiHp?AiW4kX>xj)Rc%8BDkVC`dd?T=KohB-(*iH&9{+alEm!rVa$JaqX`Fllo% zDI=I!8%d&`TDDMKEWCk**GH1JQh4JXaO6&1o(iUJ3net)OxPc_+MXS`JQ7T6xt*33 zum(5ok7T!o(%O(X;h<4&@mBPNM1zBwjUi{#E$7albLV0qq_!=)A=P2c5ojM+!m>EW zC#BX1I{?v;Vs>`YK=(HPpf(q~)I^&kBUl9B1r9FJV>n%eZXjV50%0=>g`jmy&FF)bglREDeA)lWOpt^Pw!D#9~?2af_y9dH^`= zq_DCQv@P3c;JbYZnwBM~OY=H$3AhFI(nfv-k_kG9xDULmQvJ}RO1gJ{AWK|$AYWnV z)B(i1|4_=KXrV<)m3)}=0my8Il6HkOFsq@Y@j8kz`WwY@9+uuQPVzf?Ic$K1URgR| zDqWtRwWm&<9TC(3L1)TH-`T#YjJQYvYPv6&i4ypya`aL&$&0D{xF`VPy(EOeg4=YX zNj8H;cT{qq`~EQ|=Tu%?F>(5=A7^wP-X2x-1gX*R}{1G$h~O^8As%pcvIbp6a|gs@weE6VFtzesU%2YXyT0$9Sr}YA{*LjRt?&A7RvihOj#3VG zI+nd#7OrT2x9qz+7q%=O|7OO!Wj8C@!=^)Wf=%xZg{zOgJM`U43+EOKzxC+5LpQ6B zhD{y!Oy=wZMu_nBFWC%<=~oY5IUFztydkuQ#9L>LU!0;{eCYDf%wz9c3+^UTa>!no z8M}Hgv1v# z_kNS>H$r|_Hgx_+sN_Mn@*(wB?lM;1HNcy_utX1{zvDslcRcRj%4v(lW@@}gMy0To z>x%M`C7K7MBCk$y$3%>GYf#+va;#2q*UPaw#a%DQ>J)dq97J)~DzV4Nl4EpKy&P`j z&{&66Hr*AzM>2AxXs9e})sc!EX%Cbm9XT@eb9ggBXUUQfQKpeh&@aa0}MWFm}6RA7M$wMLzkkX$+{r?OOKOPV-)sUQMOG_h*UYfQmYY{O-w@o~8mVdvn-0iueaq5x3uzI|D(xqD0RWeb(m}4bph$UxBq3V)PbgipISz>z~ zC62DEL=x9zPSbncHY#=$)>X1cELhOav3lUHB1C)4kT_u*RzeIpoDvWi-6-yi` z%(3zOI(_q=N-0c#Uy-g*pveHUPP7GDCFKEogrT!q`lURX#%*F$Rf6Oa32uMB-Uno|usK)p_P4NNE2*_+b+IpmS zKhsXY%6qICMK%lEk_Tx(3gE?DQk?f#_am`1=@;vhP`PB>^%snfoS!8SIf<-fcquonn*klP?- z_-#d9n~FGLx4a?ZYzT4cy8Y&ZoTc>6sE}?77wxaTm_DMBJrsRYww= zNNrZi^{y9tBF>_4X>-Kc9ORmB+fsuW<@4Nk9B(@2KE2SmnEu0@w{sSq!L@CfTbHJ`S2)0|w1E)JYB6TtB`*L$XFTR19)`!lLoD!q+5oZ&fOS1u!CK9+;{yi} zl&5JpzD_)(R+-hE4B!))C8x$BwJAvf+p4LP?kiVy3~0)=s+D-5evGZ;s#U~W0N1Ov zj#HNs*T?nNVD2uaepMPjq7f9(3Xa)FL=4tXNm^x)C zdXb>_Oas3Knb>!{fa0+8SR6CQzwkI_=9hN|>LX6vYR&h(F&x>rd7(bCu`%du404T# z4HUju8F6BUN1Vm;nTy8n+ar}b7W*QV553b8sc4Nj+MqcuF%3Fs+atK&DvTr|*|!T8 z3nP^~-!VoiS|YB!Am`cLGaG`YtotW5eF%Yc#G1AGqz1hiS{g{d5C%px)7(y@L#hUt zp;$J}*#PnP5EkRmd;dCl!3jHi1B2}cRUISru_4j8>n?e7l}q#>6QX#$1Mwm}qQ%uq zW_oUEyh~4^YxJ!oNAIj_{}bhHLf#YBJwflX=zeWd6k8 zNwb@=?_^%}JDET6chXqk7HeAVJDE>>((h!|8NJoM6RF9RkwYlkTL0WjxUZ)86{FeZ zp?*`lo;0y>JG*v0X_PIB>q)y}G-aWtRbp4I$*$8U?U#tJX_eSjYjWt+r2P`{HLVi6 zYE91gnq-wQ(iz5f>=}%?uupHr!6XSRiPXt#L0Ql_U^_r|gk!A#G;YB7MkHo?G~Go_ z8}6j#;DFj5;9uMWj)=ZDE-uu+@w+&C%63LlcTE>VHRJ1{HovIc<50TM9P zaexB^1qd-SEW8MCZ3X6t00)%Q8v}ETQQ~UqW516h+E4TVI`>W$KZ$Ra@m(Fr+dKgrKpg z^~6G(su8pUlh3536KG07e3qaY{w7{&F7H#5~ zo>xwy51m%&AyaCVV#h5#bW_Y62`-U?&Q7Bu1g+jc;wUyQA-%z%rb)t@tfHqeL8=GV zp3QBKplqAZrYc)S8-wOgC7-PMlI5#MJ_j_8sw_h*WhgbY;RBAly05ESJf7HngCRHo zCxPIE)T8awY{SafsMCzQeNH_cmECTiTa`l^2t17tiV}W;U*j;qKi&L3iz*d#gwjob z(tq9F6;khf)=tdUc-Q>{^)VhN?bcN{`f|E9DtaQu=;E zp1HU=`5gM3j4z*pzf>iSQkql(($eg5X-DD&cLL0)uu+(z_YHk%zEu!jsbpO7rmmD5 zN3ZHrsK0?0$XCiD4YBMSfU+$(Uy?8RgiAdtz7$_-19m`im_uKP|7w~MdjA1NXzpJ; zLhr<(K}38L9gU_|Gqz?tGyshnc~{2f6OHpHZ_$#bFB&3&e)|sp5BGn&6Px3|o_a>a zeaWbhUw;kzl9Z89_GINu0@A5EPm~p)o;gwpFntMZEhPtHjOl4j9GNlr* zp87CbvV2*pnpCGEtOl!NR4qg4>Rmm@mlYS1>vQ_DHFG2RM6#-_ntrNB6Dv8-m*>mD zzYJ}-FWm>qeYSl4=dS`NXd*&uv8yHG^-MZuFx5zR}S!81BSP z0J{h9z$_&{d;#jwVStk8(7UG+s-Yct1TQHBIwR>tvi!+xRsK4~!zj=yBZPm-l&-Uz z?2aA6g?255wF7)$M4c9t_Il{;09Kp;TQNdQDTO^lVNAI%)7wXYd1XGojiML;MzB}R zd2(c|{|qdkPL58*Y=alZnW?;(1FWWFxWqslQydy|G77qw&cc4cA{?0CPxkqt0GOqO zDoM%!qWehS=wQq_GInYhB(Pp2W9BDfiNYm`fG_xCP62O#na#lq3Z@Zrv0%SIkB`~K zJm>smbq_|zAwD7j@i_y zp6{A(TPP=!GeO(Kphm;I(GrxC>^0s=PQT8*=$Z!ScuFL3W4Ne3l2|`oaT_Mn(mwy# zYTN#p}xVG}Bvx#7jJ8E4d)8Fm&#(>F%aYZfe#^v3Y^_DEX$49rg?U#+=P6EM#; zg{yZ(igyLGcfar0b4LtzyzeN6p5wH#UnU#UvZXR@&Z~|qj@c$)S>{Mu$@{ia+?%A8 zeW=vBLJD*-%QI(sy(i+?63VQfK6^W7L*U%kvS+H$>=|n`Hy_4r(sQm?zn@+{vp<@% z@s<3!zIkgfr*5WNYN>1P$w>B=g}ssNrm$mM)ZxBbb){-{WNs(ylRXs4+`4cwlG*&e zgUsAyRKOS*_-G0ioQo$Tn;(9sELvC!CKP-5-6TV9(JxaCxf{jmQ_@gLZPHch@I)k|9B zIk&8NzwCqAv=W$5OD_85X(Noi&G?_o4VnZkxKlYNM$Y!sGEqH2=BGzAVOcV@kwq!8 zH^ZYnwLJq>N`hRLK*;8gi`y=g3sD|FnZO<7dqg9}VmjFpZ%?gginmvgDJq%T^T1hA z`DUeA53N7HY;0Ilz*=n=WUSC3sEP06dud#xw(nS@?zpU|U&*_T<(0?#K?iL88RNf| zFktJ*uNp4RMA)@z%cVZEEIMPh`+;+-hfTvtOb|WJl=WHqcvGLJji~?xdbKL`BpvXV zZ2yBZZi4D#7Yv zE;badQHC^5fH*k@46>;t7HXi6wUwsQV|dwR6ig;{8QL})4ChihMTB2UOwUJ5ddx3U zC87zSLILpg!WI*m0ph=hjIjS}VFuS3Zw4e4FCh@7Y)KH9A7w0;f`p=HTF`~_S&AN`0ySi1}GV&8~=PwlQ#gQEaG`8`=QQHiAgD>C#0$jM##6L&ezjq0se z&t5qj*#Bz#%k8(^WndNkz+L{&j4b{6@=eCBHjo+7TG$pQ0GGkT=sNgE=;wCyKbCXk zXzy|2S?O%qcerKmkygBPb+vjs9^pSkN~ZLDNEC|DX)OO3N@DA1o|*?Z5C`|BE1Rw- z-EfpcC#w^8knrBx2Wyx1@7llBvDkN``XMEPl2=}-!~t7Vsd5R&5N63>x7Nw~ga44F;25 z2B5&-0t##ex~{GP6_67R7s=b5f+R{ye?TA^%bOr_5cw0vZ8S!@EGjb*Ry~tLs2>|j zyz8FunzwO6sD^ZEmsa(-qx+DiCJ=@z4LvgBs(SMrcXl6+OKZ|2H2UPyu?d>>QuN4# zt?tu4rV?gYs|@Z6M@@qs!+{f zS5re%v}WXN&AEn}Gt4&G4iCe9VWON^%2X_OMfizbtd2s7X^X9ugsFW}Nv>fqpjz~Np<*{SU@o0OVwrJrU9 zoB3&~sBvFmk4vY*G+Q!y!}p^~w%+~$nXm!FmUimwr@*!b+%%ZgyqLe}4rUw(ISvN7 zgLgbd)0NTUQt+2B8-?V3;7)_JG+56)bcJjVI+J1LIw3dEa4W%cGXaL0OMs6i<|2t; zYz;_gQVwIUyYhIT>($3!etdoq0=LQgjnfA|1XIgnA!ivgclb;Pn6zA}Guy6>MjR!u z^2RxyIf@`I@fMeUgM)lp7`$|xo#dgzDMj-o?`#dH9C)8Qc(=fil)mIKSY0p!_atH1 zwj+d3cQCZP!+6J@8fbzTMNrS~CM_nmb$HlltjznuEU_J@FS*ucwv#q9x?rb!EQhi2 zE_nor;X#lX9?G7LWX#cmVePoiwlHPW(&u267M92djGSawZ6!`)EhGi&TGzm`mQH!} zOV_49nz}+N4K&3hA;hgo>=Sz^atWm7NP!(sRZ#s*lu9R;o+NFL43|m0S4tbeZ4oO? zeTOA)!eDYR#6h0yE|AskR<~AoSGG%qw|}dN+K)}i#2~Xa@II+P5Jbf?kG3Ce?>+u# zOGj7x6Ab$3XgLc0TN?;jexeY@a_&p<$fIp-?T@uS!spV=VvGD|D2x9&Jk<0ZD%VVu zDn!GKmIAvlp`mk=ebCD?MSvFn6jHF6H;0td1`s@8$2t9K&Xt_cpP20nTe5!QOpa#f zlfiD7O?1tj3ujl%a98bD>@ZU0g0bu2H-{IE-yQ+2NdRNQSv5BtH6H`cfZwKA}nFK zcL>dah!2)kJDDZb)pZ0-M;r={bJFzeOo)rXPp?fuWF+Z860_G60$21;>j{3J0H-M_ zydej%e6!bFEnmGM1bPoUlZ05c?}O>3=2K_~%$Kehk24MNoTO-7U9MLS1(SZMR_meW z3INy1x)*3hJ@{dT{#&Jy^lEsVq)_CN;SQyb3QVSMkJoYHvr>OL@nz#3%92HJlsKuT z@dU&}+$Y}oOr5mR$2slx*+Vm$xvlh_N$f1nc@v0aPY#eGw?gYYEP=+8TBrRAowOd} zoPNDheyAvL5<=!n_(XYKEU&GzG_F$ol#pX=n~76j_l`YX!|o4AiUJY~p1})!{SzY> z6-aC1^q^<(?BGZp=@{_vgQ7kt50pH?JR`ru0qs7Xt@SlbZM}z;RqYw@?Djwl!39s9 zXG{GyPlbO177GVFL*rYTDyu!`5Zeze1>+SE5Evb(Io&rhRO9G-y6<8Iu^&R8-pbZR51n!;Qhnby7;g@0QA@SwTLvI>|-P(;WS84jw4!GgVB;{}7#}LHt zVBzdtxYz0Es7#Alpq^>`V$9m-XXX?WnC@^`q)6Zg{iplTIXVVn=Dw4DK7i_DmZwhx z)`XzL$nZF<>W++!@iERnd}=ht0k3L~S?Cf6P@|AJk1|^(S&UHhmZlhYet7U{n2;P} zS5MFYPpZpfrqOYJFD2(_I!`#t>F1w{jP=he(B znX}gCHr%r3-LU7)r3CGHA$u`QJX#$yk3KUpT^Mz^u2x*BnB5XEzt9|bWUl@j&961j zw=AT+d2peBaZ9LpccftVqCe!=M-XaCp!~%SF!-ZvXX4duSGJL=fYX;x&+u2zUpYTJ z^xW^_nj)zgvyM!4yr%T>@%bM?}dOM&`Vn_q4YC6yp_@G=ZVx~}G3$$PFKXv$O$ zB_Q7E_uKnmdy0bsT7{ucChCicgm6l;de40x=T-CFsSL0 z69>a`uLz+S46|1-6^oNjgH4)zW-X?VbKKl+#HpyOOh($L#x0-*Sn-ztYJd%YcApW< zNXE7>_GU-7*XPh9Z_KT|Fx2SKJDv5;#GVoG=Gjz{-j172`QR47$xHr3aWYthh`zy=6f z)F!Dp8d_Ac&j!XU%uLAv7Sip3*NJr4RV1ghpw)QzmOy3dK2L_U0d=^hTD^l4lRQi{ zSAcE;Y9y#EBD==m$;si736H=};IH=J_^lD71tqQx1TwaX%Gwx5$J|tbmI5Vhog($A zgCE8G=eJ_#=5M2UQ~L$TX3*~;z=lvZ0qz;+$DRTg1~h7Vz{=nudXvama&pik6*}Ol zfSwG$XUYZ;N025ht=vgK(WIZKQhjJ8KQY`lB9sX|73T+eKWXXw6k2hQ9UqmM0)1+n z+VyADt|GLHX9OumJ14ABg?94v^E2epfgW=V@ndHNzJswirKug2IX@%r19apm;Gh7HVY7G%!Jz4~&sU zXzWSk@Qk-i!h z9$Pph8&m$kln$+Wx9qt$?71O(K7f+ke8`D1f*7NPxs#?VZ!=n$!;MRe!!I9>B$Y># za>Tdrrk#uXLz`M6NiDGQ4CapMeRrHmB;u%{mql|52t0cIcf&c206lhmX~$yy4|lx1 zWBTOnuiYtA4;!NL{vu0bK&j+NE zoIbnv%NJ(2Uw!CEoEf+}d}a8#+CWR#Q2-0jRBt%DHtehwf825w+;A3zH*CA*Y`*Di z{-whJDxEuHcHpJq7lvQ>6fpkc%4lWHcXHm$nRhNUFYf>0p|=k$?n3xMSSD(7_6xzqRo;`N?*erDR<;&zd2wMg?qCU)X@05jY?WCa( z0-T)V`Tp7R=SCu2?tSFHnSOx)D3sP3*!LpLgnI7oGZMXTKNttn0>eRjdC*k;n}?cW zR$S?hPfo-GPG80p5)JTk_^~0ym~)rB!2TtA1S?_h z1kdOO07vw!+a^4BJ$#pb-wzbY*B>7OPE?y-kii;U0_dTY3NN#=iLQ`Rrlob-(Cj#; zHD;>cF zCWTtLLO+eF-dR>78Avs$TB(~JEw4n&t@=G*vppt|J1`~5DK#Cg5-(eHM{~_G%|1(K zwoswfz5U&H$brveElHMSO4MA)P zEM@dw9rB1*2B~%$dySK(Npr7Ex&Y}XL65Wn@)!;gawrwrBijHp17y)pYg5u52APTR z1K5D6*`(2fApl#k!DFa0NH80nt@(yw=xHoK4fKyv>@*h`hK-eWGU|5xI5XhPKY=#< z8m4xS^Uonq*PA9fETKnx!cXj;P;A~ib|z+e`XmStf{-p_7F|Rx#nLnXHvK$69vRIM z%;zw(&^lLX>@F*Y>nW-PoC0SChIxJzc|orrOY0yqd<88L&`sJfg*~#JO8*Xdv|+`} zVBq9u5i5`vq=Q^Iz>@^&Gk%bsIKKH;k&x}0bkRR;_)uhv6wTB>5jJIOP1a|=p8uAA zq48T&I8K}2ZVn~x51U#g)Ain~k6(E_Y|3I+S!9+mjZVEV6eMs7q)GPq^H(o@=~AHK z^|ZOJZ#@3m<6qwpO4=OewnUTCu3mip;*2?JwKJ-A=1Y0MpBLzw+ZswKf8SaG?#JYu zJDM^Y=Sse@>9tLuub+AIOekr```k`&#NqsM8O&DFNIdiCb#q|fS8cNwU+;)yf@5@R z*whd;*`BSrTyx8mal@1m7=Vf`Q%2ZSAB95OOP4Rr;;t<_oB-*6YA&h&qwHw|3-S{ph5vi;hmK>ge>Xadhg|wC1s*K02)rcy#Kf z7mv<_6~53~<8JaLta5aIvOb!Dv+5n4)(1Q~b!*ogoie|~10J2aHxTPOI*odikkyXP zbJMyx_!sHi>_j)%`Og0V{bc7m{}R3aA$dcH z?iaht9<)&b0yf?&!^?uTjL^~^g8gIA(F+t`sLGep9O-+wq53RRA&4J0TBMH zGz8blBiMs~1#h?<85}&rgYHL~lo<^|IfdS(RP5~0LL&Uvs0=^FzK%CP-J%j*K_DzV zEC3{KZSVPSASFAKhA~d)@V|0^{b1+d|W$wyKw+KRLS zyt010{*P%15RybtN`f)}E_ne6tm82^!F~x)h#I1xa|*wW_x>Hr*kz*Id@!tB0|mQ_ z(WP~DIrqqA(A~eZyGgqsZaO#_l>lG|nl#!bIA!^t21leGmx2blF{^GYWTdBBz6}8E zU_J~Sgjt*Bm^|Cm=1Jo9WrAj&)?;m+r%%IB4``|57m0F7Hq|B7c_LY_ga0}f6o&o~ zniw-fV-OD}1YuD#v;hLEdqy7fOglc9htnAIL~9ST|x zNz-^R%%y{#buDf7(XZr2xV+D`1C+PxT-nhk$i65MU{;%Ngq#A5HVKz@IAvVk#JU8` zLAL-*v(1Y`A*!ppUUX(EYXyE=Vb5bborki05>Wdcn#B89%Rk0xfGazA^V||Wf{m^0 z87<$lWXA94!6WQ>r@C&*g5P%#5)!L4IOcCK;W%PodEx6b{XDwjm`N>Fj%p3;S@F=NQBRMVRMG^u)*pax5$T2vz_b2R~Sm&sV`B6@k07D#Wp5Vn>XS?q9Hz+|h~q;dqB~ z#4mwccJU22wTV)w&&77X5>~&m`?>VTNN)$A+h(=gL(ev;4c#B;j+hP(tJRaXLfv}# zG#dB3iIQvpcmuFv1qV!A8i6F$2*|g{z9cpR7<<4m$_S8}7WIb|OIv9Kyt-xMPCeO| zgz=GQDxf0$xh%J5qv2?dsvRl%Up{`+k%~acI9D$IK8153AM@z&m+jzJ!ZVCEC za+LHPk<^^(#Xn`P(l71r-w^(qD&wzdkQ8F#LAazA6d;f+f^eJe85qNjIO$Pf5*899 z$gbwsjrIy!4oEPOF6DKAO&FsjBc2{GijqOX&Hu8Y5#r-sLc>v@K}{u?w7~ zES1SBGCM(qD&TKLG?#$r7)Kui${EHcc^J?1lhrYH+x|OL$S!z(f?F64th!wn;Oe|M z$~!nlV8~pwiM7sd&ZHF<7kjODycU3 zS4zYG7o*I#Gjjr^w=&DGI7k~o>D<6K&b)RelwLo5hHz%$Yn&opgIoEvH}YW%uQqJ0 z1496D{Lc=C?RmIxpWPaG@_k#Sc;z2R3_DAs?v$&iubiH}6ml2Om~T7XkiBo7y%cs9 z-A>MpCf5iT6}M{l+`#{umY{ntL}^m?lPC>z5qy{e`%p^XMflJ_Gf}QM?>z8psj-jApdIqbNNwdo*>zV(m=_p)i0B7gIl)pyH$p)3S5b- zOkbcA={;A@v24qGrEP`BEcPF}%E{kqL(pGaoX0W}|2mnyXFHEoYN-sgW*BBeoBS$R zpLBg<0p@2BF+V%ues#FugV8mD5QL?S6Tu*?@!dEc`91JrZpDlj1D41R;Y&jU8s(Y-ECB^ z!3t$O>1pc41?ALL!3=hvWiXAwS14YEDatsZSH>f8wp-=2X5F+hjRR>dgZQf8473iS zg&G=)On(RNDfdLiJz?liPlSo!atZPUwCT{3`hbK8jRtd?v~+74=e%~vW@22(p**|S z%wWwwWo`_r*FT7z{lEtSdN01_g@vh~tVmpPUvvsTg^|c>$4XEoPlC zN>ImmqAyXEk|4JXDWCH6CMlM{73f;S6{zAv?f#6^cc>rCkxJ>lB!vRwAeJ=dy1%6g zRdJE_5QiCa$_;{&Ql2aNUo%-@z5qn{cNl-8GX6bU5;!Tw#f~1iN~!v$Nj|sMn%H=<>MNx~&Dly{ zFhd=;t}jFo4Rbz655X_Yb${09_F24!Lq>0=%u}k&XlY&RkAD{Kh2*qHYg8P{3AM6C zdCUzNXH)hJw)*r|RCbTFh~*NYw z7nsn$EU3tHNGUK&>M*B-GIq*o&P%-zOGt;>!`*UhmM^^<{J;$&6%A!1_Bnx0frQDyyjhJMPf%DbKJU z47=whhnYYUEWv2nI8|9wBcJALBuyGQTdTHm$6ukbNJOZtC8w5W7T-midX!!;eL^@7 zQOSFegI^*KU(YaAN`OPHC|T93E?fOBCL0)J0$3|113d9rKqnfLed6csBR=P;RKe2U z`LW^t!CKP41jTv(0nLCeH~|YGG;1W5DB&ch`|2B;Vs7cSu~&#rl7BOWmYciZuTQBnIJF3N&75M=j!L8S;LZ%N^6$}40?J}eCK}x7 zZGEJ*iyx)nP*Vm>pJx_SEW?Ub6REho@4R~^Z#TSx)cmD(_U=nm%&r`ASoS%FMq z6AzZ`SZoR9?0LtD__U+OW#}C|YMegwp({7g7;@E2x8CMbXB)#@A#t^(g8%w9mlZID z^BWgNL;1|GdP3UGgpDK^2R^Z2a^?K4u(kPD(S+pLwlJEVm~yQul(+$`*a=xT6N;h3 z;n}m7&tBXA(xKl!6x>*MYvcAC8@Io+Ex2)eXyf5f`jN2p=xs-CU}wlt4vh_oX~A?@ z+o?gns5f;BnTxC=yOfSdVQIUl?TFAey*`{(K5toQi|%fX)@=m_xToQlW>a?KFAXM3 z6YvUK^38-iRqLC>)`nlPBIb%hu2SlM2C!e^I~NVe7rlN2oO^fDGp6gJDXG)d!Y997 zzAI{T&rV?&D{5|VCDW}l&9l89a%r<|bItSK2)8-P2I_WU+*0hwE;O3?)mXaZbfO#9;SU!Dl=EM!Oy=3e3!C<2223cjI znHw$HEC&?U&}#u26t$;J*9oW;mZNM%6N^ParRd&bB{Vc%l71?+k3R*5PXAzSN8i~& zKQ_I=H4()YD-gwWR?EkRTw}pq@`CAQ>?t%1k+LH!iQ#xrG#5CY9HFOar_i8q*m(^S zR69-zD`ad-E`V<0Y|<~m{*nYs0?yOVVbSEULUxT4N?0*Rf+`0SvRpHVQ06o=zdbqZKd|coI=rzH@jGnQ9q`;~4^{1W9?V)U2t*ni^_LjX?65>X2K> zx;h3SFn*X(zDeeciMZ7aj|$BfvUn;c{g978dC@a5JPHqIF>1X)K(4=X%7Nm=6MAZs zTs9=u&JT|EQRk#s533KuzzD*WPXYm^CZ#t;?oB1~CTON4p?tp|7XcM2aMM%e@%oN? zl*Il@8ffatDGc$bAo5S&1Ms6fO1MlQv3g9Rg#Ob6VEXz0fYBC)+^vQAr@AG&zO3#^ zQtyN!Yt9e$*O5}=2`G6QK6M(12dXCvhPcOIHwXJVAC)c&1fjgMgA;vZTo4fOmryAl zbSVMVRW@lyjk#O+Qcokedw9d>Sm_`As_u#V3@_4>rZKs2v$$!suZ zLV=ul8_3eyI52XEaSt1{kzGqBQ~p7t7;iSW8y2Qe7~nNJ?%n*~zg(Q>P|9VRS|8 zT<`K91T^Cbm}!(s_G&CTXeEXe0^x&|sSw^ykk+V#jxNTyON*M+p+j)S=ul8{+~u)_ zO{k6vM`LK06Jar@ip;OV)&7#w%Z#uwhj2MZ%99!X7_$suXC$?9JhXy9ttUqt3s1TH z_%leR3OVAWULgce`=Ag6yY$>hh;6tu$#W&Dhl@eyqRniRD5 zg%neGd(-?8Eqft=S_Iu@cZAv-DSVQeLkZEG|23{T{Ld+Xr4TCP>7lNSlY*4*B4uWRaEyp^hr0bCc^{GY_vHPAyniC^ zXO#Ho$DtmqQ+zW#;qgZ3B55$txtUh`eI*n5Ib5_rsIk9zlz(hu)ZJhcbFQ zP2LcBZ&F3?(pxT-`~-PVl1JP!{HMsfO5QAa8RY#1C8p~SzLq>9f%6mOeUUsymSPmC zzo4H#ChrD$KOygL$opIJ{&(^iH7kz_eV9B(*J6aMKKglvyw8v)6Q=kd(9ds@w?G~v zb1`W`Ml4h4Uuo1xrj3A1qcY2KjK2YZ94NVfg!yg?XPPzp>5b_t_#(ZLEWbeQB369H z;wzB6*cQq*Q`*n?QuH9^c;upgV(@J1h2aTynzIc<;AgPy0^B`0GPsAoi{xmn!T(Ln z4Uis;#=BgGkz1}c7>j>yNczCw{=nd5|Lq?b67lj=L*Y*i*>@7trybFx6O&MJ8tO=n zCOD?KXj&${vX*4#1`TOTE=z$aXvkSgwdCTN2}%Q6n1}{gKqQKSLR11Jo3O9I*=M#! zLCR=}a`}P2D3=#F9_4ZZ2+I!)E~Pn=Ow-P#d_!8sbP}MejfFu&9_%zw$u>h;E*hGV z6EvhQ*|;oJleXj#pKgOCab}3B=!kN;0jToK3|ONeNM$ZLtwPddLvs1F>#h|S{Ggs0 zvpzO)MmOm!ci%G`Oxa8H^R8Qo1xo-_4j7ZfSiF7gN;7UDX}~SZ)fQv^av|Oh*m1DE8wWlP2Si<#CJ2ShVCeWTXG-%T^~< zou6hbSgzh++`N3yXf|ek>}n)ScJP+Rag(xpdDNI>Y*;RF87r4Jlp5QN%biBpZeC7? z{n}*@8Ms?IWHMMFhx)lg!KA7XS52fW_G3$ktCxRNhqxN?N7S|-N3Nv}+7wvIsEv$d zGwllE??d$ah!P*4zLnw+DDlMfVcDsYr=5YKmDkzK zwEg~bae)hply-NTPA_@R{c)cE`OkmH4I*aTU4$q$Pb1!9%aNIBGhx(K%0RQN- z0>?!;4;SIQoQFTjKPNXw#fZq>`iP#r4G{z0x|7D|Oc7HytvO=xauKVS zkJ!955&KTgBYItzc#qyI!Z$GA=;k~Z&hhwdVsU2gVZauz4!+eZz_&5q?kNIX>~X*^ zVQES|PQYcJa`+XVO88ZtYWOu?q(NP^EMFb-w|RFVrk>?)VD-9K?naio$F$Pz?Sp1^Hv!hSvIgYSl3Y+C4)t(XjI>A!Zc%B1% zlEJl}=K-H$aGmEg;4=)~<~a-a9E0mUqkzvdxZy>Y$Fui0IBu$l_Ipwz3xx!Uo5IUq$)NpK!D zbZE<1xDuETE(Ii3nwwSVQd)5#FdqrbPX1VMq2KXTdSpP!=GXO@*Gi?o0~=#EIi&?e2%G=*>r z%u7LFK1FTFcBiZT<)B|_qmSC{%a)uiEI1cbsGFyW+Q?jCqX7v$=pUU8T<`XyEiVUL zXOPb|335-W^=SKzNd59ca5A7pZ?ocDELC?%T9^yv7|>!lGCK=u-5?5vU4AJ54g-VC z1?PidV%&B#joNtZgex2fOm!S^Nr4%K(S!3djQA=qyMkfMrC_8xI0c3YPWorvE)Tjq z2=0;Q0#iYJ8e!M;g5;V*gLX1(ScO*tz-M88_6Cx={P;fRLN4$=K8YBqSOo8r1$ z;mg2I>2CHNU6X>u$&AFPZ(%;{TA1F1`0mgRx21!Zt^WD>g@~V}Jd5ArD0oB`C;ift ztef{;mGx7B5MBoGwf{y~)=dTHWOF2NJ>p|&ssh)EB=a*q78dePP0^3wlx&_~obz1{ zfPk`vKx9WR%C^hFDUiY!1}OqFOkhMEhh@X0e=+Qzy(k-jIu#{b=KR-vp@0-zm%;iX!sN)Mq0wK9LI6obbNF4c)3f)zi4^UX! z7a>s=mK(JZ0j1MGPnLBW5-bpwn>0xl!jVnW6i-SE;jqsifea*qkD$9q^!Xw;Kx-=y zqwh6@?^P~hHIoqa&7=H}YMAux&V_13t85MX=a9)q14i(j7t!e!h7=1>n1$;&EZf3? zIX`_(KH|`dh~%FSXZu#eY2_Qk6=rm5gVo+CC?nrLli87ty|Uvo|HzG>QSL=9!ej0v zc&8&e505cYU}L1{(E%12toP_629E(_p;6YIo}8Ja0Wv#Caw1@ikNhJSD{hSQ3VXkb zp%r0u+<6{@D27pw&Li%m!Bqb~RS(_=_sKd)57{6EFj&sZ;#qLVTO7V!1JS@A2~!PC za%*H^VK%(;61w&>hDwaIzB&IDbU9?BFDsoO1!SAgmz4nlpHFrGO}93Ls2;9iD1ONC zy3U8>#rrb!O7yc|j}XgEX30Y^M=;u9e$)gY%8h|gU23}#GRDorR~`?K)(QVZk6-`$ zOUhp{clIq_)`uaM0v!Uid3IXbjRavTx3yIo#7j2$d=SSG;A~THR(>&ji;*D8t(TN1 zt9GPIc0`Z=*7MRKgcWo^7D@h59ZS7=2ka#DbkYElNPFSQ`e_Ub5otf*4qiHdmqKrb zlys2N4Z(YfBAoD|+=rY_x91^w@$KxDM+Nch5#cj>{Qv;y#f>WA1wLpNA*ue-#n`4*w1tP zf0_S>xlR|)NkuT{@H9yu@4`M6poN& zpI`9#q+w*1o`)xw`yhQWCQXtYl!BKQF_a?#v4&C)@u}3OdeZ>HCGdWWdo1AV`Ou*9 zg_4>Q7BHv*0iYfKs09FV#a0r+qxTqg>YBN?jEqrboisng>Xk)|SmCAO$Recb=O8t@ zTyD4f#VjXZx&efr%E`peA-G4a41*6jJwL)fWPYqU!`^Y9^1fZJge%NV-l&Q0Au@)w2Z?svNq4f8+r$$Tx8f zK=~p!0x1#VF9@R=Ii_Y&6`;Z|^g0*SH*lxwmFAF~8_)&$w}hGR{vJL5`VgcXW2Coq zvf$n&^FHYUN(>VMt*)pd<;;ULue=_ThU8jIaeSl)xCdt!FnbCQxpR?=$ioJbU%wA* zlNHDR`17|LW8v?7@f%-ERkp1(->d9?x3c?Re*W(tiAedCmNp)h8p082aWW!{A^+qR zS&vb4Ch(TXkdQ7SnXI40%s(QT0Ea2xbgRpi6}~Di(iMa=7NKwY*Z4Xu*68;4>?P63 zbP{$RO{?c6hX6d}BnP8#;oL-s@5R)I?~Q#149oW>2H3AW znn=r@9NV78&*-yd8mEz8IeQAE9Ir?@jrZ$=6X%oVZAo*_N^8p8lN5ULP5RVdVn=pVf|!Y1mPoqE zMyL(p1u6VdH|q&yc!D(Qa%~u?x=2y(f{6zs)r;BQ+J_RJ*;`-DxA8-+ny-3DUc4+r zul2f<86CaPdLGjI$#0+LMnF{mC?>LQQ`?>!NHBm2qtuiQ&jivd&nR{MhDRNc8n`WT zlcZ(OsAj&UhS89xDc`1s=id`F?`6*zuh!+|`m;ExA!N1<>D*jMq(OL?AF1NFYe*xF zN!KsxLuO4WW;KtdUWZyo{=N6lptn?#-l9oYnvYkZMs6%uBI?Q0s!UChe>ZaD9jZz6 zQ`1Y(w8fydMJ-kRjnWO6jC5cevwX^a(T!Q2EcH~UGV*`QGR5(5f0n~En2NB*KJkp{ zG%0kNHk;J&E=;P_2UD)T4sB1N|2IBe8tyI3JItS|eOkLbdQJVgDcC2fFE@qbav~6Y zL=(ny3a2(;PA08xPS>$qo44Yce=H17ozK#R8>L}l!!x>VLjZnyU>xum_he@sz?U0s zcrsOfBHO@uOqjZwmxjm5%;(BlRW+7lvL-Xr@qQOG9iJNOa*-kvb`2ebQF;>A!mtar zlbPATlzT}WJbC{7k}!A@?);JtzaBA3ckwMjHilqWoR5TM6SMI77OtG_aL8to#C}Qg z-;fQ^)dKUd^D$c^+0SJQnIL^>9z{0-mP zBWH$XtAY_z#$+2aVM4Q#$k=D_heA|K5i8`#>60hUo*6!SQ8p;($@;k~u!_o7B@-Jb zI&9JsDv@5Hnsit;kPQpd)8RlwHY)EhENY-hc8)v6OjaUItZ7EPAO)t_j92<6$R8#t z+lpzg`Y6-iBBeS|n_0A_-$kmgQu6PCeo^iN17|M%x;5IoF6h6u#@_tgYoCi(+^W9UxaUseosDK|b=_7JZM)yte#fva zIu+Z#jw$uXeWPW0_g7x|R`6a$_nnIFHDmYv;;MVaE$W&ciru(%@?Oi}otD9L z#bDZga7`F`ud@Etxu}1+Wcf%I>*dIrSf;{@H=E;=-)oOo-wNJq9lFyxl&%^|7azJS z9)76jsv0r|PG|gD=c}Ff1m_*W89%<#pA?*H!mf3}a8D?`Bb3Ja?h2Jp&Sq!~f|UJu zR=BtHiXn*9G)1r9OVH=Zd)D+Zfka_QPO!&^TrXepki6Kj484+dC)v+DdPv6}Vn2r7 z+YpMRJJTdwc|trht7$|HY1J?bX+v}f5eh;u=);_&O{4b;c>@q)MQ!Yupfx(>h7*m5 z_7bf*aRLI~Zob=ifwdFlEuaRl>2GZ(7rZE$yKgRJEZ-LoM9Uxh(X}qQ2w;6&n ziwldljQ4Mz+bv=;l4}w0wQBKI+l=9-l}!=Mqof*ccvua#F$?DE_Fmoi_RVWj%Ys>; zT^TG2>Mh!$UNBh5>M0(xqI5^EE@T@sQ+;DLPf3>skHYpMw)Cm7&cax)ZbM&ZCm-3G z%CxD+o3xZhP2~l#ts7!1w6SC02`!HR>ugPKL%~$TeCmsv(5P|?8o?5a>D$(QQ-|)`RJ#S8{yF2Nx@Yo| zPW$aV!ArSVw@}z;1^iq<2~FEC0|$GKhqsN{m9M6NOLX8zomN(;)X(x`{bsqw`!=L% zV5wZaI&InGdp1qgSeR+*z8ARaNLnKIPwv>e0C0teoR5nn;Ii;A=4ulykAGT z`&LJ+?CS@j$JcF+oAz7RZ|iPfOttM#S`Q?}18fvqDxIBkvn64yV7iqB%Ow=7gel9U zYH$gHF->FZxNiS!=(2yQ4ng_#&JE>A^tCM+k%Y=#GG@z>Y!qBUb>tTW-O3`aTjt$M zR$3_T$=1Mr^eR6-s--0+vf|IXgY!PDn7$Nt@5&bYQC&Wguq=#Ss#Ml(-Aa?NB&D{h zsxSgsRXj+*7rE1DnrVUpcx8%&Dn8SamEFYN1d&!tZ zYpLW(luOm1JhC04Y`u8$>}Pz(&Yc}Oc>-2q+6*G$3+ANP;B^!+9+GvF(=&|YWh1RX zW7#3k^|D>5)2A%`v(<55L6DT7@@T>?y$ny*vtE`=3fe9yAhY&FO!{!EAwzm zo>QfNfiwxY@UQWa5RaOin1GwC%foM20N9FS#c7)>+O}R)66;JC)kizl&9>#SH)^9T z>!r?b1>(mNMd`Yox0};-1F4GL>C)Y+CF#=rtNwK9p=kGdMPS-&q`vp-$2fAxI2 zV(6X5bj2~uG9QbAZt$UnGn$g7y7=VE)gK*98G3$ZFeXh631`Z%i-EQA3n@b*16x;& ztLJ~REoC^AG3)JOwCw{MXDf+0V=sKW=B~9a>d(%xztI!B7QY&wOjIVzJ68H`U;T+G z<=l5y+>iOWZQC!4%VHPeb?NffyTxtMz`DITF`U@5(wMY%;ky*AKQoz^TVmVNmimM) zWop^%XR^edwz%=}o?mtTWISa$$Py(>cBU;mZ#z?_{n_6;(v}@7$5N(4+25UMOXtdq zpiK9ncaUs2V>dbsk2#~>@Z}Sjjyu{vC{r3V{_VQE)_P5Yl30x36XBJ%WO?uHV=3p6 zHSuV^wXY<463vP6WO?_>wUqPVU2zDd*&3iuG4i0a1fMQxylZVLD6)N}WJR~KCt2Qi zd;FdAe||aTJaJcij)>IEh?K0}nJ(XTx44I>)R@?ws9fQbR;&saQpuPywQHzEKQnqA zHB6^BL{EHR;)`B{tZtqE1&K5-UB%Qr!TW_*|_@^c5 z{^!?iPArjF%QI#}sp&CiFq*zR^9XAzjSuadqc>B>SxTbryau6;AS7&<2GIiGX@+uj z5ix77RhEHhO;R?IhVO%MA0H(46tLC9kEisc>3iJ4D*lk0;YsTuKQ@_R?+3LF89fDL zM7|q+O`y^O8Z6a^434Y?{|j>8&-r!!V!s0|p-lx^J%%)!LhVPN*OoSDL5BQ_;X?74 z4!;DpXT!AMxZr@Mq4)CRM^(cq3o#(1B^Odm+Q^tNrdQ_#ERE@jT;_sYRIJAkMhgTewoLLvDR6{V9WRolvwQLyhbnY75iyi#8WUk zF^*qnX(&EB&)yKPl%eyQfQRX+Wl;Rn$}o5>D8=#AGK`3SS{Y`q&Fk>Go?36*7QjQgRxYQEkrv?Ld4D`UpdY%ekP z&UzQ<8!TmdjI$TIYyMkQH4SLlP0Ebi(*%v{KFYG?F@Urv(_*0JLdt(n`iZA}%m%%r zf@u(yz4}nmn33unEAmunwF&T0c2=H^t$u23%~NA*pBh_N7z^8qxS58O#bcW`^~SWo zI{O7>Q`c)#gX?uqExSRR8uF~LESEO+3Hmf{9_zLg%B53(nboTPYgVf=LaS<4v?``c z5l{T%sZrARP`q@Lyzj$Pmb65 z3qo5q!xUD=>bNZwKN$B^)+5Casc5ig&jhKQ?76TZ6>C?AUiy8+7O00kyV;t%^v@8% z6vYlvqbU9{g|h7_MElkq(X6^CM6StITM**fFOW1$%e9z}WBPJ!KD3Q(r|EVU2+Fi;Krsb?D@p9hFg01si6{7XqXnf)@+2^>e3C@0w&z6*fZZ%in(afEai^pL99PX);b zy8?V&ubwP}SYpi1$u5W+Qi#2J#HuoIV}3pI^O#`?Tjn<)f6h|t1|w%@pdaZOF+MUX z=Puk!)$U8!_;khH^CUqkH;K9w^oMhK`nJxajiNk?Dc^wYwEnr^>Wp1| z<6!jo&rO!)7h=0(?eR8JQT4Po5%0zj>m~0zL zS`NKamNJd}Y>TWB6i!*oH_uv?GCdFV&1T4u`ld)M?O;ng5*W*j03epRfDLpWtW<(E zumC}9(yJeDhLBg;gBU;?4+hE)0dCsMEJ}Zh!lXZg2P;IL!?Vm?tO?5Zhf(@-%JgsG z$zsGG3d>fQI&yttBNK=EOVc+banU!et!n3pRG(b4dDl>!0h&{I7}?M!5 z@pWq@k+gaPNfTd6TK25gp))c@ktixYo1#1@x{+d_=$J6x|5S?RKDZ|+TF%I~Q4TCQ zHw5P?Ysgbp`U|4*e<1HK$xFaf*&8HOj4lSEQbgK{y_HxyQIU9tQE>y)4kj%JS0n3| z3Zmln4OC3Rj%YcsIzfzCNJiO+z3dRh64{s-Aa$dLr6wjP3Wov|&y>1p$q!{vg)VS<#HLY}uD*h3b?qsGe zN+5Q^JG1TgX>g^JEQ}V~KIukx$AQ-%}FxOViNk#&;|jAbnPIYsv0_OHSSH_TE}wpKOU zuG@{bVUrvBIa%uZU<2b<`^YBew2)2iSM_F{;a3%Uo#Am2yi5_Nvn*eXb;Qpnc$lgD zDSLZbXoopbFfWhAPR6e$8dLU`w9ujiebSUd;z&YD*>|Lc9ojORQugg>VY{|WA#rp? zNZC8nLT5gul;};_Thl^oKBXs7n((ZYrtDp5p-UTFf&z4E@B^T);@bCQxmOcSiAyWM zqB|{g=c_}o7BQv$pCqNf5V1`NNI?=2dZpya#`b7yC-gmu5)csECpywX2ewUMeZm=^ zPgi!QOSKw!@$q5^_48Rs%eV1;Xnn2`Nd@Q&w|u3M+uQ zmJ!>r0{%J7ec5?KJu6e08L$NzMabJ(h3bq~dJmHx|cq z@t(LpQI7q}DPwyE0)NK?V;N2uw8ds`ouLH!*erhT`SvSGM{~;9LP=UPM!iMEi4~pr z_kXA3tnM(;Zq;&RNMxpjY^QVR>W|O$=!OBLm-TFWxwHqcVk7@p`^wMJ*Fcsq`Wno% zQyE^|nW0w#hidS8@?*&1N8_&6jL?tY^o00YOv1fi(jE)L{vcR1yDF3vEEUq^Obz%V z>tI#iY+Ez*#AsKJskPfOPod>o;fF}SDiU-wc`{fznhL5?2hZ` z@WJi`9meE$-!5mbRdy6XIp|iP{O4HaC~JgfivfpCTSVmSGsbV$)sT*A0=%j7-lsf} z^AG5yll}%T=@;bv5}wj)>RC-&w$~I={5Qn21}S5}eRD-zpD6i#OJXWrxg%xnT$xFm z2a>`-p6%7o5|UTxh!z0PSj-aXL>y!p7?e|xV;?3V_X!B^Bf__`+KBKcaQcmWTEU`s zVpw^v1e9zTqJUFCNLzYXvh0 zyzs~3b%o0l*27A!{_)Ak2il7`yt_Ua3Y)@&H!hGGr^hREO1sh8=M`k2(B2; z0apT73g?6?gDZ!tfUAV7f~$tBfvbh9gWCpI57z+af@_3pf@_9rf!hw(3fBhL4%Y#< z1FjRU3$7c^4Yw0+7hDfqFI*p7KimM^Zn!;gd*SxM?T0%6H+UB3EH$5bZZkesICfwo zJ-Afr-$Hha8h+FjPMTD%?7%tBKFz^{Q(#iI7cMtWdX9RIJ?k*X zM59_1)+<$NX5Tp8aU4>8wG=nAw#j^# z%s0yXA(?NJdB4nGlKDxQpOO}Umh@S8GCw2pmzhOM!e&69|d*0ZasaNoux&c4X{ka8a8(!$gYoYR#2xc5K` zlXS)$Tz!?UpQ1vzyYG_uW?VT#ry-S692UKX!eia%#q(Voi zRjd{pXS8JOR3CO47OPN6nF>`@t59vd3b~q9XnVT~?dVpaoxLj5zek1k4XV)45nct4 z;l>CRevUttg^LKGVnV2h5XwS8C<_6hEChtI5D>~jK!~8|_76(AiiY^&-SX|PUcu%* z(fqa3ubz&z+$@hTCa(VejZ}3o4`DUn zma28jwwsfQ?caMTK9TG=aNk-HdnIjcPFl!fU~YY+6Rd3+PSCeu30pbV_1?CoJBI3L zAU1s8S|0Pq%fDaudv%GKRJA*0-AO8^>Gjj`lK7FEFMt1)@4m9qf4e-@uqS2Pn-=$K zW52NSLaL!JW$RCi{S|(B8Dr`+qZB|9VaANbFk5+>jO;=p38jsfVk+H4u0H&i=bXeSR*U z9n;jSfIitRJBkfuKZ&p{?${wa;%lfQcgM~_-ujS(w>-&S3L;39dq{*am4!}bYwd@d zUoVok=6Se1*~WxmF4^qx56)%NSk0dN&I?a&>f+JuLCy5AGoHDrt`bImZmNnk)lpkM zT!rYoISyuNa)xj`Vzfw&&%ZaF%;F*{k3lmv(T(ONg!!D$n$AvCscCWLW$x6f$D~zG zJ!X$ZQ;&e7R9c+H2~zplgEiqSRq33r$5xO|Kd~*BPCr2!Dtq#iCy!mtoqK1MZWx=I z)byh*DwKcYT)D?nq^%8`6LPE9$la=@9VL6qh9`g78VvE%O{}Bzw^{-abT3OdbWA$_dqhz z!}_a1nxjoR8gPLEUF{Q#v1zwkRJK|RedVC;m6 zyJ+04%@r!1AX^`MG%{ef8kT=!6KAoCB@7EqW5qBbbM}*U$%ip^mqEi0o^t($al2RU7AyY@FZ?YrJ4JZO?zYQk_~q3+>Nz#ryZMQV;cjz zxAvoo?45`kuB=tm54zcpwjkloKi(phLuw+##54gYWD-e*f&w&G%m$ykaPG`UrJB-l ze2DH@T`H&bc;${J*DaU5$8lepjDhU{P`y3nmBeWRY?3&R)Lh78%64?e#)8zWa21Rgtg zeE8_ObEkaAPo81xd(anYAqF>hUiSA6?8dgt*#Pc|pz9fk3L#{B~7lP(FN2kan*+IPxyEXBNlF!6KD5bKggl?UpUrrVdPS7T{ zs)3ZktgKH0RCcz)BFKg5c!{dls&7TV73ZM*N*6$EHq-oLeEl$#Tb=i;P48Np-oCbG z-Ggfy1k*jC{Eko_n}56ietlD7C{^DJ)mK;X0j@IZe_$@YXRf(pu1T5eaL~-=c=OzE zpNqCZvrXcX1V?YQb6v3C6Uy!gW%q=dJ3>vO|8`4Ks96*Cu!yQVLecw>Rk##7-sG^#x=!EWDR-4?mFYVkHb>9~%VqaVnS|3DwNtyq3|E>+#XCJfvcthgxQUBMX}Nw)4!*X>_DwO-crfsWVFwi}bR*lL8)nRpF$B!3&Dy}u2SrgCZv(F~oN7F4w*DD$m z{mG7fl>Y$o*ES_vhSD`dNwIQWth^^S-4UB^_5b1iAM9UqA4xSIedl7j`Q&<6AK}|f z_zpZMtxneUrAzywr>M#U_ni&5bcyGGSiH9P)cv-@D7fq>{{i-j9OFSLebGIUp?@XW zeK_5G_?^!6ikf8I&ec|2N2x!`qiag4k~ITq$3XNXmACKa+}p=)TYh{lRX?~U9?Ul& znCv{7Zan(V@t?M=iRV5%#&f3ak4AW1aeH(MrU;w!jiKoA`=Xhy5@~wZ-jwLa2Bf5| zW2JQE_{#32rT4Db_t1*u4{e;q`XQLsv>pBTQJAdZf3ahv>vXyPKbKoiZ#UDem&DjU zA1;+eBuqslPJV-+oQEatzeT|NB6&26d!MET?~`p1C!2R816`Rpy-3%!^Uoqql$u%aY6dM_{n~^1rS*h5kEuF?s*>d3qHa5%+Uk@r*yE64r zkpiYcK9z=14l{J zzOHzsQF}&gDi-L9l%p9Ppa)o%?#bu@fk*VV;xBa`i2CbDJIxK;U$?8f=fAlNvtvBB)E*zJ7dF{a)K*R=JHgglHe+^?6Q=p zCb)*uZ_m^cTt{i9GusHR=k(U)vzZ2hU7Wr)K9Xr9xQWx3#XJw232x!^HSwOzc7j_e z&5=wS!R`3&6Q!9Bf_HHGZSkp0C&68uzA9dl=_c4s>4!5r3Esu&E8~Vt55c{ZrXtfv za6hMSOtfSM2;R-FdAyrOZBp_j7ujyv!URc#zZMm}TZ5!9$dOF>{Du zFcBKrojJnOFEEV0JXz8E@EHA!S~JfhpfonHE;?c@tZ!(yK<_#}R9r$+7so-|+W7f( zu{w@xZ|dR!Yz-_ElAHr%tf@_M71S-CEPExgJCS``Lz1ih&|u_oF)+NxWpuIXV~677 zLczx-+)MX(FYboPyu@?Oed|^3^`^ED*$K(V7j@Vu_1J}tA&;xD@c-CEE76ZlSPFf7 mmWK%>b6DU++w#(yuKH)D(r@(t&hC_{24e~gU}6p9?EeGzl6w*W diff --git a/tools/bench/bench_dependency_manifest.cpp b/tools/bench/bench_dependency_manifest.cpp index 2389331f..c593c6f6 100644 --- a/tools/bench/bench_dependency_manifest.cpp +++ b/tools/bench/bench_dependency_manifest.cpp @@ -231,8 +231,8 @@ constexpr std::array k_records{ "suite_source"}, dependency_record{"parallel_matmul", kind::source, - "src/emel/text/generator/detail.hpp", - "parallel_matmul_lanes"}, + "src/emel/kernel/matmul", + "parallel_matmul_actor"}, dependency_record{"weight_streaming", kind::source, "tools/bench/model/tensor/window_streaming_bench.cpp", diff --git a/tools/bench/dependency_manifest.txt b/tools/bench/dependency_manifest.txt index 9d7df6f6..cb2eecc3 100644 --- a/tools/bench/dependency_manifest.txt +++ b/tools/bench/dependency_manifest.txt @@ -65,7 +65,7 @@ record runner=graph_processor kind=source path=tools/bench/graph/processor_bench record runner=decode_wavefront kind=source path=tools/bench/text/generator/decode_wavefront_bench.cpp reason=suite_source record runner=decode_wavefront kind=source path=src/emel/text/generator/decode_wavefront reason=wavefront_actor record runner=parallel_matmul kind=source path=tools/bench/text/generator/parallel_matmul_bench.cpp reason=suite_source -record runner=parallel_matmul kind=source path=src/emel/text/generator/detail.hpp reason=parallel_matmul_lanes +record runner=parallel_matmul kind=source path=src/emel/kernel/matmul reason=parallel_matmul_actor record runner=weight_streaming kind=source path=tools/bench/model/tensor/window_streaming_bench.cpp reason=suite_source record runner=weight_streaming kind=source path=src/emel/model/tensor/window reason=window_actor record runner=weight_streaming kind=source path=src/emel/io/mmap reason=mmap_actor diff --git a/tools/bench/generation_bench.cpp b/tools/bench/generation_bench.cpp index 5422471e..7ebc2ba3 100644 --- a/tools/bench/generation_bench.cpp +++ b/tools/bench/generation_bench.cpp @@ -835,7 +835,7 @@ struct emel_session { emel::model::generation::contract generation_contract = {}; emel::text::tokenizer::sm tokenizer = {}; emel::text::conditioner::sm conditioner = {}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; std::unique_ptr generator = {}; std::array samplers = {}; emel::tools::generation_formatter_contract::formatter_binding @@ -1962,7 +1962,7 @@ bool prepare_emel_session(const emel_fixture &fixture, emel_session &session) { return false; } const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::make_auto_execution_policy( session.parallel_matmul_lanes); session.generator = std::make_unique( emel::text::generator::dependencies{ diff --git a/tools/bench/kernel/aarch64_bench.cpp b/tools/bench/kernel/aarch64_bench.cpp index 582e0c47..3f28065b 100644 --- a/tools/bench/kernel/aarch64_bench.cpp +++ b/tools/bench/kernel/aarch64_bench.cpp @@ -4,18 +4,298 @@ #include "kernel/bench_common.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint64_t Q4_MULTIROW_COLS = 4096u; +constexpr uint64_t Q4_MULTIROW_ROWS = 256u; +constexpr uint64_t Q4_MULTIROW_BLOCK_COUNT = + Q4_MULTIROW_COLS / emel::kernel::detail::quant::QK_K; +constexpr const char *Q4_MULTIROW_CASE_NAME = + "kernel/aarch64/op_mul_mat_q4_k_q8_k_multirow"; +constexpr const char *Q4_PACKED_NATIVE_CASE_NAME = + "kernel/aarch64/op_mul_mat_q4_k_q8_k_native_moshi"; +constexpr const char *Q4_PACKED_CASE_NAME = + "kernel/aarch64/op_mul_mat_q4_k_x8_bl8_q8_k_packed_moshi"; + +void initialize_q4_benchmark_inputs( + std::vector &weights, + std::vector &rhs_dense, + std::vector &rhs_q8) { + for (uint64_t row = 0; row < Q4_MULTIROW_ROWS; ++row) { + for (uint64_t block = 0; block < Q4_MULTIROW_BLOCK_COUNT; ++block) { + auto &q4 = weights[row * Q4_MULTIROW_BLOCK_COUNT + block]; + q4.d = emel::kernel::detail::quant::fp32_to_fp16( + 0.0009765625f * + static_cast(((row + 3u) * (block + 5u)) % 31u + 1u)); + q4.dmin = emel::kernel::detail::quant::fp32_to_fp16( + 0.00048828125f * + static_cast(((row + 7u) * (block + 11u)) % 29u + 1u)); + for (size_t index = 0; index < q4.scales.size(); ++index) { + q4.scales[index] = static_cast( + (row * 41u + block * 37u + index * 19u + 0x5au) & 0xffu); + } + for (size_t index = 0; index < q4.qs.size(); ++index) { + q4.qs[index] = static_cast( + (row * 23u + block * 17u + index * 29u + 0x96u) & 0xffu); + } + } + } + for (uint64_t index = 0; index < Q4_MULTIROW_COLS; ++index) { + rhs_dense[index] = + static_cast(static_cast((index * 17u) % 127u) - 63) * + 0.0078125f; + } + emel::kernel::detail::quant::quantize_row_q8_k_strided( + rhs_dense.data(), 1u, rhs_q8.data(), + static_cast(Q4_MULTIROW_COLS)); +} + +struct q4_multirow_case { + std::vector weights{ + Q4_MULTIROW_ROWS * Q4_MULTIROW_BLOCK_COUNT}; + std::vector rhs_dense = std::vector(Q4_MULTIROW_COLS); + std::vector rhs_q8{ + Q4_MULTIROW_BLOCK_COUNT}; + std::vector output = std::vector(Q4_MULTIROW_ROWS); + emel::kernel::event::op_mul_mat request = {}; + + q4_multirow_case() { + initialize_q4_benchmark_inputs(weights, rhs_dense, rhs_q8); + + const uint64_t q4_row_bytes = + emel::kernel::detail::quantized_row_storage_bytes( + emel::kernel::detail::dtype_q4_k, Q4_MULTIROW_COLS); + request.src0.data = weights.data(); + request.src0.type = emel::kernel::event::dtype::q4_k; + request.src0.ne = {Q4_MULTIROW_COLS, Q4_MULTIROW_ROWS, 1u, 1u}; + request.src0.nb = {1u, q4_row_bytes, q4_row_bytes * Q4_MULTIROW_ROWS, + q4_row_bytes * Q4_MULTIROW_ROWS}; + + const uint64_t q8_row_bytes = + emel::kernel::detail::quantized_row_storage_bytes( + emel::kernel::detail::dtype_q8_k, Q4_MULTIROW_COLS); + request.src1.data = rhs_q8.data(); + request.src1.type = emel::kernel::event::dtype::q8_k; + request.src1.ne = {1u, Q4_MULTIROW_COLS, 1u, 1u}; + request.src1.nb = {1u, q8_row_bytes, q8_row_bytes, q8_row_bytes}; + + request.dst.data = output.data(); + request.dst.type = emel::kernel::event::dtype::f32; + request.dst.ne = {1u, Q4_MULTIROW_ROWS, 1u, 1u}; + request.dst.nb = {sizeof(float), sizeof(float), + sizeof(float) * Q4_MULTIROW_ROWS, + sizeof(float) * Q4_MULTIROW_ROWS}; + } +}; + +struct q4_packed_case { + const uint64_t packed_group_count = + emel::kernel::detail::quant::packed_q4_k_x8_group_count(Q4_MULTIROW_ROWS); + std::vector native_weights{ + Q4_MULTIROW_ROWS * Q4_MULTIROW_BLOCK_COUNT}; + std::vector packed_weights = std::vector( + packed_group_count * + emel::kernel::detail::quant::packed_q4_k_x8_group_storage_bytes( + Q4_MULTIROW_COLS)); + std::vector rhs_dense = std::vector(Q4_MULTIROW_COLS); + std::vector rhs_q8{ + Q4_MULTIROW_BLOCK_COUNT}; + std::vector native_output = std::vector(Q4_MULTIROW_ROWS); + std::vector packed_output = std::vector(Q4_MULTIROW_ROWS); + emel::kernel::event::op_mul_mat native_request = {}; + emel::kernel::event::op_mul_mat packed_request = {}; + + q4_packed_case() { + initialize_q4_benchmark_inputs(native_weights, rhs_dense, rhs_q8); + if (!emel::kernel::detail::quant::pack_q4_k_rows_x8_bl8( + native_weights.data(), Q4_MULTIROW_ROWS, Q4_MULTIROW_COLS, + packed_weights.data())) { + std::abort(); + } + + const uint64_t native_q4_row_bytes = + emel::kernel::detail::quantized_row_storage_bytes( + emel::kernel::detail::dtype_q4_k, Q4_MULTIROW_COLS); + const uint64_t packed_q4_group_bytes = + emel::kernel::detail::quant::packed_q4_k_x8_group_storage_bytes( + Q4_MULTIROW_COLS); + const uint64_t q8_row_bytes = + emel::kernel::detail::quantized_row_storage_bytes( + emel::kernel::detail::dtype_q8_k, Q4_MULTIROW_COLS); + + native_request.src0.data = native_weights.data(); + native_request.src0.type = emel::kernel::event::dtype::q4_k; + native_request.src0.ne = {Q4_MULTIROW_COLS, Q4_MULTIROW_ROWS, 1u, 1u}; + native_request.src0.nb = {1u, native_q4_row_bytes, + native_q4_row_bytes * Q4_MULTIROW_ROWS, + native_q4_row_bytes * Q4_MULTIROW_ROWS}; + packed_request.src0.data = packed_weights.data(); + packed_request.src0.type = emel::kernel::event::dtype::q4_k_x8_bl8; + packed_request.src0.ne = {Q4_MULTIROW_COLS, Q4_MULTIROW_ROWS, 1u, 1u}; + packed_request.src0.nb = {1u, packed_q4_group_bytes, + packed_q4_group_bytes * packed_group_count, + packed_q4_group_bytes * packed_group_count}; + native_request.src1.data = rhs_q8.data(); + native_request.src1.type = emel::kernel::event::dtype::q8_k; + native_request.src1.ne = {1u, Q4_MULTIROW_COLS, 1u, 1u}; + native_request.src1.nb = {1u, q8_row_bytes, q8_row_bytes, q8_row_bytes}; + packed_request.src1 = native_request.src1; + native_request.dst.data = native_output.data(); + native_request.dst.type = emel::kernel::event::dtype::f32; + native_request.dst.ne = {1u, Q4_MULTIROW_ROWS, 1u, 1u}; + native_request.dst.nb = {sizeof(float), sizeof(float), + sizeof(float) * Q4_MULTIROW_ROWS, + sizeof(float) * Q4_MULTIROW_ROWS}; + packed_request.dst = native_request.dst; + packed_request.dst.data = packed_output.data(); + } +}; + +} // namespace + namespace emel::bench { -void append_emel_kernel_aarch64_cases(std::vector & results, const config & cfg) { +void append_emel_kernel_aarch64_cases(std::vector &results, + const config &cfg) { emel::kernel::aarch64::sm aarch_machine{}; - auto exec = [&](const auto & ev) { - return aarch_machine.process_event(ev); - }; + auto exec = [&](const auto &ev) { return aarch_machine.process_event(ev); }; append_emel_backend_cases(results, cfg, "aarch64", exec); + + const char *q4_multirow_enabled = std::getenv("EMEL_BENCH_Q4_MULTIROW"); + const char *q4_packed_enabled = std::getenv("EMEL_BENCH_Q4_PACKED"); + const bool run_q4_multirow = q4_multirow_enabled != nullptr && + std::string_view{q4_multirow_enabled} == "1"; + const bool run_q4_packed = q4_packed_enabled != nullptr && + std::string_view{q4_packed_enabled} == "1"; + if (!run_q4_multirow && !run_q4_packed) { + return; + } + + if (run_q4_multirow) { + q4_multirow_case q4_case{}; + const bool preflight_ok = aarch_machine.process_event(q4_case.request); + if (!preflight_ok || aarch_machine.optimized_q4_dispatch_count() != 1u || + aarch_machine.optimized_q4_vector_dispatch_count() != 1u || + aarch_machine.shared_q4_dispatch_count() != 0u || + !std::isfinite(q4_case.output[0])) { + std::fprintf(stderr, + "error: q4 multi-row benchmark preflight failed accepted=%d " + "optimized=%llu vector=%llu shared=%llu output=%g\n", + preflight_ok ? 1 : 0, + static_cast( + aarch_machine.optimized_q4_dispatch_count()), + static_cast( + aarch_machine.optimized_q4_vector_dispatch_count()), + static_cast( + aarch_machine.shared_q4_dispatch_count()), + static_cast(q4_case.output[0])); + std::abort(); + } + volatile float sink = 0.0f; + auto q4_fn = [&]() { + if (!aarch_machine.process_event(q4_case.request)) { + std::abort(); + } + sink += q4_case.output[0]; + }; + results.push_back(measure_case(Q4_MULTIROW_CASE_NAME, cfg, q4_fn)); + } + + if (run_q4_packed) { + q4_packed_case packed_case{}; + emel::kernel::aarch64::sm native_machine{}; + emel::kernel::aarch64::sm packed_machine{}; + const bool native_preflight_ok = + native_machine.process_event(packed_case.native_request); + const bool packed_preflight_ok = + packed_machine.process_event(packed_case.packed_request); + bool outputs_match = native_preflight_ok && packed_preflight_ok; + for (uint64_t row = 0; outputs_match && row < Q4_MULTIROW_ROWS; ++row) { + outputs_match = std::bit_cast(packed_case.native_output[row]) == + std::bit_cast(packed_case.packed_output[row]); + } + const bool native_route_ok = + native_machine.optimized_q4_dispatch_count() == 1u && + native_machine.optimized_q4_vector_dispatch_count() == 1u && + native_machine.optimized_q4_vector_packed_dispatch_count() == 0u && + native_machine.optimized_q4_vector_packed_q8_rhs_dispatch_count() == + 0u && + native_machine.shared_q4_dispatch_count() == 0u; + const bool packed_route_ok = + packed_machine.optimized_q4_dispatch_count() == 1u && + packed_machine.optimized_q4_vector_dispatch_count() == 1u && + packed_machine.optimized_q4_vector_packed_dispatch_count() == 1u && + packed_machine.optimized_q4_vector_packed_q8_rhs_dispatch_count() == + 1u && + packed_machine.shared_q4_dispatch_count() == 0u; + if (!outputs_match || !native_route_ok || !packed_route_ok) { + std::fprintf( + stderr, + "error: q4 packed benchmark preflight failed native_accepted=%d " + "packed_accepted=%d outputs_match=%d " + "native=(%llu,%llu,%llu,%llu,%llu) " + "packed=(%llu,%llu,%llu,%llu,%llu)\n", + native_preflight_ok ? 1 : 0, packed_preflight_ok ? 1 : 0, + outputs_match ? 1 : 0, + static_cast( + native_machine.optimized_q4_dispatch_count()), + static_cast( + native_machine.optimized_q4_vector_dispatch_count()), + static_cast( + native_machine.optimized_q4_vector_packed_dispatch_count()), + static_cast( + native_machine + .optimized_q4_vector_packed_q8_rhs_dispatch_count()), + static_cast( + native_machine.shared_q4_dispatch_count()), + static_cast( + packed_machine.optimized_q4_dispatch_count()), + static_cast( + packed_machine.optimized_q4_vector_dispatch_count()), + static_cast( + packed_machine.optimized_q4_vector_packed_dispatch_count()), + static_cast( + packed_machine + .optimized_q4_vector_packed_q8_rhs_dispatch_count()), + static_cast( + packed_machine.shared_q4_dispatch_count())); + std::abort(); + } + std::printf( + "# q4_packed_preflight: exact_output_match=1 native_route=(1,1,0,0,0) " + "packed_route=(1,1,1,1,0) packed_before_timing=1 rows=%llu cols=%llu\n", + static_cast(Q4_MULTIROW_ROWS), + static_cast(Q4_MULTIROW_COLS)); + + volatile float native_sink = 0.0f; + auto native_fn = [&]() { + if (!native_machine.process_event(packed_case.native_request)) { + std::abort(); + } + native_sink += packed_case.native_output[0]; + }; + volatile float packed_sink = 0.0f; + auto packed_fn = [&]() { + if (!packed_machine.process_event(packed_case.packed_request)) { + std::abort(); + } + packed_sink += packed_case.packed_output[0]; + }; + results.push_back(measure_case(Q4_PACKED_NATIVE_CASE_NAME, cfg, native_fn)); + results.push_back(measure_case(Q4_PACKED_CASE_NAME, cfg, packed_fn)); + } } -void append_reference_kernel_aarch64_cases(std::vector & results, const config & cfg) { +void append_reference_kernel_aarch64_cases(std::vector &results, + const config &cfg) { append_reference_backend_cases(results, cfg, "aarch64"); } -} // namespace emel::bench +} // namespace emel::bench diff --git a/tools/bench/kernel/f32_matvec_bench.cpp b/tools/bench/kernel/f32_matvec_bench.cpp new file mode 100644 index 00000000..c04780a6 --- /dev/null +++ b/tools/bench/kernel/f32_matvec_bench.cpp @@ -0,0 +1,155 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "emel/kernel/f32_matvec/sm.hpp" + +namespace { + +namespace f32_matvec = emel::kernel::f32_matvec; + +struct geometry { + uint64_t inner = 0u; + uint64_t rows = 0u; +}; + +uint32_t next_bits(uint32_t &state) noexcept { + state = state * 1664525u + 1013904223u; + return state; +} + +float next_value(uint32_t &state) noexcept { + const int32_t centered = static_cast(next_bits(state) >> 8u) - + static_cast(1u << 23u); + return static_cast(centered) / static_cast(1u << 23u); +} + +double median(std::vector samples) { + std::sort(samples.begin(), samples.end()); + return samples[samples.size() / 2u]; +} + +bool run_geometry(const geometry shape, const int samples) { + const size_t elements = static_cast(shape.inner * shape.rows); + std::vector weights(elements); + std::vector packed(elements); + std::vector input(static_cast(shape.inner)); + std::vector reference(static_cast(shape.rows)); + std::vector exact(static_cast(shape.rows)); + uint32_t seed = 0x5a17c9e3u; + for (float &value : weights) { + value = next_value(seed); + } + for (float &value : input) { + value = next_value(seed); + } + + f32_matvec::sm actor{}; + f32_matvec::event::dispatch_result result{}; + const f32_matvec::event::prepare_f32_request prepare_request{ + .source = std::span{weights}, + .destination = std::span{packed}, + .inner = shape.inner, + .rows = shape.rows, + }; + if (!actor.process_event( + f32_matvec::event::prepare_f32{prepare_request, result})) { + return false; + } + const f32_matvec::event::execute_request reference_request{ + .weights = std::span{weights}, + .input = std::span{input}, + .output = std::span{reference}, + .inner = shape.inner, + .rows = shape.rows, + }; + const f32_matvec::event::execute_request exact_request{ + .weights = std::span{packed}, + .input = std::span{input}, + .output = std::span{exact}, + .inner = shape.inner, + .rows = shape.rows, + }; + +#if defined(__aarch64__) || defined(_M_ARM64) + (void)actor.process_event( + f32_matvec::event::execute_reference{reference_request, result}); + (void)actor.process_event( + f32_matvec::event::execute_exact_x4{exact_request, result}); + for (size_t row = 0u; row < exact.size(); ++row) { + if (std::bit_cast(reference[row]) != + std::bit_cast(exact[row])) { + std::fprintf(stderr, + "mismatch inner=%llu rows=%llu row=%zu ref=%08x x4=%08x\n", + static_cast(shape.inner), + static_cast(shape.rows), row, + std::bit_cast(reference[row]), + std::bit_cast(exact[row])); + return false; + } + } + + std::vector reference_ms{}; + std::vector exact_ms{}; + reference_ms.reserve(static_cast(samples)); + exact_ms.reserve(static_cast(samples)); + for (int sample = 0; sample < samples; ++sample) { + auto begin = std::chrono::steady_clock::now(); + (void)actor.process_event( + f32_matvec::event::execute_reference{reference_request, result}); + auto end = std::chrono::steady_clock::now(); + reference_ms.push_back( + std::chrono::duration(end - begin).count()); + + begin = std::chrono::steady_clock::now(); + (void)actor.process_event( + f32_matvec::event::execute_exact_x4{exact_request, result}); + end = std::chrono::steady_clock::now(); + exact_ms.push_back( + std::chrono::duration(end - begin).count()); + } + const double reference_median = median(reference_ms); + const double exact_median = median(exact_ms); + std::printf( + "inner=%llu rows=%llu reference_ms=%.6f exact_x4_ms=%.6f speedup=%.6f " + "mismatch=0\n", + static_cast(shape.inner), + static_cast(shape.rows), reference_median, + exact_median, reference_median / exact_median); + return true; +#else + (void)samples; + std::fprintf(stderr, "exact x4 is compile-time unavailable off AArch64\n"); + return false; +#endif +} + +} // namespace + +int main(int argc, char **argv) { + const int samples = argc > 1 ? std::max(5, std::atoi(argv[1])) : 7; + if (std::fesetround(FE_TONEAREST) != 0 || std::fegetround() != FE_TONEAREST) { + std::fprintf(stderr, "failed to establish FE_TONEAREST\n"); + return 1; + } + std::printf("samples=%d warmup_samples=1 rounding=FE_TONEAREST\n", samples); + constexpr geometry shapes[] = { + {512u, 1536u}, + {512u, 512u}, + {512u, 2048u}, + {2048u, 512u}, + }; + for (const geometry shape : shapes) { + if (!run_geometry(shape, samples)) { + return 1; + } + } + return 0; +} diff --git a/tools/bench/model/tensor/window_streaming_bench.cpp b/tools/bench/model/tensor/window_streaming_bench.cpp index 1b91b437..0654e821 100644 --- a/tools/bench/model/tensor/window_streaming_bench.cpp +++ b/tools/bench/model/tensor/window_streaming_bench.cpp @@ -675,7 +675,7 @@ struct emel_session { formatter_binding = {}; emel::text::tokenizer::sm tokenizer = {}; emel::text::conditioner::sm conditioner = {}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; std::unique_ptr generator = {}; initialize_capture initialize = {}; generation_capture generation = {}; @@ -773,7 +773,7 @@ bool prepare_session(const emel_fixture &fixture, emel_session &session, return false; } const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::make_auto_execution_policy( session.parallel_matmul_lanes); if (rig != nullptr) { session.generator = std::make_unique( diff --git a/tools/bench/personaplex_compare.py b/tools/bench/personaplex_compare.py index 842c917e..708ae09d 100755 --- a/tools/bench/personaplex_compare.py +++ b/tools/bench/personaplex_compare.py @@ -14,6 +14,57 @@ from pathlib import Path +def repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def implementation_identity(root: Path) -> dict[str, str]: + root = root.resolve() + result = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + capture_output=True, text=True, check=False) + if result.returncode != 0: + raise RuntimeError("cannot resolve implementation git head") + git_head = result.stdout.strip() + if not re.fullmatch(r"[0-9a-f]{40}", git_head): + raise RuntimeError("implementation git head is malformed") + + runtime_roots = ( + root / "CMakeLists.txt", + root / "cmake", + root / "scripts" / "bench_personaplex_compare.sh", + root / "src", + root / "tools" / "bench" / "CMakeLists.txt", + root / "tools" / "bench" / "personaplex_compare.py", + root / "tools" / "bench" / "speech" / "personaplex_emel_runner.cpp", + ) + files = [] + for runtime_root in runtime_roots: + if runtime_root.is_file(): + files.append(runtime_root) + elif runtime_root.is_dir(): + files.extend(path for path in runtime_root.rglob("*") if path.is_file()) + else: + raise RuntimeError(f"implementation identity path is missing: {runtime_root}") + digest = hashlib.sha256() + for path in sorted(set(files)): + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(8, "little")) + digest.update(relative) + digest.update(bytes.fromhex(sha256(path))) + return { + "git_head": git_head, + "runtime_tree_sha256": digest.hexdigest(), + } + + +def runner_identity(path: Path) -> dict[str, str]: + resolved = path.resolve(strict=True) + if not resolved.is_file(): + raise RuntimeError(f"EMEL runner is not a file: {resolved}") + return {"path": str(resolved), "sha256": sha256(resolved)} + + def run_lane(command: list[str], stdout_path: Path, stderr_path: Path) -> float: started = time.monotonic() with stdout_path.open("w") as stdout, stderr_path.open("w") as stderr: @@ -47,6 +98,49 @@ def parse_emel_log(path: Path) -> tuple[list[list[int]], list[list[int]], list[i return input_frames, output_frames, text_tokens +def parse_emel_threads(path: Path) -> tuple[int, int, int, int, int, str, str]: + pattern = re.compile( + r"^EMEL_THREADS requested_total=(\d+) owner_threads=(\d+) " + r"stage_workers=(\d+) matmul_workers=(\d+) matmul_lanes=(\d+) " + r"stage_mode=(serial|parallel) matmul_mode=(serial|parallel)$") + matches = [pattern.match(line) for line in path.read_text().splitlines()] + parsed = [match for match in matches if match is not None] + if len(parsed) != 1: + raise ValueError("expected one EMEL_THREADS contract line") + return (int(parsed[0].group(1)), int(parsed[0].group(2)), + int(parsed[0].group(3)), int(parsed[0].group(4)), + int(parsed[0].group(5)), parsed[0].group(6), parsed[0].group(7)) + + +def thread_contract_reasons( + contract: tuple[int, int, int, int, int, str, str], + requested: int) -> list[str]: + (requested_total, owner_threads, stage_workers, matmul_workers, + matmul_lanes, stage_mode, matmul_mode) = contract + reasons = [] + if requested_total != requested: + reasons.append("EMEL requested total CPU budget does not match harness") + if owner_threads != 1: + reasons.append("EMEL owner-thread count must be exactly one") + accounted_concurrency = owner_threads + stage_workers + matmul_workers + if accounted_concurrency > requested_total: + reasons.append("EMEL runnable concurrency exceeds total CPU budget") + if stage_workers not in (0, 2): + reasons.append("EMEL stage worker count is unsupported") + if ((stage_workers == 0 and stage_mode != "serial") or + (stage_workers != 0 and stage_mode != "parallel")): + reasons.append("EMEL stage worker mode does not match its worker count") + if matmul_lanes == 1: + if matmul_workers != 0 or matmul_mode != "serial": + reasons.append("EMEL serial matmul lane unexpectedly constructed a pool") + elif matmul_lanes > 1: + if matmul_workers != matmul_lanes - 1 or matmul_mode != "parallel": + reasons.append("EMEL parallel matmul worker-pool contract mismatch") + else: + reasons.append("EMEL matmul lane count must be positive") + return reasons + + def parse_reference_log(path: Path) -> tuple[list[list[int]], list[list[int]], list[int]]: input_frames = [] output_frames = [] @@ -99,6 +193,28 @@ def append_text_parity_reason(reasons: list[str], emel_text: list[int], f"text token match is {text_match:.6f}, expected 1.0") +def append_input_parity_observation(observations: list[str], + input_match: float) -> None: + if input_match != 1.0: + observations.append( + f"same-WAV input token match is {input_match:.6f}, tracked " + "separately from generated-output parity") + + +def semantic_parity_reasons(input_match: float, output_match: float, + text_match: float) -> list[str]: + reasons = [] + if input_match != 1.0: + reasons.append( + f"same-WAV input token match is {input_match:.6f}, expected 1.0") + if output_match != 1.0: + reasons.append( + f"generated output token match is {output_match:.6f}, expected 1.0") + if text_match != 1.0: + reasons.append(f"text token match is {text_match:.6f}, expected 1.0") + return reasons + + def read_wav(path: Path) -> tuple[int, list[float]]: with wave.open(str(path), "rb") as source: if (source.getnchannels(), source.getsampwidth()) != (1, 2): @@ -221,6 +337,8 @@ def main() -> int: emel_stderr = output_dir / "emel.stderr.log" reference_stdout = output_dir / "moshi_cpp.stdout.log" reference_stderr = output_dir / "moshi_cpp.stderr.log" + implementation = implementation_identity(repo_root()) + runner = runner_identity(Path(args.emel_runner)) emel_elapsed = run_lane([ args.emel_runner, @@ -238,6 +356,7 @@ def main() -> int: str(max_blocks), str(block_tokens), str(prompt_text_token), + str(args.threads), ], emel_stdout, emel_stderr) reference_elapsed = run_lane([ args.reference_driver, @@ -257,6 +376,10 @@ def main() -> int: ], reference_stdout, reference_stderr) emel_input, emel_output, emel_text = parse_emel_log(emel_stderr) + emel_thread_contract = parse_emel_threads(emel_stderr) + (emel_requested_total, emel_owner_threads, emel_stage_workers, + emel_matmul_workers, emel_matmul_lanes, emel_stage_mode, + emel_matmul_mode) = emel_thread_contract ref_input, ref_output, ref_text = parse_reference_log(reference_stdout) input_rate, input_samples = read_wav(Path(args.audio)) if input_rate != 24000: @@ -291,13 +414,16 @@ def main() -> int: float(emel_audio["active_ratio"]) - float(reference_audio["active_ratio"])) reasons = [] + observations = [] + reasons.extend(thread_contract_reasons(emel_thread_contract, args.threads)) if len(emel_input) != args.frames or len(ref_input) != args.frames: reasons.append("input frame count mismatch") if len(emel_output) != args.frames or len(ref_output) != args.frames: reasons.append("output frame count mismatch") append_text_parity_reason(reasons, emel_text, ref_text, text_match) - if input_match != 1.0: - reasons.append(f"same-WAV input token match is {input_match:.6f}, expected 1.0") + append_input_parity_observation(observations, input_match) + exact_semantic_reasons = semantic_parity_reasons( + input_match, output_match, text_match) if output_match < args.min_output_match: reasons.append( f"output token match {output_match:.6f} below {args.min_output_match:.6f}") @@ -318,9 +444,26 @@ def main() -> int: report = { "surface": "personaplex_compare/v1", - "status": "pass" if not reasons else "fail", + "acceptance_scope": "bounded_quality_performance", + "status": "pass_bounded_quality_performance" if not reasons else "fail", "reasons": reasons, - "cpu_threads": args.threads, + "observations": observations, + "semantic_parity": { + "status": "pass" if not exact_semantic_reasons else "fail", + "reasons": exact_semantic_reasons, + }, + "implementation": implementation, + "emel_runner": runner, + "threads": { + "emel_requested_total": emel_requested_total, + "emel_owner_threads": emel_owner_threads, + "emel_stage_workers": emel_stage_workers, + "emel_matmul_workers": emel_matmul_workers, + "emel_matmul_lanes": emel_matmul_lanes, + "emel_stage_mode": emel_stage_mode, + "emel_matmul_mode": emel_matmul_mode, + "reference_requested": args.threads, + }, "seed": args.seed, "frames": args.frames, "inference": { diff --git a/tools/bench/personaplex_compare_tests.cpp b/tools/bench/personaplex_compare_tests.cpp index 1ae782ce..edfbd85a 100644 --- a/tools/bench/personaplex_compare_tests.cpp +++ b/tools/bench/personaplex_compare_tests.cpp @@ -82,8 +82,88 @@ TEST_CASE("personaplex compare math reports exact and partial token behavior") { "reasons=[]; m.append_text_parity_reason(reasons, [1], [1,2], 0.0); " "assert reasons == ['text frame count mismatch']; " "reasons=[]; m.append_text_parity_reason(reasons, [1,2], [1,2], 1.0); " - "assert reasons == []"; - CHECK(command_exit_code("python3 -c " + quote_arg_posix(program)) == 0); + "assert reasons == []; observations=[]; " + "m.append_input_parity_observation(observations, 0.980769); " + "assert observations == ['same-WAV input token match is 0.980769, " + "tracked separately from generated-output parity']; " + "assert m.semantic_parity_reasons(1.0, 1.0, 1.0) == []; " + "assert m.semantic_parity_reasons(0.980769, 1.0, 1.0) == " + "['same-WAV input token match is 0.980769, expected 1.0']; " + "assert len(m.semantic_parity_reasons(0.5, 0.75, 0.25)) == 3"; + CHECK(command_exit_code("python3 -B -c " + quote_arg_posix(program)) == 0); +} + +TEST_CASE( + "personaplex compare parses the complete CPU thread budget truthfully") { + const std::string module = quote_arg_posix(compare_script_path().string()); + const std::string program = + "import importlib.util, pathlib, tempfile; " + "spec=importlib.util.spec_from_file_location('personaplex_compare', " + + module + + "); m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m); " + "p=pathlib.Path(tempfile.mkstemp()[1]); " + "p.write_text('EMEL_THREADS requested_total=1 owner_threads=1 " + "stage_workers=0 matmul_workers=0 matmul_lanes=1 stage_mode=serial " + "matmul_mode=serial\\n'); " + "assert m.parse_emel_threads(p) == (1,1,0,0,1,'serial','serial'); " + "p.write_text('EMEL_THREADS requested_total=8 owner_threads=1 " + "stage_workers=2 matmul_workers=3 matmul_lanes=4 stage_mode=parallel " + "matmul_mode=parallel\\n'); " + "contract=m.parse_emel_threads(p); " + "assert contract == (8,1,2,3,4,'parallel','parallel'); " + "assert m.thread_contract_reasons(contract, 8) == []; " + "assert m.thread_contract_reasons((8,1,2,7,8,'parallel','parallel'), 8) " + "== ['EMEL runnable concurrency exceeds total CPU budget']; " + "p.unlink()"; + CHECK(command_exit_code("python3 -B -c " + quote_arg_posix(program)) == 0); +} + +TEST_CASE("personaplex compare binds reports to source and runner identity") { + const std::string module = quote_arg_posix(compare_script_path().string()); + const std::string program = + "import importlib.util, pathlib, tempfile; " + "spec=importlib.util.spec_from_file_location('personaplex_compare', " + + module + + "); m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m); " + "root=pathlib.Path(tempfile.mkdtemp()); " + "runner=root/'runner'; runner.write_bytes(b'exact runner'); " + "identity=m.runner_identity(runner); " + "assert identity == {'path': str(runner.resolve()), " + "'sha256': m.sha256(runner)}; " + "assert len(m.implementation_identity(m.repo_root())" + "['runtime_tree_sha256']) == 64"; + CHECK(command_exit_code("python3 -B -c " + quote_arg_posix(program)) == 0); + + const std::string missing_program = + "import importlib.util, pathlib, tempfile; " + "spec=importlib.util.spec_from_file_location('personaplex_compare', " + + module + + "); m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m); " + "m.runner_identity(pathlib.Path(tempfile.mkdtemp())/'missing')"; + CHECK(command_exit_code("python3 -B -c " + + quote_arg_posix(missing_program)) != 0); +} + +TEST_CASE("personaplex compare fails closed when a child lane is killed") { + const std::string module = quote_arg_posix(compare_script_path().string()); + const std::string program = + "import importlib.util, pathlib, sys, tempfile; " + "spec=importlib.util.spec_from_file_location('personaplex_compare', " + + module + + "); m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m); " + "root=pathlib.Path(tempfile.mkdtemp()); marker=root/'attempts'; " + "payload=\"import os,pathlib,signal,sys\\n" + "p=pathlib.Path(sys.argv[1])\\n" + "count=int(p.read_text())+1 if p.exists() else 1\\n" + "p.write_text(str(count))\\n" + "os.kill(os.getpid(), signal.SIGKILL)\"; " + "failed=False\n" + "try: m.run_lane([sys.executable,'-c',payload,str(marker)], " + "root/'stdout',root/'stderr')\n" + "except RuntimeError as error: failed='status -9' in str(error)\n" + "assert failed; assert marker.read_text() == '1'; " + "assert (root/'stdout').read_text() == ''"; + CHECK(command_exit_code("python3 -B -c " + quote_arg_posix(program)) == 0); } TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { @@ -116,9 +196,23 @@ TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { std::string::npos); CHECK(compare_source.find("reference_over_emel") != std::string::npos); CHECK(wrapper_source.find("--threads \"$THREADS\"") != std::string::npos); + CHECK( + wrapper_source.find( + "for candidate in python3.12 python3.13 /usr/bin/python3 python3") != + std::string::npos); + CHECK(wrapper_source.find("import hashlib, json, subprocess") != + std::string::npos); + CHECK( + wrapper_source.find( + "\"$PYTHON_BIN\" \"$ROOT_DIR/tools/bench/personaplex_compare.py\"") != + std::string::npos); + CHECK(compare_source.find("for attempt in range(3)") == std::string::npos); CHECK(wrapper_source.find("--audio \"$AUDIO\"") != std::string::npos); CHECK(wrapper_source.find("--inference-config \"$INFERENCE_CONFIG\"") != std::string::npos); + CHECK(wrapper_source.find( + "bash \"$ROOT_DIR/scripts/setup_moshi_cpp_reference.sh\"") != + std::string::npos); CHECK(wrapper_source.find("--emel-lm PATH") != std::string::npos); CHECK(wrapper_source.find("--emel-lm) EMEL_LM=") != std::string::npos); CHECK(wrapper_source.find("--emel-mimi PATH") != std::string::npos); @@ -128,15 +222,36 @@ TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { std::string::npos); CHECK(wrapper_source.find("Hey, I'm Gabe. How are you doing?") != std::string::npos); - CHECK(cmake_source.find("set(GGML_METAL OFF CACHE BOOL \"CPU-only " - "PersonaPlex reference lane\" FORCE)") != - std::string::npos); + CHECK(cmake_source.find("set(GGML_METAL OFF CACHE BOOL \"CPU-only benchmark " + "reference lanes\" FORCE)") != std::string::npos); CHECK(emel_source.find("#include ") != + std::string::npos); + CHECK(emel_source.find("stage_worker_count") != std::string::npos); + CHECK(emel_source.find("matmul_lane_count") != std::string::npos); + CHECK(emel_source.find( + "prediction_matmul_lanes.emplace(matmul_lane_count - 1u)") != + std::string::npos); + CHECK(emel_source.find("active_worker_count()") != std::string::npos); + CHECK(compare_source.find("emel_stage_workers") != std::string::npos); + CHECK(compare_source.find("emel_matmul_lanes") != std::string::npos); + CHECK(compare_source.find("emel_matmul_workers") != std::string::npos); + CHECK(emel_source.find("accepted ?") == std::string::npos); + CHECK(compare_source.find("reference_requested") != std::string::npos); + CHECK(compare_source.find( + "\"acceptance_scope\": \"bounded_quality_performance\"") != + std::string::npos); + CHECK(compare_source.find("\"semantic_parity\": {") != std::string::npos); + CHECK(compare_source.find("pass_bounded_quality_performance") != + std::string::npos); + CHECK(compare_source.find("\"implementation\": implementation") != + std::string::npos); + CHECK(compare_source.find("\"emel_runner\": runner") != std::string::npos); CHECK(emel_source.find(".max_blocks = 256") == std::string::npos); CHECK(compare_source.find("\"--n-q\", \"8\"") == std::string::npos); CHECK(compare_source.find("--audio-top-k") == std::string::npos); @@ -147,6 +262,17 @@ TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { CHECK(compare_source.find("\"reference_mimi\": {") != std::string::npos); CHECK(setup_source.find("--inference-config \"$MOSHI_INFERENCE_CONFIG\"") != std::string::npos); + CHECK(setup_source.find("--quantize q8_0") != std::string::npos); + CHECK(setup_source.find("manifest.get(\"quantization\") != \"q8_0\"") != + std::string::npos); + CHECK(setup_source.find( + "manifest.get(\"quantized_tensors\") != expected_tensors") != + std::string::npos); + CHECK(setup_source.find( + "manifest.get(\"enriched_sha256\") != digest.hexdigest()") != + std::string::npos); + CHECK(setup_source.find("MIMI_PERSONAPLEX_Q8_TENSORS=68") != + std::string::npos); CHECK(converter_source.find("--inference-config") != std::string::npos); CHECK(inference_source.find("\"prompt_tokens\"") != std::string::npos); CHECK(inference_source.find("\"audio_top_k\"") != std::string::npos); @@ -166,6 +292,47 @@ TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { CHECK(reference_source.find("produced=%d") != std::string::npos); } +TEST_CASE("personaplex Q8 freshness reconverts a swapped F32 artifact") { + const std::string setup = quote_arg_posix( + (repo_root() / "scripts" / "setup_moshi_cpp_reference.sh").string()); + const std::string program = + "import hashlib,json,pathlib,subprocess,tempfile; " + "root=pathlib.Path(tempfile.mkdtemp()); " + "output=root/'mimi.gguf'; manifest=root/'mimi.manifest.json'; " + "source=root/'raw.gguf'; converter=root/'converter.py'; " + "marker=root/'converter-runs'; source.write_bytes(b'raw'); " + "candidate=b'candidate-q8-gguf'; output.write_bytes(b'legacy-f32-gguf'); " + "manifest.write_text(json.dumps({'quantization':'q8_0'," + "'quantized_tensors':68,'enriched_sha256':" + "hashlib.sha256(candidate).hexdigest()})); " + "converter.write_text(\"import hashlib,json,pathlib,sys\\n" + "a=sys.argv\\n" + "o=pathlib.Path(a[a.index('--output')+1])\\n" + "m=pathlib.Path(a[a.index('--manifest')+1])\\n" + "r=pathlib.Path(a[a.index('--marker')+1])\\n" + "b=b'candidate-q8-gguf'\\n" + "o.write_bytes(b)\\n" + "m.write_text(json.dumps({'quantization':'q8_0'," + "'quantized_tensors':68,'enriched_sha256':" + "hashlib.sha256(b).hexdigest()}))\\n" + "r.write_text((r.read_text() if r.exists() else '')+'x')\\n\"); " + "cmd=['bash'," + + setup + + ",'--ensure-q8-artifact',str(output),str(manifest),'68'," + "str(converter),str(source),'--marker',str(marker)]; " + "assert subprocess.run(cmd).returncode == 0; " + "assert output.read_bytes() == candidate; assert marker.read_text() == " + "'x'; " + "assert subprocess.run(cmd).returncode == 0; " + "assert marker.read_text() == 'x'; " + "data=json.loads(manifest.read_text()); data['quantized_tensors']=67; " + "manifest.write_text(json.dumps(data)); " + "assert subprocess.run(cmd).returncode == 0; " + "assert marker.read_text() == 'xx'; " + "assert json.loads(manifest.read_text())['quantized_tensors'] == 68"; + CHECK(command_exit_code("python3 -B -c " + quote_arg_posix(program)) == 0); +} + TEST_CASE("personaplex conversion injects its inference contract") { const auto bench_dir = repo_root() / "tools" / "bench"; const auto inference_path = bench_dir / "personaplex-inference.json"; diff --git a/tools/bench/speech/codec_mimi_bench.cpp b/tools/bench/speech/codec_mimi_bench.cpp index 8ab477d7..d1704bc1 100644 --- a/tools/bench/speech/codec_mimi_bench.cpp +++ b/tools/bench/speech/codec_mimi_bench.cpp @@ -142,17 +142,17 @@ bool load_fixture(codec_fixture &fixture) { fixture.model->name_bytes_used += tensor.name_length; } - fixture.prepared.resize( - mimi::prepared_arena_floats(*fixture.model)); + fixture.prepared.resize(mimi::prepared_arena_floats(*fixture.model)); fixture.state.resize(mimi::state_arena_floats(*fixture.model)); - fixture.workspace.resize( - mimi::workspace_arena_floats(*fixture.model)); + fixture.workspace.resize(mimi::workspace_arena_floats(*fixture.model)); fixture.frame.resize(mimi::frame_arena_floats(*fixture.model)); fixture.codec = std::make_unique(); - mimi::event::initialize init{ - *fixture.model, std::span{fixture.prepared}, - std::span{fixture.state}, std::span{fixture.workspace}, - std::span{fixture.frame}}; + mimi::event::initialize init{*fixture.model, + mimi::make_bind_contract(*fixture.model), + std::span{fixture.prepared}, + std::span{fixture.state}, + std::span{fixture.workspace}, + std::span{fixture.frame}}; init.on_done = emel::callback::from<&on_codec_initialized>(); if (!fixture.codec->process_event(init)) { @@ -227,10 +227,9 @@ void append_reference_speech_codec_mimi_cases(std::vector &results, // compare table stays one-to-one with the EMEL lane (the sortformer // recorded-baseline pattern); their timings are not a measurement of // moshi.cpp and the snapshot gate reads only the EMEL column. - for (const char *name : - {"speech_codec_mimi/encode_frame_tiny", - "speech_codec_mimi/decode_frame_tiny", - "speech_codec_mimi/roundtrip_frame_tiny"}) { + for (const char *name : {"speech_codec_mimi/encode_frame_tiny", + "speech_codec_mimi/decode_frame_tiny", + "speech_codec_mimi/roundtrip_frame_tiny"}) { auto record = measure_case(name, cfg, []() {}); record.ns_per_op = 0.0; results.push_back(std::move(record)); diff --git a/tools/bench/speech/lm_moshi_bench.cpp b/tools/bench/speech/lm_moshi_bench.cpp index c3f01167..fa494cea 100644 --- a/tools/bench/speech/lm_moshi_bench.cpp +++ b/tools/bench/speech/lm_moshi_bench.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -9,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -20,12 +22,17 @@ #include "emel/gguf/loader/sm.hpp" #include "emel/io/mmap/events.hpp" #include "emel/io/mmap/sm.hpp" +#include "emel/kernel/matmul/sm.hpp" +#include "emel/kernel/sm.hpp" +#include "emel/logits/sampler/sm.hpp" +#include "emel/memory/streaming/sm.hpp" #include "emel/model/data.hpp" #include "emel/model/detail.hpp" #include "emel/model/loader/errors.hpp" #include "emel/model/loader/events.hpp" #include "emel/model/loader/sm.hpp" #include "emel/model/tensor/sm.hpp" +#include "emel/speech/predictor/moshi/executor/any.hpp" namespace { @@ -78,6 +85,274 @@ struct lm_fixture { load_capture load = {}; }; +uint64_t append_checksum(uint64_t checksum, const void *data, + const std::size_t bytes) noexcept { + const auto *input = static_cast(data); + for (std::size_t index = 0u; index < bytes; ++index) { + checksum ^= input[index]; + checksum *= 1099511628211ull; + } + return checksum; +} + +template +uint64_t append_checksum(uint64_t checksum, + const std::span values) noexcept { + return append_checksum(checksum, values.data(), values.size_bytes()); +} + +struct attention_route_snapshot { + std::vector temporal_state = {}; + std::vector key_cache = {}; + std::vector value_cache = {}; + std::vector audio_tokens = {}; + int32_t text_token = -1; + + bool operator==(const attention_route_snapshot &) const = default; + + uint64_t checksum() const noexcept { + uint64_t value = 1469598103934665603ull; + value = append_checksum(value, std::span{temporal_state}); + value = append_checksum(value, std::span{key_cache}); + value = append_checksum(value, std::span{value_cache}); + value = append_checksum(value, std::span{audio_tokens}); + return append_checksum(value, &text_token, sizeof(text_token)); + } +}; + +template +result measure_prepared_case(const char *name, const config &cfg, + prepare_type &&prepare, run_type &&run, + finish_type &&finish) { + const auto runs = std::max(cfg.runs, 1u); + const auto iterations = std::max(cfg.iterations, 1u); + std::vector samples; + samples.reserve(runs); + + for (std::size_t warmup_run = 0u; warmup_run < cfg.warmup_runs; + ++warmup_run) { + for (uint64_t index = 0u; index < cfg.warmup_iterations; ++index) { + auto prepared = prepare(); + run(*prepared); + } + } + + for (std::size_t measured_run = 0u; measured_run < runs; ++measured_run) { + int64_t elapsed_ns = 0; + for (uint64_t index = 0u; index < iterations; ++index) { + auto prepared = prepare(); + const auto start = std::chrono::steady_clock::now(); + run(*prepared); + const auto end = std::chrono::steady_clock::now(); + elapsed_ns += + std::chrono::duration_cast(end - start) + .count(); + finish(*prepared); + } + samples.push_back(static_cast(elapsed_ns) / + static_cast(iterations)); + } + + std::sort(samples.begin(), samples.end()); + double sum = 0.0; + for (const double sample : samples) { + sum += sample; + } + result measured; + measured.name = name; + measured.ns_per_op = emel::bench::select_reported_ns_per_op(samples); + measured.ns_min_per_op = samples.front(); + measured.ns_mean_per_op = sum / static_cast(samples.size()); + measured.ns_max_per_op = samples.back(); + measured.iterations = iterations; + measured.runs = runs; + return measured; +} + +struct attention_executor_bench_fixture { + static constexpr int32_t k_valid_positions = 250; + + attention_executor_bench_fixture(const emel::model::data &model_ref, + const std::size_t lane_count_ref) + : model(model_ref), lane_count(lane_count_ref), + temporal_positions(emel::memory::streaming::dependencies{ + .capacity = model.moshi_lm.context}), + depformer_positions(emel::memory::streaming::dependencies{ + .capacity = model.moshi_lm.depformer_context}), + layer_offsets(static_cast(model.moshi_lm.num_layers), 0u), + key_cache(static_cast(model.moshi_lm.context) * + static_cast(model.moshi_lm.dim)), + value_cache(key_cache.size()), + input_sequence(static_cast(model.moshi_lm.n_q + 1), -1), + input_embedding(static_cast(model.moshi_lm.dim)), + temporal_state(input_embedding.size()), + audio_tokens(static_cast(model.moshi_lm.dep_q), -1), + matmul_policy{ + .parallel_matmul_lanes = nullptr, + .kernel_kind = emel::kernel::detect_host_kind(), + .active_lanes = 1u, + .mode = emel::kernel::matmul::lane_mode::serial, + }, + matmul(matmul_policy) { + if (lane_count > 1u) { + attention_lanes.emplace(); + } + for (std::size_t index = 0u; index < input_embedding.size(); ++index) { + input_embedding[index] = + static_cast(static_cast(index % 127u) - 63) / 128.0f; + } + input_sequence[0] = model.moshi_lm.text_padding_id; + initialize_position(temporal_positions); + initialize_position(depformer_positions); + executor = + std::make_unique( + emel::speech::predictor::moshi::executor::dependencies{ + .kv = + emel::speech::predictor::moshi::executor::kv_views{ + .temporal = + { + .key_cache = std::span{key_cache}, + .value_cache = std::span{value_cache}, + .layer_cache_offsets = + std::span{layer_offsets}, + .layer_count = model.moshi_lm.num_layers, + .position_capacity = model.moshi_lm.context, + .kv_dim = model.moshi_lm.dim, + }, + .temporal_positions = &temporal_positions, + .depformer_positions = &depformer_positions, + }, + .kernel = kernel, + .matmul = matmul, + .matmul_lane_mode = emel::kernel::matmul::lane_mode::serial, + .attention_lanes = + attention_lanes ? &*attention_lanes : nullptr, + .active_attention_lanes = lane_count, + .sampler = nullptr, + .policy = + { + .rms_norm_epsilon = 1.0e-8f, + .zero_seed_state = 123459876u, + .sampling_modulus = 2147483647u, + .token_zero = -1, + }, + .capacity = + { + .hidden_dim = + static_cast(std::max(model.moshi_lm.dim, + model.moshi_lm.depformer_dim)), + .temporal_context = + static_cast(model.moshi_lm.context), + .depformer_context = + static_cast(model.moshi_lm + .depformer_context), + .sampling_card = + static_cast( + std:: + max(model.moshi_lm.text_card, model.moshi_lm + .card)), + .sampling_top_k = 1u, + }, + }); + emel::error::type err = emel::error::cast( + emel::speech::predictor::moshi::executor::error::none); + emel::speech::predictor::moshi::executor::event::initialize initialize{ + model}; + initialize.error_out = &err; + if (!executor->process_event(initialize)) { + std::fprintf(stderr, + "error: Moshi executor attention benchmark init failed " + "lanes=%zu err=%d\n", + lane_count, static_cast(err)); + std::exit(1); + } + prime_temporal_window(); + } + + static void initialize_position(emel::memory::streaming::sm &position) { + int32_t err = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + if (!position.process_event( + emel::memory::streaming::event::initialize{err}) || + err != static_cast( + emel::error::cast(emel::memory::streaming::error::none))) { + std::fprintf(stderr, + "error: Moshi executor attention position init failed\n"); + std::exit(1); + } + } + + void prime_temporal_window() noexcept { + for (int32_t index = 0; index < k_valid_positions - 1; ++index) { + emel::memory::streaming::advance_result result{}; + int32_t err = static_cast( + emel::error::cast(emel::memory::streaming::error::none)); + if (!temporal_positions.process_event( + emel::memory::streaming::event::advance{result, err}) || + err != static_cast( + emel::error::cast(emel::memory::streaming::error::none))) { + std::exit(1); + } + } + } + + void run_prediction() noexcept { + emel::error::type err = emel::error::cast( + emel::speech::predictor::moshi::executor::error::none); + std::fill(temporal_state.begin(), temporal_state.end(), 0.0f); + std::fill(audio_tokens.begin(), audio_tokens.end(), -1); + text_token = -1; + emel::memory::view::snapshot memory{}; + memory.max_sequences = 1; + memory.sequence_active[0] = 1; + memory.sequence_length_values[0] = k_valid_positions; + memory.sequence_kv_block_count[0] = 1; + memory.sequence_kv_blocks[0][0] = 0; + emel::speech::predictor::moshi::event::graph_step step{ + model, memory, std::span{input_sequence}, + std::span{audio_tokens}, text_token}; + step.phase = emel::speech::predictor::moshi::event::graph_phase::prediction; + step.input_embedding = std::span{input_embedding}; + step.temporal_state = std::span{temporal_state}; + step.error_out = &err; + if (!executor->process_event(step)) { + std::fprintf(stderr, + "error: Moshi executor attention prediction failed " + "lanes=%zu err=%d\n", + lane_count, static_cast(err)); + std::exit(1); + } + } + + attention_route_snapshot capture_snapshot() const { + return attention_route_snapshot{ + .temporal_state = temporal_state, + .key_cache = key_cache, + .value_cache = value_cache, + .audio_tokens = audio_tokens, + .text_token = text_token, + }; + } + + const emel::model::data &model; + std::size_t lane_count = 1u; + emel::memory::streaming::sm temporal_positions; + emel::memory::streaming::sm depformer_positions; + std::vector layer_offsets = {}; + std::vector key_cache = {}; + std::vector value_cache = {}; + std::vector input_sequence = {}; + std::vector input_embedding = {}; + std::vector temporal_state = {}; + std::vector audio_tokens = {}; + int32_t text_token = -1; + emel::kernel::sm kernel = {}; + emel::kernel::matmul::execution_policy matmul_policy; + emel::kernel::matmul::sm matmul; + std::optional attention_lanes = {}; + std::unique_ptr executor = {}; +}; + bool bench_enabled() { const char *flag = std::getenv("EMEL_BENCH_SPEECH_LM_MOSHI"); return flag != nullptr && flag[0] != '\0' && flag[0] != '0'; @@ -457,6 +732,67 @@ void append_emel_speech_lm_moshi_cases(std::vector &results, measured.note = "component=lm contract=moshi_execution dtype=q4_k"; results.push_back(std::move(measured)); + auto append_attention_result = [&](const char *name, const int32_t lanes, + const attention_route_snapshot + *reference) { + attention_route_snapshot measured_snapshot; + auto attention_result = measure_prepared_case( + name, cfg, + [&]() { + return std::make_unique( + fixture->model_data, static_cast(lanes)); + }, + [](attention_executor_bench_fixture &attention) { + attention.run_prediction(); + }, + [&](const attention_executor_bench_fixture &attention) { + measured_snapshot = attention.capture_snapshot(); + }); + const uint64_t measured_checksum = measured_snapshot.checksum(); + if (reference != nullptr && measured_snapshot != *reference) { + std::fprintf(stderr, + "error: measured %d-lane owning Moshi attention route is " + "not exact\n", + lanes); + std::exit(1); + } + attention_result.lane = "emel"; + attention_result.backend_id = "emel"; + attention_result.backend_language = "c++"; + attention_result.thread_count = lanes; + attention_result.thread_contract = + "owning_moshi_executor_same_rtc_attention_head_ranges"; + attention_result.model_id = "Codes4Fun/personaplex-7b-v1-q4_k-GGUF"; + attention_result.fixture_id = fixture_id_from_path(model_path); + attention_result.workload_id = + "moshi_executor_prediction_attention_capacity_3000_start_valid_250"; + attention_result.output_checksum = measured_checksum; + attention_result.output_dim = + static_cast(measured_snapshot.temporal_state.size()); + attention_result.note = + "component=moshi_executor_prediction only_delta=attention_lanes " + "nonattention_matmul=serial_identical capacity=3000 " + "start_valid_positions=250 measured_checksum=temporal_state+kv+outputs " + "iterations=" + + std::to_string(attention_result.iterations) + + " runs=" + std::to_string(attention_result.runs); + results.push_back(std::move(attention_result)); + return measured_snapshot; + }; + const attention_route_snapshot serial_snapshot = + append_attention_result("speech_lm_moshi/executor_prediction_attention/" + "personaplex_capacity_3000_valid_250_lanes_1", + 1, nullptr); + append_attention_result("speech_lm_moshi/executor_prediction_attention/" + "personaplex_capacity_3000_valid_250_lanes_2", + 2, &serial_snapshot); + append_attention_result("speech_lm_moshi/executor_prediction_attention/" + "personaplex_capacity_3000_valid_250_lanes_4", + 4, &serial_snapshot); + append_attention_result("speech_lm_moshi/executor_prediction_attention/" + "personaplex_capacity_3000_valid_250_lanes_8", + 8, &serial_snapshot); + release_file(fixture->mapping); } diff --git a/tools/bench/speech/mimi_emel_parity_runner.cpp b/tools/bench/speech/mimi_emel_parity_runner.cpp index 8bc70cc0..96e7d690 100644 --- a/tools/bench/speech/mimi_emel_parity_runner.cpp +++ b/tools/bench/speech/mimi_emel_parity_runner.cpp @@ -252,11 +252,9 @@ int main(int argc, char **argv) { return 1; } - std::vector prepared( - mimi::prepared_arena_floats(*loaded.model)); + std::vector prepared(mimi::prepared_arena_floats(*loaded.model)); std::vector state(mimi::state_arena_floats(*loaded.model)); - std::vector workspace( - mimi::workspace_arena_floats(*loaded.model)); + std::vector workspace(mimi::workspace_arena_floats(*loaded.model)); std::vector frame(mimi::frame_arena_floats(*loaded.model)); if (prepared.empty() || state.empty() || workspace.empty() || frame.empty()) { std::fprintf(stderr, "model does not describe a mimi codec component\n"); @@ -265,9 +263,12 @@ int main(int argc, char **argv) { mimi::sm codec{}; emel::error::type err = emel::error::cast(mimi::error::none); - mimi::event::initialize init{ - *loaded.model, std::span{prepared}, std::span{state}, - std::span{workspace}, std::span{frame}}; + mimi::event::initialize init{*loaded.model, + mimi::make_bind_contract(*loaded.model), + std::span{prepared}, + std::span{state}, + std::span{workspace}, + std::span{frame}}; init.error_out = &err; init.on_done = emel::callback::from<&on_codec_initialized>(); diff --git a/tools/bench/speech/personaplex_emel_runner.cpp b/tools/bench/speech/personaplex_emel_runner.cpp index 40979580..a29472de 100644 --- a/tools/bench/speech/personaplex_emel_runner.cpp +++ b/tools/bench/speech/personaplex_emel_runner.cpp @@ -9,8 +9,10 @@ #include #include #include +#include #include #include +#include #include #include "emel/batch/planner/sm.hpp" @@ -18,12 +20,15 @@ #include "emel/gguf/loader/detail.hpp" #include "emel/gguf/loader/events.hpp" #include "emel/gguf/loader/sm.hpp" +#include "emel/kernel/matmul/sm.hpp" #include "emel/kernel/sm.hpp" #include "emel/memory/streaming/sm.hpp" #include "emel/model/data.hpp" #include "emel/model/detail.hpp" +#include "emel/sm.hpp" #include "emel/speech/codec/mimi/any.hpp" #include "emel/speech/generator/any.hpp" +#include "emel/speech/generator/frame/any.hpp" #include "emel/speech/predictor/moshi/any.hpp" #include "emel/speech/predictor/moshi/executor/any.hpp" #include "emel/speech/tokenizer/moshi/any.hpp" @@ -32,6 +37,7 @@ namespace { namespace mimi = emel::speech::codec::mimi; namespace generator = emel::speech::generator; +namespace frame = emel::speech::generator::frame; namespace predictor = emel::speech::predictor::moshi; namespace runtime = emel::speech::predictor::moshi::executor; namespace tokenizer = emel::speech::tokenizer::moshi; @@ -51,6 +57,7 @@ struct runner_config { int32_t max_blocks = 0; int32_t block_tokens = 0; int32_t prompt_text_token = 0; + size_t cpu_threads = 1u; }; void noop_probe_done(const emel::gguf::loader::events::probe_done &) {} @@ -126,6 +133,74 @@ struct personaplex_dependencies { bool frame_plan_output_all = false; }; +struct personaplex_frame_dependencies { + using tokenize_event = tokenizer::event::tokenize; + using predict_event = predictor::event::predict; + using graph_event = predictor::event::execute; + using sample_event = predictor::event::sample; + using detokenize_event = tokenizer::event::detokenize; + + emel::batch::planner::sm &planner; + tokenizer::sm &tokenizer; + predictor::sm &predictor; + predictor::sm &graph; + predictor::sm &sampler; + predictor::event::predict::workspace &prediction_workspace; + std::span model_codes = {}; + std::span predicted_codes = {}; + int32_t codebook_count = 0; + emel::batch::planner::event::plan_mode frame_plan_mode = + emel::batch::planner::event::plan_mode::simple; + int32_t frame_plan_steps = 0; + int32_t frame_plan_token_count = 0; + bool frame_plan_output_all = false; +}; + +struct reset_codec_stream { + explicit reset_codec_stream(emel::error::type &error_out_ref) noexcept + : error_out(error_out_ref) {} + + emel::error::type &error_out; +}; + +struct mimi_wavefront_actor { + mimi::sm &actor; + + bool process_event(const mimi::event::encode_frame &ev) noexcept { + return actor.process_event(ev); + } + + bool process_event(const mimi::event::decode_frame &ev) noexcept { + return actor.process_event(ev); + } + + bool process_event(const reset_codec_stream &ev) noexcept { + ev.error_out = emel::error::cast(generator::error::none); + return actor.process_event(mimi::event::reset_stream{}); + } +}; + +template struct personaplex_wavefront_dependencies { + using generator_mode = generator::wavefront; + using wavefront_encode_event = mimi::event::encode_frame; + using wavefront_encode_reset_event = reset_codec_stream; + using wavefront_middle_reset_event = frame::event::reset; + using wavefront_decode_event = mimi::event::decode_frame; + using wavefront_decode_reset_event = reset_codec_stream; + + static constexpr size_t wavefront_frame_capacity = 4096u; + static constexpr size_t wavefront_codebook_capacity = 64u; + + mimi_wavefront_actor &wavefront_encoder; + middle_type &wavefront_middle; + mimi_wavefront_actor &wavefront_decoder; + generator::action::wavefront_stage_pool *stage_pool; + generator::action::wavefront_diagnostics &stage_diagnostics; + generator::action::wavefront_stage_mode stage_mode; + int32_t frame_samples = 0; + int32_t codebook_count = 0; +}; + bool read_binary_file(const std::filesystem::path &path, std::vector &bytes_out) { std::ifstream file{path, std::ios::binary}; @@ -342,13 +417,13 @@ bool prepare_streaming_cache(streaming_cache &cache, const int32_t layer_count, } // namespace int main(int argc, char **argv) { - if (argc != 15) { + if (argc != 16) { std::fprintf(stderr, "usage: %s " " " " " " " - "\n", + " \n", argv[0]); return 2; } @@ -378,6 +453,9 @@ int main(int argc, char **argv) { char *prompt_text_token_end = nullptr; const long parsed_prompt_text_token = std::strtol(argv[14], &prompt_text_token_end, 10); + char *cpu_threads_end = nullptr; + const unsigned long parsed_cpu_threads = + std::strtoul(argv[15], &cpu_threads_end, 10); if (seed_end == argv[7] || *seed_end != '\0' || parsed_seed == 0 || parsed_seed > std::numeric_limits::max() || audio_temperature_end == argv[8] || *audio_temperature_end != '\0' || @@ -397,7 +475,10 @@ int main(int argc, char **argv) { parsed_block_tokens > std::numeric_limits::max() || prompt_text_token_end == argv[14] || *prompt_text_token_end != '\0' || parsed_prompt_text_token < std::numeric_limits::min() || - parsed_prompt_text_token > std::numeric_limits::max()) { + parsed_prompt_text_token > std::numeric_limits::max() || + cpu_threads_end == argv[15] || *cpu_threads_end != '\0' || + (parsed_cpu_threads != 1u && parsed_cpu_threads != 2u && + parsed_cpu_threads != 4u && parsed_cpu_threads != 8u)) { std::fprintf(stderr, "invalid sampling configuration\n"); return 2; } @@ -416,6 +497,7 @@ int main(int argc, char **argv) { .max_blocks = static_cast(parsed_max_blocks), .block_tokens = static_cast(parsed_block_tokens), .prompt_text_token = static_cast(parsed_prompt_text_token), + .cpu_threads = static_cast(parsed_cpu_threads), }; loaded_model mimi_model = {}; @@ -527,6 +609,42 @@ int main(int argc, char **argv) { }, }; emel::kernel::sm prediction_kernel{}; + const char *stage_pipeline_probe = + std::getenv("EMEL_PERSONAPLEX_STAGE_PIPELINE_PROBE"); + const bool run_stage_pipeline = stage_pipeline_probe != nullptr && + std::string_view{stage_pipeline_probe} == "1"; + const size_t stage_worker_count = + run_stage_pipeline && config.cpu_threads >= 4u + ? generator::action::wavefront_stage_pool::static_worker_count + : 0u; + const size_t available_matmul_lanes = config.cpu_threads - stage_worker_count; + const size_t matmul_lane_count = available_matmul_lanes >= 8u ? 8u + : available_matmul_lanes >= 4u ? 4u + : available_matmul_lanes >= 2u ? 2u + : 1u; + std::optional + wavefront_stage_workers = {}; + if (stage_worker_count != 0u) { + wavefront_stage_workers.emplace(); + } + std::optional prediction_matmul_lanes = {}; + if (matmul_lane_count > 1u) { + prediction_matmul_lanes.emplace(matmul_lane_count - 1u); + } + const size_t matmul_worker_count = + prediction_matmul_lanes + ? prediction_matmul_lanes->active_worker_count() + : 0u; + emel::kernel::matmul::execution_policy prediction_matmul_policy{ + .parallel_matmul_lanes = + prediction_matmul_lanes ? &prediction_matmul_lanes.value() : nullptr, + .kernel_kind = emel::kernel::detect_host_kind(), + .active_lanes = matmul_lane_count, + .mode = matmul_lane_count == 1u + ? emel::kernel::matmul::lane_mode::serial + : emel::kernel::matmul::lane_mode::parallel, + }; + emel::kernel::matmul::sm prediction_matmul{prediction_matmul_policy}; emel::logits::sampler::sm prediction_sampler{}; runtime::sm prediction_runtime{runtime::dependencies{ .kv = @@ -537,6 +655,11 @@ int main(int argc, char **argv) { .depformer_positions = &secondary_positions, }, .kernel = prediction_kernel, + .matmul = prediction_matmul, + .matmul_lane_mode = prediction_matmul_policy.mode, + .attention_lanes = + prediction_matmul_lanes ? &prediction_matmul_lanes.value() : nullptr, + .active_attention_lanes = matmul_lane_count, .sampler = &prediction_sampler, .policy = runtime::policies{ @@ -561,6 +684,17 @@ int main(int argc, char **argv) { std::max(config.audio_top_k, config.text_top_k)), }, }}; + std::fprintf(stderr, + "EMEL_THREADS requested_total=%zu owner_threads=1 " + "stage_workers=%zu matmul_workers=%zu matmul_lanes=%zu " + "stage_mode=%s matmul_mode=%s\n", + config.cpu_threads, stage_worker_count, matmul_worker_count, + matmul_lane_count, + stage_worker_count == 0u ? "serial" : "parallel", + prediction_matmul_policy.mode == + emel::kernel::matmul::lane_mode::parallel + ? "parallel" + : "serial"); emel::memory::hybrid::sm prediction_memory{}; predictor::sm token_predictor{predictor::dependencies{ .memory = prediction_memory, @@ -594,6 +728,12 @@ int main(int argc, char **argv) { }}; mimi::sm encoder{}; mimi::sm decoder{}; + mimi::event::diagnostics encoder_diagnostics_before{}; + mimi::event::diagnostics decoder_diagnostics_before{}; + (void)encoder.process_event( + mimi::event::capture_diagnostics{encoder_diagnostics_before}); + (void)decoder.process_event( + mimi::event::capture_diagnostics{decoder_diagnostics_before}); emel::batch::planner::sm frame_planner{}; predictor::event::initialize predictor_initialize{*lm_model.model}; @@ -624,12 +764,14 @@ int main(int argc, char **argv) { .prediction_workspace = prediction_workspace, .encoder_initialize = mimi::event::initialize{*mimi_model.model, + mimi::make_bind_contract(*mimi_model.model), std::span{encoder_prepared}, std::span{encoder_state}, std::span{encoder_workspace}, std::span{encoder_frame}}, .decoder_initialize = mimi::event::initialize{*mimi_model.model, + mimi::make_bind_contract(*mimi_model.model), std::span{decoder_prepared}, std::span{decoder_state}, std::span{decoder_workspace}, @@ -659,116 +801,309 @@ int main(int argc, char **argv) { .frame_plan_token_count = 1, .frame_plan_output_all = true, }; - generator::sm session{dependencies}; + std::optional> session{}; + session.emplace(dependencies); emel::error::type session_err = emel::error::cast(generator::error::none); - if (!session.process_event(generator::event::initialize{session_err})) { + if (!session->process_event(generator::event::initialize{session_err})) { std::fprintf(stderr, "PersonaPlex session initialize failed err=%d\n", static_cast(session_err)); return 1; } + mimi::event::diagnostics encoder_diagnostics_initialized{}; + mimi::event::diagnostics decoder_diagnostics_initialized{}; + (void)encoder.process_event( + mimi::event::capture_diagnostics{encoder_diagnostics_initialized}); + (void)decoder.process_event( + mimi::event::capture_diagnostics{decoder_diagnostics_initialized}); while ( - session.is(stateforward::sml::state)) { + session->is(stateforward::sml::state)) { bool complete = false; int32_t remaining = -1; session_err = emel::error::cast(generator::error::none); - (void)session.process_event(generator::event::condition{ + (void)session->process_event(generator::event::condition{ config.prompt_text_token, complete, remaining, session_err}); } - while ( - session.is(stateforward::sml::state)) { + while (session->is( + stateforward::sml::state)) { bool complete = false; int32_t remaining = -1; session_err = emel::error::cast(generator::error::none); - (void)session.process_event(generator::event::condition{ + (void)session->process_event(generator::event::condition{ config.prompt_text_token, complete, remaining, session_err}); } - if (!session.is(stateforward::sml::state)) { + if (!session->is(stateforward::sml::state)) { std::fprintf(stderr, "PersonaPlex prefill failed err=%d\n", static_cast(session_err)); return 1; } - for (size_t frame_index = 0; frame_index < input_frames; ++frame_index) { - std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); - const size_t begin = frame_index * frame_samples; - const size_t copy_count = std::min(frame_samples, input_pcm.size() - begin); - std::copy_n(input_pcm.data() + begin, copy_count, pcm_frame.data()); - int32_t text_token = -1; - bool produced = false; - int32_t sample_count = 0; - session_err = emel::error::cast(generator::error::none); - generator::event::stream_frame frame{ - std::span{pcm_frame}, - std::span{output_pcm.data() + frame_index * frame_samples, - frame_samples}, - std::span{input_codes}, - std::span{output_codes}, - text_token, - sample_count, - produced, - session_err}; - if (!session.process_event(frame)) { - std::fprintf(stderr, "PersonaPlex stream frame=%zu failed err=%d\n", - frame_index, static_cast(session_err)); - return 1; - } - std::fprintf(stderr, "EMEL_INPUT frame=%zu codes=", frame_index); - for (int32_t index = 0; index < public_n_q; ++index) { - std::fprintf(stderr, "%s%d", index == 0 ? "" : ",", - input_codes[static_cast(index)]); - } - std::fprintf(stderr, "\nEMEL_OUTPUT frame=%zu text=%d codes=", frame_index, - text_token); - for (int32_t index = 0; index < public_n_q; ++index) { - std::fprintf(stderr, "%s%d", index == 0 ? "" : ",", - output_codes[static_cast(index)]); + if (run_stage_pipeline) { + session.reset(); + + personaplex_frame_dependencies middle_dependencies{ + .planner = frame_planner, + .tokenizer = token_delay, + .predictor = token_predictor, + .graph = token_predictor, + .sampler = token_predictor, + .prediction_workspace = prediction_workspace, + .model_codes = std::span{model_codes}, + .predicted_codes = std::span{predicted_codes}, + .codebook_count = public_n_q, + .frame_plan_mode = emel::batch::planner::event::plan_mode::simple, + .frame_plan_steps = 1, + .frame_plan_token_count = 1, + .frame_plan_output_all = true, + }; + frame::sm middle{middle_dependencies}; + mimi_wavefront_actor wavefront_encoder{encoder}; + mimi_wavefront_actor wavefront_decoder{decoder}; + generator::action::wavefront_diagnostics stage_diagnostics{}; + using wavefront_dependencies = + personaplex_wavefront_dependencies; + wavefront_dependencies wavefront_deps{ + .wavefront_encoder = wavefront_encoder, + .wavefront_middle = middle, + .wavefront_decoder = wavefront_decoder, + .stage_pool = wavefront_stage_workers ? &wavefront_stage_workers.value() + : nullptr, + .stage_diagnostics = stage_diagnostics, + .stage_mode = stage_worker_count == 0u + ? generator::action::wavefront_stage_mode::serial + : generator::action::wavefront_stage_mode::parallel, + .frame_samples = frame_samples_i32, + .codebook_count = public_n_q, + }; + generator::sm wavefront{wavefront_deps}; + std::vector wavefront_pcm(frame_samples, 0.0f); + size_t produced_frames = 0u; + std::fprintf(stderr, + "EMEL_PHASE name=steady_begin implementation=canonical " + "frames=%zu\n", + config.target_output_frames); + + for (size_t sequence = 0u; sequence < config.target_output_frames; + ++sequence) { + std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); + if (sequence < input_frames) { + const size_t begin = sequence * frame_samples; + const size_t copy_count = + std::min(frame_samples, input_pcm.size() - begin); + std::copy_n(input_pcm.data() + begin, copy_count, pcm_frame.data()); + } + + generator::event::wavefront_attribution output_attribution{}; + int32_t text_token = -1; + int32_t sample_count = 0; + bool produced = false; + session_err = emel::error::cast(generator::error::none); + generator::event::wavefront_frame request{ + std::span{pcm_frame}, + std::span{wavefront_pcm}, + std::span{output_codes}, + {.sequence = sequence, .source = 1u}, + output_attribution, + text_token, + sample_count, + produced, + session_err}; + if (!wavefront.process_event(request)) { + std::fprintf( + stderr, + "canonical wavefront frame=%zu failed err=%d produced=%d " + "output_sequence=%llu\n", + sequence, static_cast(session_err), static_cast(produced), + static_cast(output_attribution.sequence)); + return 1; + } + if (produced) { + if (output_attribution.sequence != produced_frames || + output_attribution.sequence >= config.target_output_frames || + sample_count != frame_samples_i32) { + std::fprintf( + stderr, + "canonical wavefront attribution failed input=%zu " + "output=%llu expected=%zu samples=%d\n", + sequence, + static_cast(output_attribution.sequence), + produced_frames, sample_count); + return 1; + } + std::copy(wavefront_pcm.begin(), wavefront_pcm.end(), + output_pcm.begin() + static_cast( + produced_frames * frame_samples)); + ++produced_frames; + } } - std::fprintf(stderr, "\n"); - } - std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); - const size_t flush_steps = config.target_output_frames - input_frames; - for (size_t index = 0; index < flush_steps; ++index) { - const size_t frame_index = input_frames + index; - int32_t text_token = -1; - int32_t sample_count = 0; bool complete = false; - session_err = emel::error::cast(generator::error::none); - generator::event::flush frame{ - std::span{output_pcm.data() + frame_index * frame_samples, - frame_samples}, - std::span{input_codes}, - std::span{output_codes}, - text_token, - sample_count, - complete, - session_err}; - if (!session.process_event(frame)) { - std::fprintf(stderr, "PersonaPlex flush frame=%zu failed err=%d\n", - frame_index, static_cast(session_err)); + for (size_t drain_step = 0u; drain_step < 2u; ++drain_step) { + generator::event::wavefront_attribution output_attribution{}; + int32_t text_token = -1; + int32_t sample_count = 0; + bool produced = false; + session_err = emel::error::cast(generator::error::none); + generator::event::wavefront_flush request{ + std::span{wavefront_pcm}, + std::span{output_codes}, + output_attribution, + text_token, + sample_count, + produced, + complete, + session_err}; + if (!wavefront.process_event(request)) { + std::fprintf( + stderr, + "canonical wavefront drain=%zu failed err=%d produced=%d " + "complete=%d output_sequence=%llu\n", + drain_step, static_cast(session_err), + static_cast(produced), static_cast(complete), + static_cast(output_attribution.sequence)); + return 1; + } + if (produced) { + if (output_attribution.sequence != produced_frames || + output_attribution.sequence >= config.target_output_frames || + sample_count != frame_samples_i32) { + std::fprintf( + stderr, + "canonical wavefront drain attribution failed step=%zu " + "output=%llu expected=%zu samples=%d\n", + drain_step, + static_cast(output_attribution.sequence), + produced_frames, sample_count); + return 1; + } + std::copy(wavefront_pcm.begin(), wavefront_pcm.end(), + output_pcm.begin() + static_cast( + produced_frames * frame_samples)); + ++produced_frames; + } + } + if (!complete || produced_frames != config.target_output_frames || + !wavefront.is( + stateforward::sml::state)) { + std::fprintf(stderr, + "canonical wavefront incomplete produced=%zu expected=%zu " + "complete=%d\n", + produced_frames, config.target_output_frames, + static_cast(complete)); + return 1; + } + const uint64_t submissions = + stage_diagnostics.submissions.load(std::memory_order_relaxed); + const uint64_t joins = + stage_diagnostics.joins.load(std::memory_order_relaxed); + const uint64_t worker_entries = + stage_diagnostics.worker_entries.load(std::memory_order_relaxed); + const uint64_t worker_exits = + stage_diagnostics.worker_exits.load(std::memory_order_relaxed); + if (submissions != joins || worker_entries != worker_exits) { + std::fprintf(stderr, + "canonical wavefront ownership failed submissions=%llu " + "joins=%llu worker_entries=%llu worker_exits=%llu\n", + static_cast(submissions), + static_cast(joins), + static_cast(worker_entries), + static_cast(worker_exits)); return 1; } - std::fprintf(stderr, "EMEL_INPUT frame=%zu codes=", frame_index); - for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { - std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", - input_codes[static_cast(codebook)]); + std::fprintf(stderr, + "EMEL_PHASE name=steady_end implementation=canonical " + "frames=%zu submissions=%llu joins=%llu worker_entries=%llu " + "worker_exits=%llu\n", + config.target_output_frames, + static_cast(submissions), + static_cast(joins), + static_cast(worker_entries), + static_cast(worker_exits)); + } else { + for (size_t frame_index = 0; frame_index < input_frames; ++frame_index) { + std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); + const size_t begin = frame_index * frame_samples; + const size_t copy_count = + std::min(frame_samples, input_pcm.size() - begin); + std::copy_n(input_pcm.data() + begin, copy_count, pcm_frame.data()); + int32_t text_token = -1; + bool produced = false; + int32_t sample_count = 0; + session_err = emel::error::cast(generator::error::none); + generator::event::stream_frame frame{ + std::span{pcm_frame}, + std::span{output_pcm.data() + frame_index * frame_samples, + frame_samples}, + std::span{input_codes}, + std::span{output_codes}, + text_token, + sample_count, + produced, + session_err}; + if (!session->process_event(frame)) { + std::fprintf(stderr, "PersonaPlex stream frame=%zu failed err=%d\n", + frame_index, static_cast(session_err)); + return 1; + } + std::fprintf(stderr, "EMEL_INPUT frame=%zu codes=", frame_index); + for (int32_t index = 0; index < public_n_q; ++index) { + std::fprintf(stderr, "%s%d", index == 0 ? "" : ",", + input_codes[static_cast(index)]); + } + std::fprintf(stderr, + "\nEMEL_OUTPUT frame=%zu text=%d codes=", frame_index, + text_token); + for (int32_t index = 0; index < public_n_q; ++index) { + std::fprintf(stderr, "%s%d", index == 0 ? "" : ",", + output_codes[static_cast(index)]); + } + std::fprintf(stderr, "\n"); } - std::fprintf(stderr, "\nEMEL_OUTPUT frame=%zu text=%d codes=", frame_index, - text_token); - for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { - std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", - output_codes[static_cast(codebook)]); + + std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); + const size_t flush_steps = config.target_output_frames - input_frames; + for (size_t index = 0; index < flush_steps; ++index) { + const size_t frame_index = input_frames + index; + int32_t text_token = -1; + int32_t sample_count = 0; + bool complete = false; + session_err = emel::error::cast(generator::error::none); + generator::event::flush frame{ + std::span{output_pcm.data() + frame_index * frame_samples, + frame_samples}, + std::span{input_codes}, + std::span{output_codes}, + text_token, + sample_count, + complete, + session_err}; + if (!session->process_event(frame)) { + std::fprintf(stderr, "PersonaPlex flush frame=%zu failed err=%d\n", + frame_index, static_cast(session_err)); + return 1; + } + std::fprintf(stderr, "EMEL_INPUT frame=%zu codes=", frame_index); + for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { + std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", + input_codes[static_cast(codebook)]); + } + std::fprintf(stderr, + "\nEMEL_OUTPUT frame=%zu text=%d codes=", frame_index, + text_token); + for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { + std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", + output_codes[static_cast(codebook)]); + } + std::fprintf(stderr, "\n"); + } + const bool session_finished = + flush_steps == 0 + ? session->is(stateforward::sml::state) + : session->is(stateforward::sml::state); + if (!session_finished) { + std::fprintf(stderr, "PersonaPlex session failed err=%d\n", + static_cast(session_err)); + return 1; } - std::fprintf(stderr, "\n"); - } - const bool session_finished = - flush_steps == 0 - ? session.is(stateforward::sml::state) - : session.is(stateforward::sml::state); - if (!session_finished) { - std::fprintf(stderr, "PersonaPlex session failed err=%d\n", - static_cast(session_err)); - return 1; } if (output_pcm.empty()) { @@ -790,6 +1125,40 @@ int main(int argc, char **argv) { return 1; } + mimi::event::diagnostics encoder_diagnostics_final{}; + mimi::event::diagnostics decoder_diagnostics_final{}; + (void)encoder.process_event( + mimi::event::capture_diagnostics{encoder_diagnostics_final}); + (void)decoder.process_event( + mimi::event::capture_diagnostics{decoder_diagnostics_final}); + const auto print_mimi_diagnostics = + [](const char *facade, const mimi::event::diagnostics &before, + const mimi::event::diagnostics &init, + const mimi::event::diagnostics &final) { + std::fprintf( + stderr, + "EMEL_MIMI_F32_X4 facade=%s init_prepare_calls=%llu " + "init_prepared_floats=%llu run_exact_x4_calls=%llu " + "run_reference_calls=%llu run_legacy_f32_calls=%llu\n", + facade, + static_cast(init.projection_prepare_calls - + before.projection_prepare_calls), + static_cast(init.projection_prepared_floats - + before.projection_prepared_floats), + static_cast(final.projection_exact_x4_calls - + init.projection_exact_x4_calls), + static_cast(final.projection_reference_calls - + init.projection_reference_calls), + static_cast(final.legacy_f32_projection_calls - + init.legacy_f32_projection_calls)); + }; + print_mimi_diagnostics("encoder", encoder_diagnostics_before, + encoder_diagnostics_initialized, + encoder_diagnostics_final); + print_mimi_diagnostics("decoder", decoder_diagnostics_before, + decoder_diagnostics_initialized, + decoder_diagnostics_final); + std::printf("input_frames=%zu output_frames=%zu output_seconds=%.3f " "peak=%.6f rms=%.6f seed=%u output=%s\n", input_frames, output_pcm.size() / frame_samples, diff --git a/tools/bench/text/generator/parallel_matmul_bench.cpp b/tools/bench/text/generator/parallel_matmul_bench.cpp index e2dd8501..139f2099 100644 --- a/tools/bench/text/generator/parallel_matmul_bench.cpp +++ b/tools/bench/text/generator/parallel_matmul_bench.cpp @@ -11,6 +11,7 @@ #include "emel/kernel/detail.hpp" #include "emel/kernel/events.hpp" +#include "emel/kernel/matmul/sm.hpp" #include "emel/kernel/sm.hpp" #include "emel/sm.hpp" @@ -47,19 +48,13 @@ constexpr size_t k_worker_lanes = k_lanes - 1u; constexpr int32_t k_dim = 2048; constexpr int32_t k_gemm_tokens = 8; -using lane_pool = emel::policy::fork_join_lane_pool; +using lane_pool = emel::kernel::matmul::lane_pool; uint64_t weight_group_rows(const dtype type) noexcept { - const uint8_t code = emel::kernel::detail::dtype_code(type); - if (code == emel::kernel::detail::dtype_q8_0_x4_bl4 || - code == emel::kernel::detail::dtype_q8_0_x4_bl8) { + if (emel::kernel::matmul::guard::guard_uses_x4_row_groups(type)) { return emel::kernel::detail::quant::Q8_0_X4_ROWS; } - if (code == emel::kernel::detail::dtype_q4_k_x8_bl4 || - code == emel::kernel::detail::dtype_q4_k_x8_bl8 || - code == emel::kernel::detail::dtype_q6_k_x8 || - code == emel::kernel::detail::dtype_q6_k_x8_q8_prepared || - code == emel::kernel::detail::dtype_q6_k_x8_q8_argmax_prepared) { + if (emel::kernel::matmul::guard::guard_uses_x8_row_groups(type)) { return emel::kernel::detail::quant::Q4_K_X8_ROWS; } return 1u; @@ -78,10 +73,8 @@ emel::kernel::kernel_kind host_kernel_kind() { #endif } -tensor_view make_weight_view(const void * data, - const dtype type, - const uint64_t row_bytes, - const int32_t cols, +tensor_view make_weight_view(const void *data, const dtype type, + const uint64_t row_bytes, const int32_t cols, const int32_t rows) { tensor_view view{}; view.data = data; @@ -89,16 +82,19 @@ tensor_view make_weight_view(const void * data, view.ne = {static_cast(cols), static_cast(rows), 1u, 1u}; view.nb[0] = type == dtype::f32 ? sizeof(float) : 1u; view.nb[1] = row_bytes; - view.nb[2] = row_bytes * weight_storage_rows(type, static_cast(rows)); + view.nb[2] = + row_bytes * weight_storage_rows(type, static_cast(rows)); view.nb[3] = view.nb[2]; return view; } -tensor_view make_input_view(const float * data, const int32_t tokens, const int32_t cols) { +tensor_view make_input_view(const float *data, const int32_t tokens, + const int32_t cols) { tensor_view view{}; view.data = data; view.type = dtype::f32; - view.ne = {static_cast(tokens), static_cast(cols), 1u, 1u}; + view.ne = {static_cast(tokens), static_cast(cols), 1u, + 1u}; view.nb[0] = sizeof(float); view.nb[1] = sizeof(float) * static_cast(tokens); view.nb[2] = view.nb[1] * static_cast(cols); @@ -106,7 +102,7 @@ tensor_view make_input_view(const float * data, const int32_t tokens, const int3 return view; } -tensor_view make_q8_k_vector_view(const void * data, const int32_t cols) { +tensor_view make_q8_k_vector_view(const void *data, const int32_t cols) { const size_t row_bytes = emel::kernel::detail::quantized_row_storage_bytes( emel::kernel::detail::dtype_q8_k, static_cast(cols)); tensor_view view{}; @@ -120,7 +116,7 @@ tensor_view make_q8_k_vector_view(const void * data, const int32_t cols) { return view; } -tensor_view make_q8_0_vector_view(const void * data, const int32_t cols) { +tensor_view make_q8_0_vector_view(const void *data, const int32_t cols) { const size_t row_bytes = emel::kernel::detail::quantized_row_storage_bytes( emel::kernel::detail::dtype_q8_0, static_cast(cols)); tensor_view view{}; @@ -134,11 +130,13 @@ tensor_view make_q8_0_vector_view(const void * data, const int32_t cols) { return view; } -tensor_view_mut make_output_view(float * data, const int32_t tokens, const int32_t rows) { +tensor_view_mut make_output_view(float *data, const int32_t tokens, + const int32_t rows) { tensor_view_mut view{}; view.data = data; view.type = dtype::f32; - view.ne = {static_cast(tokens), static_cast(rows), 1u, 1u}; + view.ne = {static_cast(tokens), static_cast(rows), 1u, + 1u}; view.nb[0] = sizeof(float); view.nb[1] = sizeof(float) * static_cast(tokens); view.nb[2] = view.nb[1] * static_cast(rows); @@ -146,36 +144,12 @@ tensor_view_mut make_output_view(float * data, const int32_t tokens, const int32 return view; } -// Mirror the production group-aligned contiguous row split; k_dim rows divide -// evenly by lane count for every case in this suite. -op_mul_mat make_sliced_event(const op_mul_mat & ev, - const uint64_t row_begin, - const uint64_t row_count) { - op_mul_mat sliced = ev; - const uint64_t group_rows = weight_group_rows(ev.src0.type); - const uint64_t slice_groups = (row_count + group_rows - 1u) / group_rows; - sliced.src0.data = - static_cast(ev.src0.data) + - (row_begin / group_rows) * ev.src0.nb[1]; - sliced.src0.ne[1] = row_count; - sliced.src0.nb[2] = ev.src0.nb[1] * slice_groups; - sliced.src0.nb[3] = sliced.src0.nb[2]; - sliced.dst.data = static_cast(ev.dst.data) + row_begin * ev.dst.nb[1]; - sliced.dst.ne[1] = row_count; - sliced.dst.nb[2] = ev.dst.nb[1] * row_count; - sliced.dst.nb[3] = sliced.dst.nb[2]; - return sliced; -} - struct lane_fixture { - std::array kernels = {}; lane_pool pool = {}; - - lane_fixture() { - for (auto & kernel : kernels) { - kernel.set_kind(host_kernel_kind()); - } - } + emel::kernel::matmul::execution_policy policy = + emel::kernel::matmul::make_execution_policy(pool, host_kernel_kind(), + k_lanes); + emel::kernel::matmul::sm actor{policy}; }; struct case_buffers { @@ -191,16 +165,16 @@ case_buffers make_case(const dtype type, const int32_t tokens) { case_buffers buffers; const uint8_t code = emel::kernel::detail::dtype_code(type); const uint64_t raw_row_bytes = - type == dtype::f32 - ? sizeof(float) * static_cast(k_dim) - : emel::kernel::detail::quantized_row_storage_bytes( - code, static_cast(k_dim)); - std::vector raw_weights(raw_row_bytes * static_cast(k_dim), 0u); + type == dtype::f32 ? sizeof(float) * static_cast(k_dim) + : emel::kernel::detail::quantized_row_storage_bytes( + code, static_cast(k_dim)); + std::vector raw_weights(raw_row_bytes * static_cast(k_dim), + 0u); for (size_t idx = 0; idx < raw_weights.size(); ++idx) { raw_weights[idx] = static_cast((idx * 31u + 7u) & 0x3fu); } if (type == dtype::f32) { - auto * values = reinterpret_cast(raw_weights.data()); + auto *values = reinterpret_cast(raw_weights.data()); const size_t count = raw_weights.size() / sizeof(float); for (size_t idx = 0; idx < count; ++idx) { values[idx] = 0.25f * static_cast((idx * 13u + 5u) % 17u) - 2.0f; @@ -219,8 +193,7 @@ case_buffers make_case(const dtype type, const int32_t tokens) { const bool packed = emel::kernel::detail::quant::pack_q4_k_rows_x8_bl8( reinterpret_cast( raw_weights.data()), - static_cast(k_dim), - static_cast(k_dim), + static_cast(k_dim), static_cast(k_dim), buffers.weights.data()); if (!packed) { std::fprintf(stderr, "error: parallel matmul q4_k pack failed\n"); @@ -237,8 +210,7 @@ case_buffers make_case(const dtype type, const int32_t tokens) { const bool packed = emel::kernel::detail::quant::pack_q8_0_rows_x4_bl8( reinterpret_cast( raw_weights.data()), - static_cast(k_dim), - static_cast(k_dim), + static_cast(k_dim), static_cast(k_dim), buffers.weights.data()); if (!packed) { std::fprintf(stderr, "error: parallel matmul q8_0 pack failed\n"); @@ -248,16 +220,17 @@ case_buffers make_case(const dtype type, const int32_t tokens) { buffers.weights = std::move(raw_weights); } - buffers.input.assign( - static_cast(tokens) * static_cast(k_dim), 0.0f); + buffers.input.assign(static_cast(tokens) * static_cast(k_dim), + 0.0f); for (size_t idx = 0; idx < buffers.input.size(); ++idx) { - buffers.input[idx] = 0.125f * static_cast((idx * 7u + 3u) % 19u) - 1.0f; + buffers.input[idx] = + 0.125f * static_cast((idx * 7u + 3u) % 19u) - 1.0f; } buffers.output.assign( static_cast(tokens) * static_cast(k_dim), 0.0f); - buffers.ev.src0 = - make_weight_view(buffers.weights.data(), storage_type, row_bytes, k_dim, k_dim); + buffers.ev.src0 = make_weight_view(buffers.weights.data(), storage_type, + row_bytes, k_dim, k_dim); buffers.ev.src1 = make_input_view(buffers.input.data(), tokens, k_dim); if (tokens == 1 && (type == dtype::q4_k || type == dtype::q6_k)) { buffers.input_q8_k.resize( @@ -272,7 +245,7 @@ case_buffers make_case(const dtype type, const int32_t tokens) { } struct bench_case_spec { - const char * name; + const char *name; dtype type; int32_t tokens; }; @@ -295,10 +268,11 @@ constexpr std::array k_ggml_cases = {{ {"parallel_matmul/ggml_gemm8_f32", dtype::f32, k_gemm_tokens}, }}; -bool prepare_emel_case_rhs(case_buffers & buffers, - const bench_case_spec & spec) noexcept { +bool prepare_emel_case_rhs(case_buffers &buffers, + const bench_case_spec &spec) noexcept { buffers.ev.src1 = make_input_view(buffers.input.data(), spec.tokens, k_dim); - if (spec.tokens == 1 && (spec.type == dtype::q4_k || spec.type == dtype::q6_k)) { + if (spec.tokens == 1 && + (spec.type == dtype::q4_k || spec.type == dtype::q6_k)) { if (buffers.input_q8_k.size() != static_cast(k_dim / emel::kernel::detail::quant::QK_K)) { return false; @@ -323,14 +297,14 @@ bool prepare_emel_case_rhs(case_buffers & buffers, ggml_type ggml_type_for(const dtype type) { switch (type) { - case dtype::q8_0: - return GGML_TYPE_Q8_0; - case dtype::q4_k: - return GGML_TYPE_Q4_K; - case dtype::q6_k: - return GGML_TYPE_Q6_K; - default: - return GGML_TYPE_F32; + case dtype::q8_0: + return GGML_TYPE_Q8_0; + case dtype::q4_k: + return GGML_TYPE_Q4_K; + case dtype::q6_k: + return GGML_TYPE_Q6_K; + default: + return GGML_TYPE_F32; } } @@ -340,9 +314,9 @@ ggml_type ggml_type_for(const dtype type) { // bytes use the identical deterministic pattern as make_case; block layouts // (q8_0/q4_K/q6_K) are byte-compatible between the two implementations. struct ggml_matmul_reference { - ggml_context * ctx = nullptr; - ggml_cgraph * graph = nullptr; - ggml_threadpool * threadpool = nullptr; + ggml_context *ctx = nullptr; + ggml_cgraph *graph = nullptr; + ggml_threadpool *threadpool = nullptr; std::vector work = {}; ggml_cplan plan = {}; volatile float sink = 0.0f; @@ -359,33 +333,36 @@ struct ggml_matmul_reference { init.no_alloc = false; ctx = ggml_init(init); if (ctx == nullptr) { - std::fprintf(stderr, "error: parallel matmul ggml reference init failed\n"); + std::fprintf(stderr, + "error: parallel matmul ggml reference init failed\n"); std::abort(); } graph = ggml_new_graph(ctx); - ggml_tensor * weights = ggml_new_tensor_2d(ctx, type, k_dim, k_dim); - ggml_tensor * activation = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_dim, tokens); + ggml_tensor *weights = ggml_new_tensor_2d(ctx, type, k_dim, k_dim); + ggml_tensor *activation = + ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_dim, tokens); if (type == GGML_TYPE_F32) { - auto * values = static_cast(weights->data); + auto *values = static_cast(weights->data); const size_t count = weight_bytes / sizeof(float); for (size_t idx = 0; idx < count; ++idx) { values[idx] = 0.25f * static_cast((idx * 13u + 5u) % 17u) - 2.0f; } } else { - auto * bytes = static_cast(weights->data); + auto *bytes = static_cast(weights->data); for (size_t idx = 0; idx < weight_bytes; ++idx) { bytes[idx] = static_cast((idx * 31u + 7u) & 0x3fu); } } - auto * input = static_cast(activation->data); - const size_t input_count = static_cast(tokens) * static_cast(k_dim); + auto *input = static_cast(activation->data); + const size_t input_count = + static_cast(tokens) * static_cast(k_dim); for (size_t idx = 0; idx < input_count; ++idx) { input[idx] = 0.125f * static_cast((idx * 7u + 3u) % 19u) - 1.0f; } ggml_build_forward_expand(graph, ggml_mul_mat(ctx, weights, activation)); ggml_threadpool_params tp = ggml_threadpool_params_default(static_cast(k_lanes)); - tp.poll = 100; // warm polling, matching EMEL's warm lane pool + tp.poll = 100; // warm polling, matching EMEL's warm lane pool threadpool = ggml_threadpool_new(&tp); plan = ggml_graph_plan(graph, static_cast(k_lanes), threadpool); work.resize(plan.work_size != 0u ? plan.work_size : 1u); @@ -402,106 +379,100 @@ struct ggml_matmul_reference { } ggml_matmul_reference(const ggml_matmul_reference &) = delete; - ggml_matmul_reference & operator=(const ggml_matmul_reference &) = delete; + ggml_matmul_reference &operator=(const ggml_matmul_reference &) = delete; void run() noexcept { if (ggml_graph_compute(graph, &plan) != GGML_STATUS_SUCCESS) { - std::fprintf(stderr, "error: parallel matmul ggml reference compute failed\n"); + std::fprintf(stderr, + "error: parallel matmul ggml reference compute failed\n"); std::abort(); } sink += 1.0f; } }; -} // namespace +} // namespace namespace emel::bench { -void append_emel_parallel_matmul_cases(std::vector & results, const config & cfg) { +void append_emel_parallel_matmul_cases(std::vector &results, + const config &cfg) { volatile float sink = 0.0f; - const auto measure_parallel = [&](const bench_case_spec & spec) { + const auto measure_parallel = [&](const bench_case_spec &spec) { lane_fixture fixture; auto buffers = make_case(spec.type, spec.tokens); - std::array lane_events = {}; - constexpr uint64_t rows_per_lane = static_cast(k_dim) / k_lanes; - for (size_t lane = 0; lane < k_lanes; ++lane) { - lane_events[lane] = make_sliced_event( - buffers.ev, static_cast(lane) * rows_per_lane, rows_per_lane); - } - auto fn = [&]() { + const auto dispatch_once = [&]() { if (!prepare_emel_case_rhs(buffers, spec)) { - std::fprintf(stderr, "error: parallel matmul RHS preparation failed\n"); - std::abort(); + return false; } - for (auto & lane_ev : lane_events) { - lane_ev.src1 = buffers.ev.src1; - } - std::array lane_ok = {}; - lane_pool::join_group group{}; - bool all_submitted = true; - for (size_t lane = 1; lane < k_lanes; ++lane) { - auto & kernel = fixture.kernels[lane]; - const auto & lane_ev = lane_events[lane]; - auto & ok_flag = lane_ok[lane]; - const bool submitted = fixture.pool.try_submit( - group, [&kernel, &lane_ev, &ok_flag]() noexcept { - ok_flag = kernel.process_event(lane_ev); - }); - all_submitted = all_submitted && submitted; - } - if (!all_submitted) { - (void)group.wait(); - std::fprintf(stderr, "error: parallel matmul lane submit failed\n"); - std::abort(); - } - lane_ok[0] = fixture.kernels[0].process_event(lane_events[0]); - const bool joined = group.wait(); - if (!joined) { - std::fprintf(stderr, "error: parallel matmul lane join failed\n"); + emel::kernel::matmul::event::dispatch_result dispatch = {}; + bool accepted = false; + const emel::kernel::matmul::event::execute_parallel run{ + buffers.ev, dispatch, accepted}; + const bool dispatched = fixture.actor.process_event(run); + return dispatched && accepted && dispatch.all_submitted && + dispatch.submitted_worker_lanes == dispatch.drained_worker_lanes && + dispatch.all_lanes_accepted; + }; + if (!dispatch_once()) { + std::fprintf(stderr, + "error: parallel matmul canonical preflight failed\n"); + std::abort(); + } + auto fn = [&]() { + if (!dispatch_once()) { + std::fprintf(stderr, + "error: parallel matmul canonical dispatch failed\n"); std::abort(); } - bool all_ok = true; - for (const bool ok : lane_ok) { - all_ok = all_ok && ok; - } - sink += all_ok ? buffers.output[0] : -1.0f; + sink += buffers.output[0]; }; results.push_back(measure_case(spec.name, cfg, fn)); }; - for (const auto & spec : k_ggml_cases) { + for (const auto &spec : k_ggml_cases) { measure_parallel(spec); } - for (const auto & spec : k_cases) { + for (const auto &spec : k_cases) { measure_parallel(spec); } } -void append_reference_parallel_matmul_cases(std::vector & results, const config & cfg) { +void append_reference_parallel_matmul_cases(std::vector &results, + const config &cfg) { static emel::kernel::sm kernel; kernel.set_kind(host_kernel_kind()); volatile float sink = 0.0f; - for (const auto & spec : k_ggml_cases) { - auto fixture = - std::make_unique(ggml_type_for(spec.type), spec.tokens); + for (const auto &spec : k_ggml_cases) { + auto fixture = std::make_unique( + ggml_type_for(spec.type), spec.tokens); auto fn = [&fixture]() { fixture->run(); }; results.push_back(measure_case(spec.name, cfg, fn)); } - for (const auto & spec : k_cases) { + for (const auto &spec : k_cases) { auto buffers = make_case(spec.type, spec.tokens); - auto fn = [&]() { + const auto dispatch_once = [&]() { if (!prepare_emel_case_rhs(buffers, spec)) { - std::fprintf(stderr, "error: parallel matmul serial RHS preparation failed\n"); + return false; + } + return kernel.process_event(buffers.ev); + }; + if (!dispatch_once()) { + std::fprintf(stderr, "error: parallel matmul serial preflight failed\n"); + std::abort(); + } + auto fn = [&]() { + if (!dispatch_once()) { + std::fprintf(stderr, "error: parallel matmul serial dispatch failed\n"); std::abort(); } - const bool ok = kernel.process_event(buffers.ev); - sink += ok ? buffers.output[0] : -1.0f; + sink += buffers.output[0]; }; results.push_back(measure_case(spec.name, cfg, fn)); } } -} // namespace emel::bench +} // namespace emel::bench diff --git a/tools/decode_wavefront_eval/main.cpp b/tools/decode_wavefront_eval/main.cpp index 6b846f25..e97cfedd 100644 --- a/tools/decode_wavefront_eval/main.cpp +++ b/tools/decode_wavefront_eval/main.cpp @@ -124,7 +124,7 @@ struct lane_session { emel::text::tokenizer::sm tokenizer = {}; emel::text::conditioner::sm conditioner = {}; emel::model::generation::contract generation_contract = {}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; std::unique_ptr generator = {}; initialize_capture initialize = {}; generation_capture generation = {}; @@ -656,7 +656,7 @@ int main(int argc, char **argv) { return 1; } const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::make_auto_execution_policy( s->parallel_matmul_lanes); s->generator = std::make_unique( emel::text::generator::dependencies{ diff --git a/tools/determinism_check/main.cpp b/tools/determinism_check/main.cpp index 39ea485d..4af151f2 100644 --- a/tools/determinism_check/main.cpp +++ b/tools/determinism_check/main.cpp @@ -182,7 +182,7 @@ struct session { emel::text::tokenizer::sm tokenizer = {}; emel::text::conditioner::sm conditioner = {}; emel::model::generation::contract generation_contract = {}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; std::unique_ptr generator = {}; std::array samplers = {}; logits_probe probe = {}; @@ -667,7 +667,7 @@ make_session(const emel::model::data &model, return nullptr; } const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::make_auto_execution_policy( s->parallel_matmul_lanes); s->generator = std::make_unique( emel::text::generator::dependencies{ diff --git a/tools/docsgen/docsgen.cpp b/tools/docsgen/docsgen.cpp index af54fbea..cafa630c 100644 --- a/tools/docsgen/docsgen.cpp +++ b/tools/docsgen/docsgen.cpp @@ -372,6 +372,10 @@ render_template(const fs::path &template_path, } } + if (rendered.empty() || rendered.back() != '\n') { + rendered.push_back('\n'); + } + return rendered; } diff --git a/tools/embedded_size/emel_probe/main.cpp b/tools/embedded_size/emel_probe/main.cpp index cf9c4c29..04ffd674 100644 --- a/tools/embedded_size/emel_probe/main.cpp +++ b/tools/embedded_size/emel_probe/main.cpp @@ -145,7 +145,7 @@ struct emel_session { emel::text::tokenizer::sm tokenizer = {}; emel::text::conditioner::sm conditioner = {}; emel::model::generation::contract generation_contract = {}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> parallel_matmul_lanes = {}; + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; std::unique_ptr generator = {}; formatter_binding formatter = {}; initialize_capture initialize = {}; @@ -1269,7 +1269,7 @@ int main(int argc, char **argv) { #endif } const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::make_auto_execution_policy( session->parallel_matmul_lanes); session->generator = std::make_unique( emel::text::generator::dependencies{ diff --git a/tools/paritychecker/parity_engines.cpp b/tools/paritychecker/parity_engines.cpp index 8db475f1..6ddca185 100644 --- a/tools/paritychecker/parity_engines.cpp +++ b/tools/paritychecker/parity_engines.cpp @@ -1014,7 +1014,7 @@ struct generation_load_state { emel::text::tokenizer::sm tokenizer = {}; emel::text::conditioner::sm conditioner = {}; emel::model::generation::contract generation_contract = {}; - emel::text::generator::matmul::lane_pool<7u, 128u, 1048576u> + emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; std::unique_ptr generator = {}; reference_backend reference = {}; @@ -1615,7 +1615,7 @@ run_emel_initialize_generator(generation_load_state &state, emel::memory::view::DEFAULT_BLOCK_TOKENS, session_tokens)); const auto matmul_policy = - emel::text::generator::matmul::make_auto_execution_policy( + emel::kernel::matmul::make_auto_execution_policy( state.parallel_matmul_lanes); state.generator = std::make_unique( emel::text::generator::dependencies{ From 24537c8f36464323169a52059020ddcf067b8b1f Mon Sep 17 00:00:00 2001 From: gabewillen Date: Thu, 23 Jul 2026 23:29:08 -0500 Subject: [PATCH 2/6] fix: address PersonaPlex PR review findings --- scripts/test_with_coverage.sh | 1 + src/emel/kernel/attention/actions.hpp | 6 +++++- src/emel/sm.hpp | 21 +++++++-------------- tests/kernel/attention_tests.cpp | 12 ++++++++++++ tests/sm/sm_policy_tests.cpp | 6 ++++++ 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/scripts/test_with_coverage.sh b/scripts/test_with_coverage.sh index 03940a7e..26b344ee 100755 --- a/scripts/test_with_coverage.sh +++ b/scripts/test_with_coverage.sh @@ -44,6 +44,7 @@ fi required_tools=( cmake ctest + ninja gcovr clang-format llvm-cov diff --git a/src/emel/kernel/attention/actions.hpp b/src/emel/kernel/attention/actions.hpp index a8024ba3..ffa824ec 100644 --- a/src/emel/kernel/attention/actions.hpp +++ b/src/emel/kernel/attention/actions.hpp @@ -119,7 +119,11 @@ struct effect_emit_execute_error { struct effect_on_unexpected { template - void operator()(const event_type &, context &) const noexcept {} + void operator()(const event_type &ev, context &) const noexcept { + if constexpr (requires { ev.result.accepted; }) { + ev.result.accepted = false; + } + } }; } // namespace emel::kernel::attention::action diff --git a/src/emel/sm.hpp b/src/emel/sm.hpp index 56c35792..20eb14ce 100644 --- a/src/emel/sm.hpp +++ b/src/emel/sm.hpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -131,13 +130,12 @@ class fork_join_lane_pool { static constexpr std::size_t static_worker_count = worker_count; - fork_join_lane_pool() : fork_join_lane_pool(worker_count) {} + fork_join_lane_pool() noexcept : fork_join_lane_pool(worker_count) {} - explicit fork_join_lane_pool(const std::size_t active_worker_count) + explicit fork_join_lane_pool(const std::size_t active_worker_count) noexcept : active_worker_count_(active_worker_count) { if (active_worker_count_ == 0u || active_worker_count_ > worker_count) { - throw std::invalid_argument( - "fork_join_lane_pool active worker count is out of range"); + std::terminate(); } start_workers(); } @@ -299,15 +297,10 @@ class fork_join_lane_pool { std::atomic stopping = false; }; - void start_workers() { - try { - for (std::size_t index = 0u; index < active_worker_count_; ++index) { - workers_[index].thread = - std::thread([this, index]() noexcept { worker_loop(index); }); - } - } catch (...) { - stop_workers(); - throw; + void start_workers() noexcept { + for (std::size_t index = 0u; index < active_worker_count_; ++index) { + workers_[index].thread = + std::thread([this, index]() noexcept { worker_loop(index); }); } } diff --git a/tests/kernel/attention_tests.cpp b/tests/kernel/attention_tests.cpp index 1270243a..ea863b85 100644 --- a/tests/kernel/attention_tests.cpp +++ b/tests/kernel/attention_tests.cpp @@ -358,3 +358,15 @@ TEST_CASE("kernel attention rejects unsafe public span storage") { CHECK_FALSE(actor->process_event(attention::event::execute{request, result})); CHECK_FALSE(result.accepted); } + +TEST_CASE("kernel attention rejects unexpected result-bearing dispatches") { + struct unexpected_dispatch { + attention::event::dispatch_result &result; + }; + + attention::action::context ctx{}; + attention::event::dispatch_result result{.accepted = true}; + attention::action::effect_on_unexpected{}(unexpected_dispatch{result}, ctx); + + CHECK_FALSE(result.accepted); +} diff --git a/tests/sm/sm_policy_tests.cpp b/tests/sm/sm_policy_tests.cpp index 46bc5bf7..e39b8f40 100644 --- a/tests/sm/sm_policy_tests.cpp +++ b/tests/sm/sm_policy_tests.cpp @@ -784,6 +784,12 @@ TEST_CASE("fork_join_lane_pool_starts_only_the_runtime_worker_budget") { CHECK(ran.load(std::memory_order_acquire) == 1); } +TEST_CASE("fork_join_lane_pool_runtime_budget_constructor is noexcept") { + using pool_type = emel::policy::fork_join_lane_pool<7u, 128u, 64u>; + CHECK(std::is_nothrow_default_constructible_v); + CHECK(std::is_nothrow_constructible_v); +} + TEST_CASE("fork_join_lane_pool_batch_rejects_partial_claim_and_reuses_slots") { using pool_type = emel::policy::fork_join_lane_pool<2u, 128u, 64u>; pool_type pool{}; From d56cfc5f6e9de22fa9472a3a0bc6ce0ffdae0fec Mon Sep 17 00:00:00 2001 From: gabewillen Date: Fri, 24 Jul 2026 00:01:51 -0500 Subject: [PATCH 3/6] fix: exercise maintained PersonaPlex wavefront --- src/emel/speech/generator/actions.hpp | 11 ++ src/emel/speech/generator/events.hpp | 3 + src/emel/speech/generator/guards.hpp | 24 ++- .../generator/wavefront_lifecycle_tests.cpp | 110 +++++++++---- tools/bench/bench_runner_tests.cpp | 100 ++++++------ tools/bench/personaplex_compare_tests.cpp | 10 +- tools/bench/speech/lm_moshi_bench.cpp | 9 +- .../bench/speech/personaplex_emel_runner.cpp | 144 +++++------------- 8 files changed, 226 insertions(+), 185 deletions(-) diff --git a/src/emel/speech/generator/actions.hpp b/src/emel/speech/generator/actions.hpp index 0b0a3415..1a18b743 100644 --- a/src/emel/speech/generator/actions.hpp +++ b/src/emel/speech/generator/actions.hpp @@ -931,6 +931,11 @@ struct effect_execute_wavefront_phase_parallel { (void)group.wait(); effect_record_joins(ctx, submitted_tasks); + if constexpr (encode_active) { + std::copy(encoded_lane(ctx).begin(), + encoded_lane(ctx).end(), + runtime_ev.request.encoded_tokens_out.begin()); + } runtime_ev.ctx.all_submitted = all_submitted; runtime_ev.ctx.joined = true; } @@ -987,6 +992,12 @@ struct effect_execute_wavefront_phase_serial { generated_text_token(ctx); } + if constexpr (encode_active) { + std::copy(encoded_lane(ctx).begin(), + encoded_lane(ctx).end(), + runtime_ev.request.encoded_tokens_out.begin()); + } + runtime_ev.ctx.all_submitted = true; runtime_ev.ctx.joined = true; } diff --git a/src/emel/speech/generator/events.hpp b/src/emel/speech/generator/events.hpp index 2fbd2e65..900b27fa 100644 --- a/src/emel/speech/generator/events.hpp +++ b/src/emel/speech/generator/events.hpp @@ -132,6 +132,7 @@ struct wavefront_attribution { struct wavefront_frame { wavefront_frame(std::span pcm_in_ref, std::span pcm_out_ref, + std::span encoded_tokens_out_ref, std::span generated_tokens_out_ref, const wavefront_attribution input_attribution_ref, wavefront_attribution &output_attribution_ref, @@ -139,6 +140,7 @@ struct wavefront_frame { bool &produced_out_ref, emel::error::type &error_out_ref) noexcept : pcm_in(pcm_in_ref), pcm_out(pcm_out_ref), + encoded_tokens_out(encoded_tokens_out_ref), generated_tokens_out(generated_tokens_out_ref), input_attribution(input_attribution_ref), output_attribution(output_attribution_ref), @@ -148,6 +150,7 @@ struct wavefront_frame { const std::span pcm_in; const std::span pcm_out; + const std::span encoded_tokens_out; const std::span generated_tokens_out; const wavefront_attribution input_attribution; wavefront_attribution &output_attribution; diff --git a/src/emel/speech/generator/guards.hpp b/src/emel/speech/generator/guards.hpp index eed7f6be..05b0228a 100644 --- a/src/emel/speech/generator/guards.hpp +++ b/src/emel/speech/generator/guards.hpp @@ -264,6 +264,19 @@ struct guard_condition_complete_done_callback_absent { } }; +template +struct guard_wavefront_stage_configuration_valid { + template + bool + operator()(const runtime_event_type &, + const action::context &ctx) const noexcept { + return ctx.collaborators.stage_mode != + action::wavefront_stage_mode::parallel || + static_cast( + ctx.collaborators.stage_pool) != nullptr; + } +}; + template struct guard_wavefront_frame_valid { bool operator()(const detail::wavefront_frame_run &runtime_ev, @@ -273,10 +286,13 @@ template struct guard_wavefront_frame_valid { static_cast(ctx.collaborators.codebook_count); return ctx.collaborators.frame_samples > 0 && ctx.collaborators.codebook_count > 0 && + guard_wavefront_stage_configuration_valid{}( + runtime_ev, ctx) && samples <= dependencies_type::wavefront_frame_capacity && codebooks <= dependencies_type::wavefront_codebook_capacity && runtime_ev.request.pcm_in.size() == samples && runtime_ev.request.pcm_out.size() >= samples && + runtime_ev.request.encoded_tokens_out.size() >= codebooks && runtime_ev.request.generated_tokens_out.size() >= codebooks && runtime_ev.request.input_attribution.sequence != event::k_invalid_wavefront_sequence && @@ -302,6 +318,8 @@ template struct guard_wavefront_flush_valid { static_cast(ctx.collaborators.codebook_count); return ctx.collaborators.frame_samples > 0 && ctx.collaborators.codebook_count > 0 && + guard_wavefront_stage_configuration_valid{}( + runtime_ev, ctx) && samples <= dependencies_type::wavefront_frame_capacity && codebooks <= dependencies_type::wavefront_codebook_capacity && runtime_ev.request.pcm_out.size() >= samples && @@ -321,9 +339,11 @@ template struct guard_wavefront_stage_mode { template bool - operator()(const runtime_event_type &, + operator()(const runtime_event_type &runtime_ev, const action::context &ctx) const noexcept { - return ctx.collaborators.stage_mode == stage_mode; + return ctx.collaborators.stage_mode == stage_mode && + guard_wavefront_stage_configuration_valid{}( + runtime_ev, ctx); } }; diff --git a/tests/speech/generator/wavefront_lifecycle_tests.cpp b/tests/speech/generator/wavefront_lifecycle_tests.cpp index 434bce92..9aef29ec 100644 --- a/tests/speech/generator/wavefront_lifecycle_tests.cpp +++ b/tests/speech/generator/wavefront_lifecycle_tests.cpp @@ -374,6 +374,7 @@ struct fixture { generator::action::wavefront_diagnostics diagnostics{}; wavefront_dependencies dependencies; generator::sm machine; + std::array last_encoded_tokens{}; int32_t last_text_token = -1; int32_t last_sample_count = 0; @@ -406,6 +407,7 @@ struct fixture { generator::event::wavefront_frame request{ pcm_in, pcm_out, + last_encoded_tokens, tokens_out, {.sequence = sequence, .source = 99u}, output, @@ -556,6 +558,8 @@ TEST_CASE( for (uint64_t sequence = 0u; sequence < frame_total; ++sequence) { bool produced = true; REQUIRE(test.push(sequence, produced, output, pcm, tokens, err)); + CHECK(test.last_encoded_tokens[0] == static_cast(sequence + 1u)); + CHECK(test.last_encoded_tokens[1] == static_cast(sequence + 2u)); CHECK(produced == (sequence >= 2u)); if (produced) { CHECK(output.sequence == output_sequence++); @@ -624,25 +628,31 @@ TEST_CASE("speech_generator_wavefront_emits_done_callbacks_for_each_public_" TEST_CASE("speech_generator_wavefront_rejects_each_invalid_public_request_" "shape") { - const auto check_invalid_frame = [](fixture &test, - std::span pcm_in, - std::span pcm_out, - std::span tokens_out, - const uint64_t sequence) { - generator::event::wavefront_attribution output{}; - int32_t text_token = 7; - int32_t samples = 7; - bool produced = true; - emel::error::type err = 0u; - const generator::event::wavefront_frame request{ - pcm_in, pcm_out, tokens_out, {.sequence = sequence, .source = 99u}, - output, text_token, samples, produced, - err}; - - CHECK_FALSE(test.machine.process_event(request)); - CHECK_FALSE(produced); - CHECK(err == error_code(generator::error::invalid_request)); - }; + const auto check_invalid_frame = + [](fixture &test, std::span pcm_in, std::span pcm_out, + std::span tokens_out, const uint64_t sequence) { + std::array encoded_tokens{}; + generator::event::wavefront_attribution output{}; + int32_t text_token = 7; + int32_t samples = 7; + bool produced = true; + emel::error::type err = 0u; + const generator::event::wavefront_frame request{ + pcm_in, + pcm_out, + encoded_tokens, + tokens_out, + {.sequence = sequence, .source = 99u}, + output, + text_token, + samples, + produced, + err}; + + CHECK_FALSE(test.machine.process_event(request)); + CHECK_FALSE(produced); + CHECK(err == error_code(generator::error::invalid_request)); + }; const auto check_invalid_flush = [](fixture &test, std::span pcm_out, std::span tokens_out) { @@ -730,13 +740,20 @@ TEST_CASE( test.overlap.hold_workers.store(true); callback_probe probe{}; std::array pcm_in{}; + std::array encoded_tokens{}; pcm_in.fill(3.0f); int32_t text_token = -1; int32_t samples = 0; - generator::event::wavefront_frame request{ - pcm_in, pcm, tokens, {.sequence = 2u, .source = 99u}, - output, text_token, samples, produced, - err}; + generator::event::wavefront_frame request{pcm_in, + pcm, + encoded_tokens, + tokens, + {.sequence = 2u, .source = 99u}, + output, + text_token, + samples, + produced, + err}; request.on_done = decltype(request.on_done)::from( &probe); @@ -775,6 +792,17 @@ TEST_CASE("speech_generator_wavefront_serial_stage_mode_uses_no_workers") { CHECK(test.diagnostics.worker_exits.load() == 0u); } +TEST_CASE("speech_generator_wavefront_parallel_mode_requires_stage_pool") { + fixture test{}; + auto dependencies = test.dependencies; + dependencies.stage_pool = nullptr; + generator::action::context ctx{dependencies}; + + CHECK_FALSE((generator::guard::guard_wavefront_stage_mode< + wavefront_dependencies, + generator::action::wavefront_stage_mode::parallel>{})(0, ctx)); +} + TEST_CASE( "speech_generator_wavefront_drains_partial_submit_and_phase_failures") { SUBCASE("encode failure publishes no output") { @@ -1140,12 +1168,20 @@ TEST_CASE("speech_generator_wavefront_rejects_wrong_lifecycle_with_immediate_" REQUIRE(complete); std::array pcm_in{}; + std::array encoded_tokens{}; int32_t text_token = -1; int32_t samples = 0; callback_probe probe{}; generator::event::wavefront_frame frame_request{ - pcm_in, pcm, tokens, {.sequence = 0u, .source = 99u}, - output, text_token, samples, produced, + pcm_in, + pcm, + encoded_tokens, + tokens, + {.sequence = 0u, .source = 99u}, + output, + text_token, + samples, + produced, err}; frame_request.on_error = decltype(frame_request.on_error)::from< callback_probe, &callback_probe::frame_error>(&probe); @@ -1194,6 +1230,7 @@ TEST_CASE("speech_generator_wavefront_detects_corrupt_lane_attribution") { run_ctx.encode_accepted = true; std::array pcm_in{}; std::array pcm_out{}; + std::array encoded_tokens{}; std::array tokens{}; generator::event::wavefront_attribution output{}; int32_t text_token = -1; @@ -1201,8 +1238,15 @@ TEST_CASE("speech_generator_wavefront_detects_corrupt_lane_attribution") { bool produced = false; emel::error::type err = 0u; const generator::event::wavefront_frame request{ - pcm_in, pcm_out, tokens, {.sequence = 0u, .source = 99u}, - output, text_token, samples, produced, + pcm_in, + pcm_out, + encoded_tokens, + tokens, + {.sequence = 0u, .source = 99u}, + output, + text_token, + samples, + produced, err}; const generator::detail::wavefront_frame_run runtime_ev{request, run_ctx}; using phase_fill0 = generator::guard::guard_wavefront_phase_succeeded< @@ -1317,6 +1361,7 @@ TEST_CASE("speech_generator_wavefront_trace_is_deterministic_and_statically_" traced_machine machine{ctx, logger}; std::array pcm_in{}; std::array pcm_out{}; + std::array encoded_tokens{}; std::array tokens{}; generator::event::wavefront_attribution output{}; int32_t text_token = -1; @@ -1324,8 +1369,15 @@ TEST_CASE("speech_generator_wavefront_trace_is_deterministic_and_statically_" bool produced = false; emel::error::type err = 0u; const generator::event::wavefront_frame request{ - pcm_in, pcm_out, tokens, {.sequence = 0u, .source = 99u}, - output, text_token, samples, produced, + pcm_in, + pcm_out, + encoded_tokens, + tokens, + {.sequence = 0u, .source = 99u}, + output, + text_token, + samples, + produced, err}; logger.reset(); diff --git a/tools/bench/bench_runner_tests.cpp b/tools/bench/bench_runner_tests.cpp index cff754b8..e9cedffe 100644 --- a/tools/bench/bench_runner_tests.cpp +++ b/tools/bench/bench_runner_tests.cpp @@ -82,12 +82,11 @@ void write_file(const std::filesystem::path &path, } void make_executable(const std::filesystem::path &path) { - std::filesystem::permissions( - path, - std::filesystem::perms::owner_read | - std::filesystem::perms::owner_write | - std::filesystem::perms::owner_exec, - std::filesystem::perm_options::add); + std::filesystem::permissions(path, + std::filesystem::perms::owner_read | + std::filesystem::perms::owner_write | + std::filesystem::perms::owner_exec, + std::filesystem::perm_options::add); } void replace_all(std::string &text, const std::string_view needle, @@ -1156,15 +1155,14 @@ TEST_CASE( has_speech_lm_script = has_speech_lm_script || record.path == "scripts/bench_moshi_lm_compare.sh"; has_speech_lm_setup = has_speech_lm_setup || - record.path == - "scripts/setup_moshi_cpp_reference.sh"; + record.path == "scripts/setup_moshi_cpp_reference.sh"; has_speech_lm_moshi_binding = has_speech_lm_moshi_binding || record.path == std::string_view{"src/emel/model/moshi"}; has_personaplex_emel_runner = has_personaplex_emel_runner || - record.path == std::string_view{ - "tools/bench/speech/personaplex_emel_runner.cpp"}; + record.path == + std::string_view{"tools/bench/speech/personaplex_emel_runner.cpp"}; } CHECK(has_speech_lm_script); CHECK(has_speech_lm_setup); @@ -1191,9 +1189,8 @@ TEST_CASE( CHECK(rendered.find("record runner=speech_lm_moshi kind=source " "path=tools/bench/speech/lm_moshi_bench.cpp") != std::string::npos); - CHECK(rendered.find( - "record runner=speech_lm_moshi kind=source " - "path=tools/bench/speech/personaplex_emel_runner.cpp") != + CHECK(rendered.find("record runner=speech_lm_moshi kind=source " + "path=tools/bench/speech/personaplex_emel_runner.cpp") != std::string::npos); const std::filesystem::path manifest_path = @@ -1213,11 +1210,23 @@ TEST_CASE( CHECK(docs.find("full_gate_on=missing,stale,uncertain") != std::string::npos); } +TEST_CASE("speech Moshi attention benchmark owns per-layer KV cache slices") { + const std::string source = read_file(repo_root() / "tools" / "bench" / + "speech" / "lm_moshi_bench.cpp"); + + CHECK( + source.find( + "key_cache(static_cast(model.moshi_lm.num_layers) *") != + std::string::npos); + CHECK(source.find("layer_offsets[index] = index * per_layer_cache;") != + std::string::npos); +} + #if !defined(_WIN32) TEST_CASE("moshi lm wrapper gives --model priority over inherited env") { - const std::filesystem::path tmp_dir = - std::filesystem::temp_directory_path() / "emel-bench-runner-tests" / - "moshi-lm-model-priority"; + const std::filesystem::path tmp_dir = std::filesystem::temp_directory_path() / + "emel-bench-runner-tests" / + "moshi-lm-model-priority"; const std::filesystem::path fake_bin_dir = tmp_dir / "bin"; const std::filesystem::path build_dir = tmp_dir / "build"; const std::filesystem::path fake_runner = build_dir / "bench_runner"; @@ -1245,18 +1254,20 @@ TEST_CASE("moshi lm wrapper gives --model priority over inherited env") { "exit 0\n"); make_executable(fake_runner); - std::string command = "PATH=" + quote_arg_posix(fake_bin_dir.string()) + - ":$PATH "; - command += "EMEL_TEST_INVOKED_PATH=" + - quote_arg_posix(invoked_path.string()) + " "; - command += "EMEL_MOSHI_LM_COMPARE_BUILD_DIR=" + - quote_arg_posix(build_dir.string()) + " "; - command += "EMEL_PERSONAPLEX_LM_MODEL=" + - quote_arg_posix(stale_model.string()) + " "; + std::string command = + "PATH=" + quote_arg_posix(fake_bin_dir.string()) + ":$PATH "; + command += + "EMEL_TEST_INVOKED_PATH=" + quote_arg_posix(invoked_path.string()) + " "; + command += + "EMEL_MOSHI_LM_COMPARE_BUILD_DIR=" + quote_arg_posix(build_dir.string()) + + " "; + command += + "EMEL_PERSONAPLEX_LM_MODEL=" + quote_arg_posix(stale_model.string()) + + " "; command += quote_arg_posix("/bin/bash") + " " + quote_arg_posix(bench_moshi_lm_compare_wrapper_path().string()); - command += " --run-only --system --model " + - quote_arg_posix(cli_model.string()); + command += + " --run-only --system --model " + quote_arg_posix(cli_model.string()); command += " > " + quote_arg_posix(stdout_path.string()); command += " 2> " + quote_arg_posix(stderr_path.string()); @@ -1265,8 +1276,7 @@ TEST_CASE("moshi lm wrapper gives --model priority over inherited env") { const std::string invoked = read_file(invoked_path); CHECK_MESSAGE(!invoked.empty(), capture.stderr_text); - CHECK(invoked.find("personaplex=" + cli_model.string()) != - std::string::npos); + CHECK(invoked.find("personaplex=" + cli_model.string()) != std::string::npos); CHECK(invoked.find("moshi=" + cli_model.string()) != std::string::npos); CHECK(invoked.find("bench=" + cli_model.string()) != std::string::npos); CHECK(invoked.find("personaplex=" + stale_model.string()) == @@ -1278,8 +1288,8 @@ TEST_CASE("moshi lm wrapper keeps build-only model-free") { const std::size_t mutual_exclusion = script.find("if $BUILD_ONLY && $RUN_ONLY; then"); const std::size_t model_guard = script.find("if ! $BUILD_ONLY; then"); - const std::size_t setup_call = - script.find("setup_output=\"$(\"$ROOT_DIR/scripts/setup_moshi_cpp_reference.sh\")\""); + const std::size_t setup_call = script.find( + "setup_output=\"$(\"$ROOT_DIR/scripts/setup_moshi_cpp_reference.sh\")\""); const std::size_t tool_check = script.find("for tool in cmake ninja git"); const std::size_t build_only_exit = script.find("if $BUILD_ONLY; then"); @@ -1545,8 +1555,7 @@ TEST_CASE("generation_stage_probe_emel_path_does_not_bypass_generator_actor") { TEST_CASE("sortformer_diarization_bench_uses_public_actor_surfaces") { const std::array source_paths = { - repo_root() / "tools" / "bench" / "diarization" / - "sortformer_bench.cpp", + repo_root() / "tools" / "bench" / "diarization" / "sortformer_bench.cpp", repo_root() / "tools" / "bench" / "diarization" / "sortformer_fixture.hpp", }; @@ -1556,7 +1565,8 @@ TEST_CASE("sortformer_diarization_bench_uses_public_actor_surfaces") { "#include \"emel/diarization/sortformer/request/guards.hpp\"", "#include \"emel/diarization/sortformer/encoder/actions.hpp\"", "#include \"emel/diarization/sortformer/encoder/detail.hpp\"", - "#include \"emel/diarization/sortformer/encoder/feature_extractor/detail.hpp\"", + "#include " + "\"emel/diarization/sortformer/encoder/feature_extractor/detail.hpp\"", "#include \"emel/diarization/sortformer/encoder/guards.hpp\"", "#include \"emel/diarization/sortformer/executor/actions.hpp\"", "#include \"emel/diarization/sortformer/executor/detail.hpp\"", @@ -1602,14 +1612,14 @@ TEST_CASE("sortformer_diarization_bench_uses_public_actor_surfaces") { const std::array public_facade_paths = { repo_root() / "src" / "emel" / "model" / "sortformer" / "any.hpp", - repo_root() / "src" / "emel" / "diarization" / "sortformer" / - "request" / "events.hpp", - repo_root() / "src" / "emel" / "diarization" / "sortformer" / - "output" / "any.hpp", - repo_root() / "src" / "emel" / "diarization" / "sortformer" / - "pipeline" / "any.hpp", - repo_root() / "src" / "emel" / "diarization" / "sortformer" / - "executor" / "events.hpp", + repo_root() / "src" / "emel" / "diarization" / "sortformer" / "request" / + "events.hpp", + repo_root() / "src" / "emel" / "diarization" / "sortformer" / "output" / + "any.hpp", + repo_root() / "src" / "emel" / "diarization" / "sortformer" / "pipeline" / + "any.hpp", + repo_root() / "src" / "emel" / "diarization" / "sortformer" / "executor" / + "events.hpp", }; for (const auto &facade_path : public_facade_paths) { const std::string source = read_file(facade_path); @@ -1708,10 +1718,10 @@ TEST_CASE("bench_runner generation jsonl emits manifest-driven workload " std::string::npos); CHECK(emel_capture.stdout_text.find("\"kernel_dispatch_calls\":") != std::string::npos); - CHECK(emel_capture.stdout_text.find( - "\"flash_attention_dispatch_calls\":") != std::string::npos); - CHECK(emel_capture.stdout_text.find( - "\"native_quantized_stage_count\":") != std::string::npos); + CHECK(emel_capture.stdout_text.find("\"flash_attention_dispatch_calls\":") != + std::string::npos); + CHECK(emel_capture.stdout_text.find("\"native_quantized_stage_count\":") != + std::string::npos); const std::filesystem::path gemma4_fixture_path = repo_root() / "tests" / "models" / "gemma-4-e2b-it-Q8_0.gguf"; if (std::filesystem::exists(gemma4_fixture_path)) { diff --git a/tools/bench/personaplex_compare_tests.cpp b/tools/bench/personaplex_compare_tests.cpp index edfbd85a..cbd21c0a 100644 --- a/tools/bench/personaplex_compare_tests.cpp +++ b/tools/bench/personaplex_compare_tests.cpp @@ -228,12 +228,20 @@ TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { CHECK(emel_source.find( "predictor_initialize.sampling_seed = config.sampling_seed") != std::string::npos); - CHECK(emel_source.find("flush_steps == 0") != std::string::npos); + CHECK(emel_source.find("state_wavefront_complete") != std::string::npos); CHECK(emel_source.find("sampling_seed = 1234") == std::string::npos); CHECK(emel_source.find("std::optional") != std::string::npos); CHECK(emel_source.find("stage_worker_count") != std::string::npos); CHECK(emel_source.find("matmul_lane_count") != std::string::npos); + CHECK(emel_source.find("EMEL_PERSONAPLEX_STAGE_PIPELINE_PROBE") == + std::string::npos); + CHECK(emel_source.find("if (run_stage_pipeline)") == std::string::npos); + CHECK(emel_source.find("generator::event::wavefront_frame") != + std::string::npos); + CHECK(emel_source.find("EMEL_INPUT frame=%llu codes=") != std::string::npos); + CHECK(emel_source.find("EMEL_OUTPUT frame=%llu text=%d codes=") != + std::string::npos); CHECK(emel_source.find( "prediction_matmul_lanes.emplace(matmul_lane_count - 1u)") != std::string::npos); diff --git a/tools/bench/speech/lm_moshi_bench.cpp b/tools/bench/speech/lm_moshi_bench.cpp index fa494cea..2d95b186 100644 --- a/tools/bench/speech/lm_moshi_bench.cpp +++ b/tools/bench/speech/lm_moshi_bench.cpp @@ -180,7 +180,8 @@ struct attention_executor_bench_fixture { depformer_positions(emel::memory::streaming::dependencies{ .capacity = model.moshi_lm.depformer_context}), layer_offsets(static_cast(model.moshi_lm.num_layers), 0u), - key_cache(static_cast(model.moshi_lm.context) * + key_cache(static_cast(model.moshi_lm.num_layers) * + static_cast(model.moshi_lm.context) * static_cast(model.moshi_lm.dim)), value_cache(key_cache.size()), input_sequence(static_cast(model.moshi_lm.n_q + 1), -1), @@ -194,6 +195,12 @@ struct attention_executor_bench_fixture { .mode = emel::kernel::matmul::lane_mode::serial, }, matmul(matmul_policy) { + const std::size_t per_layer_cache = + static_cast(model.moshi_lm.context) * + static_cast(model.moshi_lm.dim); + for (std::size_t index = 0u; index < layer_offsets.size(); ++index) { + layer_offsets[index] = index * per_layer_cache; + } if (lane_count > 1u) { attention_lanes.emplace(); } diff --git a/tools/bench/speech/personaplex_emel_runner.cpp b/tools/bench/speech/personaplex_emel_runner.cpp index a29472de..c18a0de0 100644 --- a/tools/bench/speech/personaplex_emel_runner.cpp +++ b/tools/bench/speech/personaplex_emel_runner.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include "emel/batch/planner/sm.hpp" @@ -609,12 +608,8 @@ int main(int argc, char **argv) { }, }; emel::kernel::sm prediction_kernel{}; - const char *stage_pipeline_probe = - std::getenv("EMEL_PERSONAPLEX_STAGE_PIPELINE_PROBE"); - const bool run_stage_pipeline = stage_pipeline_probe != nullptr && - std::string_view{stage_pipeline_probe} == "1"; const size_t stage_worker_count = - run_stage_pipeline && config.cpu_threads >= 4u + config.cpu_threads >= 4u ? generator::action::wavefront_stage_pool::static_worker_count : 0u; const size_t available_matmul_lanes = config.cpu_threads - stage_worker_count; @@ -632,9 +627,8 @@ int main(int argc, char **argv) { prediction_matmul_lanes.emplace(matmul_lane_count - 1u); } const size_t matmul_worker_count = - prediction_matmul_lanes - ? prediction_matmul_lanes->active_worker_count() - : 0u; + prediction_matmul_lanes ? prediction_matmul_lanes->active_worker_count() + : 0u; emel::kernel::matmul::execution_policy prediction_matmul_policy{ .parallel_matmul_lanes = prediction_matmul_lanes ? &prediction_matmul_lanes.value() : nullptr, @@ -684,17 +678,16 @@ int main(int argc, char **argv) { std::max(config.audio_top_k, config.text_top_k)), }, }}; - std::fprintf(stderr, - "EMEL_THREADS requested_total=%zu owner_threads=1 " - "stage_workers=%zu matmul_workers=%zu matmul_lanes=%zu " - "stage_mode=%s matmul_mode=%s\n", - config.cpu_threads, stage_worker_count, matmul_worker_count, - matmul_lane_count, - stage_worker_count == 0u ? "serial" : "parallel", - prediction_matmul_policy.mode == - emel::kernel::matmul::lane_mode::parallel - ? "parallel" - : "serial"); + std::fprintf( + stderr, + "EMEL_THREADS requested_total=%zu owner_threads=1 " + "stage_workers=%zu matmul_workers=%zu matmul_lanes=%zu " + "stage_mode=%s matmul_mode=%s\n", + config.cpu_threads, stage_worker_count, matmul_worker_count, + matmul_lane_count, stage_worker_count == 0u ? "serial" : "parallel", + prediction_matmul_policy.mode == emel::kernel::matmul::lane_mode::parallel + ? "parallel" + : "serial"); emel::memory::hybrid::sm prediction_memory{}; predictor::sm token_predictor{predictor::dependencies{ .memory = prediction_memory, @@ -837,7 +830,7 @@ int main(int argc, char **argv) { static_cast(session_err)); return 1; } - if (run_stage_pipeline) { + { session.reset(); personaplex_frame_dependencies middle_dependencies{ @@ -877,6 +870,25 @@ int main(int argc, char **argv) { generator::sm wavefront{wavefront_deps}; std::vector wavefront_pcm(frame_samples, 0.0f); size_t produced_frames = 0u; + const auto print_emel_input = [&](const uint64_t sequence) { + std::fprintf(stderr, "EMEL_INPUT frame=%llu codes=", + static_cast(sequence)); + for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { + std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", + input_codes[static_cast(codebook)]); + } + std::fprintf(stderr, "\n"); + }; + const auto print_emel_output = [&](const uint64_t sequence, + const int32_t text_token) { + std::fprintf(stderr, "EMEL_OUTPUT frame=%llu text=%d codes=", + static_cast(sequence), text_token); + for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { + std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", + output_codes[static_cast(codebook)]); + } + std::fprintf(stderr, "\n"); + }; std::fprintf(stderr, "EMEL_PHASE name=steady_begin implementation=canonical " "frames=%zu\n", @@ -900,6 +912,7 @@ int main(int argc, char **argv) { generator::event::wavefront_frame request{ std::span{pcm_frame}, std::span{wavefront_pcm}, + std::span{input_codes}, std::span{output_codes}, {.sequence = sequence, .source = 1u}, output_attribution, @@ -916,6 +929,7 @@ int main(int argc, char **argv) { static_cast(output_attribution.sequence)); return 1; } + print_emel_input(sequence); if (produced) { if (output_attribution.sequence != produced_frames || output_attribution.sequence >= config.target_output_frames || @@ -929,6 +943,7 @@ int main(int argc, char **argv) { produced_frames, sample_count); return 1; } + print_emel_output(output_attribution.sequence, text_token); std::copy(wavefront_pcm.begin(), wavefront_pcm.end(), output_pcm.begin() + static_cast( produced_frames * frame_samples)); @@ -975,6 +990,7 @@ int main(int argc, char **argv) { produced_frames, sample_count); return 1; } + print_emel_output(output_attribution.sequence, text_token); std::copy(wavefront_pcm.begin(), wavefront_pcm.end(), output_pcm.begin() + static_cast( produced_frames * frame_samples)); @@ -1018,92 +1034,6 @@ int main(int argc, char **argv) { static_cast(joins), static_cast(worker_entries), static_cast(worker_exits)); - } else { - for (size_t frame_index = 0; frame_index < input_frames; ++frame_index) { - std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); - const size_t begin = frame_index * frame_samples; - const size_t copy_count = - std::min(frame_samples, input_pcm.size() - begin); - std::copy_n(input_pcm.data() + begin, copy_count, pcm_frame.data()); - int32_t text_token = -1; - bool produced = false; - int32_t sample_count = 0; - session_err = emel::error::cast(generator::error::none); - generator::event::stream_frame frame{ - std::span{pcm_frame}, - std::span{output_pcm.data() + frame_index * frame_samples, - frame_samples}, - std::span{input_codes}, - std::span{output_codes}, - text_token, - sample_count, - produced, - session_err}; - if (!session->process_event(frame)) { - std::fprintf(stderr, "PersonaPlex stream frame=%zu failed err=%d\n", - frame_index, static_cast(session_err)); - return 1; - } - std::fprintf(stderr, "EMEL_INPUT frame=%zu codes=", frame_index); - for (int32_t index = 0; index < public_n_q; ++index) { - std::fprintf(stderr, "%s%d", index == 0 ? "" : ",", - input_codes[static_cast(index)]); - } - std::fprintf(stderr, - "\nEMEL_OUTPUT frame=%zu text=%d codes=", frame_index, - text_token); - for (int32_t index = 0; index < public_n_q; ++index) { - std::fprintf(stderr, "%s%d", index == 0 ? "" : ",", - output_codes[static_cast(index)]); - } - std::fprintf(stderr, "\n"); - } - - std::fill(pcm_frame.begin(), pcm_frame.end(), 0.0f); - const size_t flush_steps = config.target_output_frames - input_frames; - for (size_t index = 0; index < flush_steps; ++index) { - const size_t frame_index = input_frames + index; - int32_t text_token = -1; - int32_t sample_count = 0; - bool complete = false; - session_err = emel::error::cast(generator::error::none); - generator::event::flush frame{ - std::span{output_pcm.data() + frame_index * frame_samples, - frame_samples}, - std::span{input_codes}, - std::span{output_codes}, - text_token, - sample_count, - complete, - session_err}; - if (!session->process_event(frame)) { - std::fprintf(stderr, "PersonaPlex flush frame=%zu failed err=%d\n", - frame_index, static_cast(session_err)); - return 1; - } - std::fprintf(stderr, "EMEL_INPUT frame=%zu codes=", frame_index); - for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { - std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", - input_codes[static_cast(codebook)]); - } - std::fprintf(stderr, - "\nEMEL_OUTPUT frame=%zu text=%d codes=", frame_index, - text_token); - for (int32_t codebook = 0; codebook < public_n_q; ++codebook) { - std::fprintf(stderr, "%s%d", codebook == 0 ? "" : ",", - output_codes[static_cast(codebook)]); - } - std::fprintf(stderr, "\n"); - } - const bool session_finished = - flush_steps == 0 - ? session->is(stateforward::sml::state) - : session->is(stateforward::sml::state); - if (!session_finished) { - std::fprintf(stderr, "PersonaPlex session failed err=%d\n", - static_cast(session_err)); - return 1; - } } if (output_pcm.empty()) { From fb4ab721d6bb28072348c2854a4ad2821b692c7a Mon Sep 17 00:00:00 2001 From: gabewillen Date: Fri, 24 Jul 2026 01:25:34 -0500 Subject: [PATCH 4/6] fix(personaplex): resolve current-head review findings --- snapshots/quality_gates/timing.txt | 12 ++--- src/emel/kernel/matmul/guards.hpp | 16 ++++++- src/emel/speech/generator/guards.hpp | 5 +- tests/kernel/matmul_tests.cpp | 24 ++++++++++ .../generator/wavefront_lifecycle_tests.cpp | 25 +++++++++- .../predictor/moshi_lifecycle_tests.cpp | 11 +++++ tools/bench/speech/lm_moshi_bench.cpp | 46 ++++++++++++++++--- 7 files changed, 122 insertions(+), 17 deletions(-) diff --git a/snapshots/quality_gates/timing.txt b/snapshots/quality_gates/timing.txt index 3c5d14cc..6e41ef50 100644 --- a/snapshots/quality_gates/timing.txt +++ b/snapshots/quality_gates/timing.txt @@ -1,12 +1,12 @@ # quality_gates timing (seconds) domain_boundaries 1 legacy_sml_surface 0 -build_with_zig 9 -bench_snapshot 0 -test_with_coverage 1160 -paritychecker 0 +build_with_zig 25 +bench_snapshot 38 +test_with_coverage 69 +paritychecker 2 fuzz_smoke 0 -determinism_check 0 +determinism_check 2 lint_snapshot 12 generate_docs 0 -total 1182 +total 108 diff --git a/src/emel/kernel/matmul/guards.hpp b/src/emel/kernel/matmul/guards.hpp index e9e09219..571b48e1 100644 --- a/src/emel/kernel/matmul/guards.hpp +++ b/src/emel/kernel/matmul/guards.hpp @@ -18,6 +18,17 @@ inline lane_mode guard_policy_lane_mode(const size_t active_lanes) noexcept { : lane_mode::serial; } +inline size_t guard_auto_active_lane_count(const lane_pool &pool) noexcept { + const size_t active_workers = pool.active_worker_count(); + if (active_workers >= 7u) { + return 8u; + } + if (active_workers >= 3u) { + return 4u; + } + return 2u; +} + } // namespace emel::kernel::matmul::guard namespace emel::kernel::matmul { @@ -36,8 +47,9 @@ make_execution_policy(lane_pool ¶llel_matmul_lanes, inline execution_policy make_auto_execution_policy(lane_pool ¶llel_matmul_lanes) noexcept { - return make_execution_policy(parallel_matmul_lanes, - emel::kernel::detect_host_kind(), 8u); + return make_execution_policy( + parallel_matmul_lanes, emel::kernel::detect_host_kind(), + guard::guard_auto_active_lane_count(parallel_matmul_lanes)); } } // namespace emel::kernel::matmul diff --git a/src/emel/speech/generator/guards.hpp b/src/emel/speech/generator/guards.hpp index 05b0228a..b42ce2aa 100644 --- a/src/emel/speech/generator/guards.hpp +++ b/src/emel/speech/generator/guards.hpp @@ -297,7 +297,10 @@ template struct guard_wavefront_frame_valid { runtime_ev.request.input_attribution.sequence != event::k_invalid_wavefront_sequence && runtime_ev.request.input_attribution.sequence == - ctx.expected_input.sequence; + ctx.expected_input.sequence && + (ctx.expected_input.sequence == 0u || + runtime_ev.request.input_attribution.source == + ctx.expected_input.source); } }; diff --git a/tests/kernel/matmul_tests.cpp b/tests/kernel/matmul_tests.cpp index 6ce0efdb..93ad5f70 100644 --- a/tests/kernel/matmul_tests.cpp +++ b/tests/kernel/matmul_tests.cpp @@ -410,6 +410,30 @@ TEST_CASE("parallel matmul uses the concrete canonical lane pool") { CHECK(policy.active_lanes == 8u); } +TEST_CASE( + "parallel matmul auto policy stays within the runtime worker budget") { + { + matmul::lane_pool pool{1u}; + CHECK(matmul::make_auto_execution_policy(pool).active_lanes == 2u); + } + { + matmul::lane_pool pool{2u}; + CHECK(matmul::make_auto_execution_policy(pool).active_lanes == 2u); + } + { + matmul::lane_pool pool{3u}; + CHECK(matmul::make_auto_execution_policy(pool).active_lanes == 4u); + } + { + matmul::lane_pool pool{6u}; + CHECK(matmul::make_auto_execution_policy(pool).active_lanes == 4u); + } + { + matmul::lane_pool pool{7u}; + CHECK(matmul::make_auto_execution_policy(pool).active_lanes == 8u); + } +} + struct parallel_backend_fixture { emel::kernel::matmul::lane_pool parallel_matmul_lanes = {}; emel::kernel::matmul::execution_policy policy = diff --git a/tests/speech/generator/wavefront_lifecycle_tests.cpp b/tests/speech/generator/wavefront_lifecycle_tests.cpp index 9aef29ec..ec79693f 100644 --- a/tests/speech/generator/wavefront_lifecycle_tests.cpp +++ b/tests/speech/generator/wavefront_lifecycle_tests.cpp @@ -400,6 +400,17 @@ struct fixture { std::array &tokens_out, emel::error::type &err, emel::callback on_done = {}) { + return push_from_source(sequence, 99u, produced, output, pcm_out, + tokens_out, err, on_done); + } + + bool push_from_source( + const uint64_t sequence, const uint64_t source, bool &produced, + generator::event::wavefront_attribution &output, + std::array &pcm_out, + std::array &tokens_out, emel::error::type &err, + emel::callback + on_done = {}) { std::array pcm_in{}; pcm_in.fill(static_cast(sequence + 1u)); last_text_token = -1; @@ -409,7 +420,7 @@ struct fixture { pcm_out, last_encoded_tokens, tokens_out, - {.sequence = sequence, .source = 99u}, + {.sequence = sequence, .source = source}, output, last_text_token, last_sample_count, @@ -626,6 +637,18 @@ TEST_CASE("speech_generator_wavefront_emits_done_callbacks_for_each_public_" } } +TEST_CASE("speech_generator_wavefront_rejects_source_switch_without_reset") { + fixture test{}; + wavefront_io io{}; + + REQUIRE(test.push_from_source(0u, 99u, io.produced, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(test.push_from_source(1u, 100u, io.produced, io.output, io.pcm, + io.tokens, io.err)); + CHECK_FALSE(io.produced); + CHECK(io.err == error_code(generator::error::invalid_request)); +} + TEST_CASE("speech_generator_wavefront_rejects_each_invalid_public_request_" "shape") { const auto check_invalid_frame = diff --git a/tests/speech/predictor/moshi_lifecycle_tests.cpp b/tests/speech/predictor/moshi_lifecycle_tests.cpp index 1c3c39a0..98f7a220 100644 --- a/tests/speech/predictor/moshi_lifecycle_tests.cpp +++ b/tests/speech/predictor/moshi_lifecycle_tests.cpp @@ -1704,6 +1704,17 @@ TEST_CASE("speech_moshi_executor_runtime_choice_uses_explicit_transitions") { std::string::npos); } +TEST_CASE("speech Moshi attention benchmark binds temporal and depformer KV") { + const auto bytes = read_binary_file(repo_root() / "tools" / "bench" / + "speech" / "lm_moshi_bench.cpp"); + const std::string source{bytes.begin(), bytes.end()}; + + CHECK(source.find("depformer_layer_offsets") != std::string::npos); + CHECK(source.find("depformer_key_cache") != std::string::npos); + CHECK(source.find("depformer_value_cache") != std::string::npos); + CHECK(source.find(".depformer =") != std::string::npos); +} + TEST_CASE("speech_moshi_generator_graph_outputs_use_explicit_transitions") { const auto action_bytes = read_binary_file(repo_root() / "src" / "emel" / "speech" / "predictor" / diff --git a/tools/bench/speech/lm_moshi_bench.cpp b/tools/bench/speech/lm_moshi_bench.cpp index 2d95b186..f09c8df9 100644 --- a/tools/bench/speech/lm_moshi_bench.cpp +++ b/tools/bench/speech/lm_moshi_bench.cpp @@ -184,6 +184,13 @@ struct attention_executor_bench_fixture { static_cast(model.moshi_lm.context) * static_cast(model.moshi_lm.dim)), value_cache(key_cache.size()), + depformer_layer_offsets( + static_cast(model.moshi_lm.depformer_num_layers), 0u), + depformer_key_cache( + static_cast(model.moshi_lm.depformer_num_layers) * + static_cast(model.moshi_lm.depformer_context) * + static_cast(model.moshi_lm.depformer_dim)), + depformer_value_cache(depformer_key_cache.size()), input_sequence(static_cast(model.moshi_lm.n_q + 1), -1), input_embedding(static_cast(model.moshi_lm.dim)), temporal_state(input_embedding.size()), @@ -201,6 +208,13 @@ struct attention_executor_bench_fixture { for (std::size_t index = 0u; index < layer_offsets.size(); ++index) { layer_offsets[index] = index * per_layer_cache; } + const std::size_t depformer_per_layer_cache = + static_cast(model.moshi_lm.depformer_context) * + static_cast(model.moshi_lm.depformer_dim); + for (std::size_t index = 0u; index < depformer_layer_offsets.size(); + ++index) { + depformer_layer_offsets[index] = index * depformer_per_layer_cache; + } if (lane_count > 1u) { attention_lanes.emplace(); } @@ -226,6 +240,21 @@ struct attention_executor_bench_fixture { .position_capacity = model.moshi_lm.context, .kv_dim = model.moshi_lm.dim, }, + .depformer = + { + .key_cache = + std::span{depformer_key_cache}, + .value_cache = + std::span{depformer_value_cache}, + .layer_cache_offsets = + std::span{ + depformer_layer_offsets}, + .layer_count = + model.moshi_lm.depformer_num_layers, + .position_capacity = + model.moshi_lm.depformer_context, + .kv_dim = model.moshi_lm.depformer_dim, + }, .temporal_positions = &temporal_positions, .depformer_positions = &depformer_positions, }, @@ -246,18 +275,18 @@ struct attention_executor_bench_fixture { .capacity = { .hidden_dim = - static_cast(std::max(model.moshi_lm.dim, - model.moshi_lm.depformer_dim)), + static_cast( + std::max(model.moshi_lm.dim, + model.moshi_lm.depformer_dim)), .temporal_context = static_cast(model.moshi_lm.context), .depformer_context = - static_cast(model.moshi_lm - .depformer_context), + static_cast( + model.moshi_lm.depformer_context), .sampling_card = static_cast( - std:: - max(model.moshi_lm.text_card, model.moshi_lm - .card)), + std::max(model.moshi_lm.text_card, + model.moshi_lm.card)), .sampling_top_k = 1u, }, }); @@ -348,6 +377,9 @@ struct attention_executor_bench_fixture { std::vector layer_offsets = {}; std::vector key_cache = {}; std::vector value_cache = {}; + std::vector depformer_layer_offsets = {}; + std::vector depformer_key_cache = {}; + std::vector depformer_value_cache = {}; std::vector input_sequence = {}; std::vector input_embedding = {}; std::vector temporal_state = {}; From e4ce5d73c04364242cb573674cb6afb53c6ac4f0 Mon Sep 17 00:00:00 2001 From: gabewillen Date: Fri, 24 Jul 2026 02:19:29 -0500 Subject: [PATCH 5/6] fix(personaplex): resolve latest review findings --- .planning/architecture/kernel_attention.md | 48 ++++++++ .planning/architecture/kernel_f32_matvec.md | 108 ++++++++++++++++++ .planning/architecture/kernel_matmul.md | 94 +++++++++++++++ .planning/architecture/logits_sampler.md | 6 +- .../architecture/mermaid/kernel_attention.mmd | 19 +++ .../mermaid/kernel_f32_matvec.mmd | 49 ++++++++ .../architecture/mermaid/kernel_matmul.mmd | 42 +++++++ .../architecture/mermaid/logits_sampler.mmd | 3 +- .../mermaid/speech_codec_mimi.mmd | 22 +++- .../speech_predictor_moshi_executor.mmd | 43 ++++--- .../mermaid/speech_tokenizer_moshi.mmd | 10 +- .../mermaid/text_generator_matmul.mmd | 18 --- .planning/architecture/speech_codec_mimi.md | 44 +++++-- .../speech_predictor_moshi_executor.md | 86 +++++++++----- .../architecture/speech_tokenizer_moshi.md | 20 ++-- .../architecture/text_generator_matmul.md | 46 -------- scripts/bench_moshi_lm_compare.sh | 9 +- snapshots/quality_gates/timing.txt | 14 +-- .../predictor/moshi_lifecycle_tests.cpp | 3 + tools/bench/bench_runner_tests.cpp | 15 +++ tools/bench/speech/lm_moshi_bench.cpp | 2 +- 21 files changed, 554 insertions(+), 147 deletions(-) create mode 100644 .planning/architecture/kernel_attention.md create mode 100644 .planning/architecture/kernel_f32_matvec.md create mode 100644 .planning/architecture/kernel_matmul.md create mode 100644 .planning/architecture/mermaid/kernel_attention.mmd create mode 100644 .planning/architecture/mermaid/kernel_f32_matvec.mmd create mode 100644 .planning/architecture/mermaid/kernel_matmul.mmd delete mode 100644 .planning/architecture/mermaid/text_generator_matmul.mmd delete mode 100644 .planning/architecture/text_generator_matmul.md diff --git a/.planning/architecture/kernel_attention.md b/.planning/architecture/kernel_attention.md new file mode 100644 index 00000000..0e594a29 --- /dev/null +++ b/.planning/architecture/kernel_attention.md @@ -0,0 +1,48 @@ +# kernel_attention + +Source: [`emel/kernel/attention/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> state_ready + state_ready --> state_execute_result_decision : execute [guard_execute_supported_] / effect_execute_head_range_ + state_ready --> state_execute_error_callback_decision : execute [guard_execute_unsupported_] / effect_reject_execute_ + state_execute_result_decision --> state_execute_done_callback_decision : completion_execute_ [guard_execute_succeeded_] / effect_accept_execute_ + state_execute_result_decision --> state_execute_error_callback_decision : completion_execute_ [guard_execute_failed_] / effect_reject_execute_ + state_execute_done_callback_decision --> state_execute_done : completion_execute_ [guard_has_done_callback_] / effect_emit_execute_done_ + state_execute_done_callback_decision --> state_execute_done : completion_execute_ [guard_no_done_callback_] / none + state_execute_error_callback_decision --> state_execute_errored : completion_execute_ [guard_has_error_callback_] / effect_emit_execute_error_ + state_execute_error_callback_decision --> state_execute_errored : completion_execute_ [guard_no_error_callback_] / none + state_execute_done --> state_ready : completion_execute_ [always] / none + state_execute_errored --> state_ready : completion_execute_ [always] / none + state_ready --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_result_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_done_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_error_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_done --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_errored --> state_ready : _ [always] / effect_on_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`execute`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`guard_execute_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_execute_head_range>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_execute_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`execute`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`guard_execute_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_reject_execute>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_execute_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`guard_execute_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_accept_execute>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_execute_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`guard_execute_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_reject_execute>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_execute_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`guard_has_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_emit_execute_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_execute_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`guard_no_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_execute_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`guard_has_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_emit_execute_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_execute_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`guard_no_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_execute_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | +| [`state_execute_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/attention/sm.hpp) | diff --git a/.planning/architecture/kernel_f32_matvec.md b/.planning/architecture/kernel_f32_matvec.md new file mode 100644 index 00000000..7b1089b4 --- /dev/null +++ b/.planning/architecture/kernel_f32_matvec.md @@ -0,0 +1,108 @@ +# kernel_f32_matvec + +Source: [`emel/kernel/f32_matvec/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> state_ready + state_ready --> state_prepare_f32_result_decision : prepare_f32 [guard_prepare_f32_supported_] / effect_prepare_f32_ + state_ready --> state_error_callback_decision : prepare_f32 [guard_prepare_f32_unsupported_] / effect_reject_prepare_f32_ + state_prepare_f32_result_decision --> state_done_callback_decision : completion_prepare_f32_ [guard_prepare_f32_succeeded_] / effect_accept_prepare_f32_ + state_prepare_f32_result_decision --> state_error_callback_decision : completion_prepare_f32_ [guard_prepare_f32_failed_] / effect_reject_prepare_f32_ + state_ready --> state_prepare_f16_result_decision : prepare_f16 [guard_prepare_f16_supported_] / effect_prepare_f16_ + state_ready --> state_error_callback_decision : prepare_f16 [guard_prepare_f16_unsupported_] / effect_reject_prepare_f16_ + state_prepare_f16_result_decision --> state_done_callback_decision : completion_prepare_f16_ [guard_prepare_f16_succeeded_] / effect_accept_prepare_f16_ + state_prepare_f16_result_decision --> state_error_callback_decision : completion_prepare_f16_ [guard_prepare_f16_failed_] / effect_reject_prepare_f16_ + state_ready --> state_reference_result_decision : execute_reference [guard_execute_reference_supported_] / effect_execute_reference_ + state_ready --> state_error_callback_decision : execute_reference [guard_execute_reference_unsupported_] / effect_reject_execute_reference_ + state_reference_result_decision --> state_done_callback_decision : completion_execute_reference_ [guard_execute_reference_succeeded_] / effect_accept_execute_reference_ + state_reference_result_decision --> state_error_callback_decision : completion_execute_reference_ [guard_execute_reference_failed_] / effect_reject_execute_reference_ + state_ready --> state_exact_x4_result_decision : execute_exact_x4 [guard_execute_exact_x4_supported_] / effect_execute_exact_x4_ + state_ready --> state_error_callback_decision : execute_exact_x4 [guard_execute_exact_x4_unsupported_] / effect_reject_execute_exact_x4_ + state_exact_x4_result_decision --> state_done_callback_decision : completion_execute_exact_x4_ [guard_execute_exact_x4_succeeded_] / effect_accept_execute_exact_x4_ + state_exact_x4_result_decision --> state_error_callback_decision : completion_execute_exact_x4_ [guard_execute_exact_x4_failed_] / effect_reject_execute_exact_x4_ + state_done_callback_decision --> state_done : completion_prepare_f32_ [guard_prepare_f32_has_done_callback_] / effect_emit_prepare_f32_done_ + state_done_callback_decision --> state_done : completion_prepare_f32_ [guard_prepare_f32_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_prepare_f32_ [guard_prepare_f32_has_error_callback_] / effect_emit_prepare_f32_error_ + state_error_callback_decision --> state_errored : completion_prepare_f32_ [guard_prepare_f32_no_error_callback_] / none + state_done_callback_decision --> state_done : completion_prepare_f16_ [guard_prepare_f16_has_done_callback_] / effect_emit_prepare_f16_done_ + state_done_callback_decision --> state_done : completion_prepare_f16_ [guard_prepare_f16_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_prepare_f16_ [guard_prepare_f16_has_error_callback_] / effect_emit_prepare_f16_error_ + state_error_callback_decision --> state_errored : completion_prepare_f16_ [guard_prepare_f16_no_error_callback_] / none + state_done_callback_decision --> state_done : completion_execute_reference_ [guard_execute_reference_has_done_callback_] / effect_emit_execute_reference_done_ + state_done_callback_decision --> state_done : completion_execute_reference_ [guard_execute_reference_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_execute_reference_ [guard_execute_reference_has_error_callback_] / effect_emit_execute_reference_error_ + state_error_callback_decision --> state_errored : completion_execute_reference_ [guard_execute_reference_no_error_callback_] / none + state_done_callback_decision --> state_done : completion_execute_exact_x4_ [guard_execute_exact_x4_has_done_callback_] / effect_emit_execute_exact_x4_done_ + state_done_callback_decision --> state_done : completion_execute_exact_x4_ [guard_execute_exact_x4_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_execute_exact_x4_ [guard_execute_exact_x4_has_error_callback_] / effect_emit_execute_exact_x4_error_ + state_error_callback_decision --> state_errored : completion_execute_exact_x4_ [guard_execute_exact_x4_no_error_callback_] / none + state_done --> state_ready : completion_prepare_f32_ [always] / none + state_done --> state_ready : completion_prepare_f16_ [always] / none + state_done --> state_ready : completion_execute_reference_ [always] / none + state_done --> state_ready : completion_execute_exact_x4_ [always] / none + state_errored --> state_ready : completion_prepare_f32_ [always] / none + state_errored --> state_ready : completion_prepare_f16_ [always] / none + state_errored --> state_ready : completion_execute_reference_ [always] / none + state_errored --> state_ready : completion_execute_exact_x4_ [always] / none + state_ready --> state_ready : capture_diagnostics [always] / effect_capture_diagnostics_ + state_ready --> state_ready : _ [always] / effect_on_unexpected_ + state_done_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_error_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_done --> state_ready : _ [always] / effect_on_unexpected_ + state_errored --> state_ready : _ [always] / effect_on_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`prepare_f32`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f32_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_prepare_f32>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_prepare_f32_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`prepare_f32`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f32_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_reject_prepare_f32>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_prepare_f32_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f32_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_accept_prepare_f32>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_prepare_f32_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f32_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_reject_prepare_f32>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`prepare_f16`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f16_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_prepare_f16>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_prepare_f16_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`prepare_f16`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f16_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_reject_prepare_f16>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_prepare_f16_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f16_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_accept_prepare_f16>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_prepare_f16_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f16_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_reject_prepare_f16>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`execute_reference`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_reference_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_execute_reference>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_reference_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`execute_reference`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_reference_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_reject_execute_reference>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_reference_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_reference_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_accept_execute_reference>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_reference_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_reference_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_reject_execute_reference>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`execute_exact_x4`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_exact_x4_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_execute_exact_x4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_exact_x4_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`execute_exact_x4`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_exact_x4_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_reject_execute_exact_x4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_exact_x4_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_exact_x4_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_accept_execute_exact_x4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_exact_x4_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_exact_x4_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_reject_execute_exact_x4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f32_has_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_emit_prepare_f32_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f32_no_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f32_has_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_emit_prepare_f32_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f32_no_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f16_has_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_emit_prepare_f16_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f16_no_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f16_has_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_emit_prepare_f16_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_prepare_f16_no_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_reference_has_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_emit_execute_reference_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_reference_no_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_reference_has_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_emit_execute_reference_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_reference_no_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_exact_x4_has_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_emit_execute_exact_x4_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_exact_x4_no_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_exact_x4_has_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_emit_execute_exact_x4_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`guard_execute_exact_x4_no_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`capture_diagnostics`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_capture_diagnostics>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/f32_matvec/sm.hpp) | diff --git a/.planning/architecture/kernel_matmul.md b/.planning/architecture/kernel_matmul.md new file mode 100644 index 00000000..ebd0e970 --- /dev/null +++ b/.planning/architecture/kernel_matmul.md @@ -0,0 +1,94 @@ +# kernel_matmul + +Source: [`emel/kernel/matmul/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> state_ready + state_ready --> state_ready : configure_kernel_kind [always] / effect_configure_kernel_kind_ + state_ready --> state_serial_result_decision : execute_serial [always] / effect_execute_serial_ + state_serial_result_decision --> state_done_callback_decision : completion_execute_serial_ [guard_serial_accepted_] / effect_accept_serial_execution_ + state_serial_result_decision --> state_error_callback_decision : completion_execute_serial_ [guard_serial_rejected_] / effect_reject_serial_execution_ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_8__8__] / effect_execute_parallel_8__8__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_8__4__] / effect_execute_parallel_8__4__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_8__1__] / effect_execute_parallel_8__1__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_4__8__] / effect_execute_parallel_4__8__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_4__4__] / effect_execute_parallel_4__4__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_4__1__] / effect_execute_parallel_4__1__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_2__8__] / effect_execute_parallel_2__8__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_2__4__] / effect_execute_parallel_2__4__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_2__1__] / effect_execute_parallel_2__1__ + state_ready --> state_error_callback_decision : execute_parallel [guard_parallel_request_invalid_] / effect_reject_parallel_execution_ + state_ready --> state_error_callback_decision : execute_parallel [guard_parallel_unavailable_] / effect_reject_parallel_execution_ + state_ready --> state_error_callback_decision : execute_parallel [guard_parallel_no_lane_count_] / effect_reject_parallel_execution_ + state_parallel_result_decision --> state_error_callback_decision : completion_execute_parallel_ [guard_parallel_submission_failed_] / effect_reject_parallel_execution_ + state_parallel_result_decision --> state_error_callback_decision : completion_execute_parallel_ [guard_parallel_lane_rejected_] / effect_reject_parallel_execution_ + state_parallel_result_decision --> state_done_callback_decision : completion_execute_parallel_ [guard_parallel_all_lanes_accepted_] / effect_accept_parallel_execution_ + state_done_callback_decision --> state_done : completion_execute_serial_ [guard_serial_has_done_callback_] / effect_emit_serial_done_ + state_done_callback_decision --> state_done : completion_execute_serial_ [guard_serial_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_execute_serial_ [guard_serial_has_error_callback_] / effect_emit_serial_error_ + state_error_callback_decision --> state_errored : completion_execute_serial_ [guard_serial_no_error_callback_] / none + state_done_callback_decision --> state_done : completion_execute_parallel_ [guard_parallel_has_done_callback_] / effect_emit_parallel_done_ + state_done_callback_decision --> state_done : completion_execute_parallel_ [guard_parallel_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_execute_parallel_ [guard_parallel_has_error_callback_] / effect_emit_parallel_error_ + state_error_callback_decision --> state_errored : completion_execute_parallel_ [guard_parallel_no_error_callback_] / none + state_done --> state_ready : completion_execute_serial_ [always] / none + state_done --> state_ready : completion_execute_parallel_ [always] / none + state_errored --> state_ready : completion_execute_serial_ [always] / none + state_errored --> state_ready : completion_execute_parallel_ [always] / none + state_ready --> state_ready : capture_diagnostics [always] / effect_capture_diagnostics_ + state_ready --> state_ready : _ [always] / effect_on_unexpected_ + state_serial_result_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_parallel_result_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_done_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_error_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_done --> state_ready : _ [always] / effect_on_unexpected_ + state_errored --> state_ready : _ [always] / effect_on_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`configure_kernel_kind`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_configure_kernel_kind>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_serial`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_serial_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_serial_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_serial_accepted>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_accept_serial_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_serial_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_serial_rejected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_reject_serial_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<8, 8>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<8, 8>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<8, 4>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<8, 4>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<8, 1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<8, 1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<4, 8>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<4, 8>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<4, 4>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<4, 4>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<4, 1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<4, 1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<2, 8>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<2, 8>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<2, 4>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<2, 4>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_ready<2, 1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_execute_parallel<2, 1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_request_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_unavailable>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_no_lane_count>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_submission_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_lane_rejected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_all_lanes_accepted>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_accept_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_serial_has_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_emit_serial_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_serial_no_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_serial_has_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_emit_serial_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_serial_no_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_has_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_emit_parallel_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_no_done_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_has_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_emit_parallel_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`guard_parallel_no_error_callback>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`capture_diagnostics`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_capture_diagnostics>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_serial_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_done_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_error_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | +| [`state_errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/kernel/matmul/sm.hpp) | diff --git a/.planning/architecture/logits_sampler.md b/.planning/architecture/logits_sampler.md index c887550c..731f9ab8 100644 --- a/.planning/architecture/logits_sampler.md +++ b/.planning/architecture/logits_sampler.md @@ -33,7 +33,8 @@ stateDiagram-v2 state_temperature_top_k_scale --> state_temperature_top_k_probabilities : completion_sample_temperature_top_k_runtime_ [always] / compute_temperature_probabilities_ state_temperature_top_k_probabilities --> state_temperature_top_k_rank : completion_sample_temperature_top_k_runtime_ [always] / compute_temperature_top_k_ state_temperature_top_k_rank --> state_temperature_top_k_select : completion_sample_temperature_top_k_runtime_ [always] / select_temperature_top_k_ - state_temperature_top_k_select --> done : completion_sample_temperature_top_k_runtime_ [always] / none + state_temperature_top_k_select --> done : completion_sample_temperature_top_k_runtime_ [temperature_top_k_selected_token_valid_] / none + state_temperature_top_k_select --> errored : completion_sample_temperature_top_k_runtime_ [temperature_top_k_selected_token_invalid_] / mark_invalid_request_ done --> ready : completion_configure_runtime_ [always] / publish_done_ errored --> ready : completion_configure_runtime_ [always] / publish_error_ done --> ready : completion_sample_logits_runtime_ [always] / publish_done_ @@ -89,7 +90,8 @@ stateDiagram-v2 | [`state_temperature_top_k_scale`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`compute_temperature_probabilities>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`state_temperature_top_k_probabilities`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`state_temperature_top_k_probabilities`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`compute_temperature_top_k>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`state_temperature_top_k_rank`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`state_temperature_top_k_rank`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`select_temperature_top_k>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`state_temperature_top_k_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | -| [`state_temperature_top_k_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`temperature_top_k_selected_token_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | +| [`state_temperature_top_k_select`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`temperature_top_k_selected_token_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`mark_invalid_request>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`publish_done>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | [`ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/logits/sampler/sm.hpp) | diff --git a/.planning/architecture/mermaid/kernel_attention.mmd b/.planning/architecture/mermaid/kernel_attention.mmd new file mode 100644 index 00000000..444d07af --- /dev/null +++ b/.planning/architecture/mermaid/kernel_attention.mmd @@ -0,0 +1,19 @@ +stateDiagram-v2 + direction TB + [*] --> state_ready + state_ready --> state_execute_result_decision : execute [guard_execute_supported_] / effect_execute_head_range_ + state_ready --> state_execute_error_callback_decision : execute [guard_execute_unsupported_] / effect_reject_execute_ + state_execute_result_decision --> state_execute_done_callback_decision : completion_execute_ [guard_execute_succeeded_] / effect_accept_execute_ + state_execute_result_decision --> state_execute_error_callback_decision : completion_execute_ [guard_execute_failed_] / effect_reject_execute_ + state_execute_done_callback_decision --> state_execute_done : completion_execute_ [guard_has_done_callback_] / effect_emit_execute_done_ + state_execute_done_callback_decision --> state_execute_done : completion_execute_ [guard_no_done_callback_] / none + state_execute_error_callback_decision --> state_execute_errored : completion_execute_ [guard_has_error_callback_] / effect_emit_execute_error_ + state_execute_error_callback_decision --> state_execute_errored : completion_execute_ [guard_no_error_callback_] / none + state_execute_done --> state_ready : completion_execute_ [always] / none + state_execute_errored --> state_ready : completion_execute_ [always] / none + state_ready --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_result_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_done_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_error_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_done --> state_ready : _ [always] / effect_on_unexpected_ + state_execute_errored --> state_ready : _ [always] / effect_on_unexpected_ diff --git a/.planning/architecture/mermaid/kernel_f32_matvec.mmd b/.planning/architecture/mermaid/kernel_f32_matvec.mmd new file mode 100644 index 00000000..8ae54d72 --- /dev/null +++ b/.planning/architecture/mermaid/kernel_f32_matvec.mmd @@ -0,0 +1,49 @@ +stateDiagram-v2 + direction TB + [*] --> state_ready + state_ready --> state_prepare_f32_result_decision : prepare_f32 [guard_prepare_f32_supported_] / effect_prepare_f32_ + state_ready --> state_error_callback_decision : prepare_f32 [guard_prepare_f32_unsupported_] / effect_reject_prepare_f32_ + state_prepare_f32_result_decision --> state_done_callback_decision : completion_prepare_f32_ [guard_prepare_f32_succeeded_] / effect_accept_prepare_f32_ + state_prepare_f32_result_decision --> state_error_callback_decision : completion_prepare_f32_ [guard_prepare_f32_failed_] / effect_reject_prepare_f32_ + state_ready --> state_prepare_f16_result_decision : prepare_f16 [guard_prepare_f16_supported_] / effect_prepare_f16_ + state_ready --> state_error_callback_decision : prepare_f16 [guard_prepare_f16_unsupported_] / effect_reject_prepare_f16_ + state_prepare_f16_result_decision --> state_done_callback_decision : completion_prepare_f16_ [guard_prepare_f16_succeeded_] / effect_accept_prepare_f16_ + state_prepare_f16_result_decision --> state_error_callback_decision : completion_prepare_f16_ [guard_prepare_f16_failed_] / effect_reject_prepare_f16_ + state_ready --> state_reference_result_decision : execute_reference [guard_execute_reference_supported_] / effect_execute_reference_ + state_ready --> state_error_callback_decision : execute_reference [guard_execute_reference_unsupported_] / effect_reject_execute_reference_ + state_reference_result_decision --> state_done_callback_decision : completion_execute_reference_ [guard_execute_reference_succeeded_] / effect_accept_execute_reference_ + state_reference_result_decision --> state_error_callback_decision : completion_execute_reference_ [guard_execute_reference_failed_] / effect_reject_execute_reference_ + state_ready --> state_exact_x4_result_decision : execute_exact_x4 [guard_execute_exact_x4_supported_] / effect_execute_exact_x4_ + state_ready --> state_error_callback_decision : execute_exact_x4 [guard_execute_exact_x4_unsupported_] / effect_reject_execute_exact_x4_ + state_exact_x4_result_decision --> state_done_callback_decision : completion_execute_exact_x4_ [guard_execute_exact_x4_succeeded_] / effect_accept_execute_exact_x4_ + state_exact_x4_result_decision --> state_error_callback_decision : completion_execute_exact_x4_ [guard_execute_exact_x4_failed_] / effect_reject_execute_exact_x4_ + state_done_callback_decision --> state_done : completion_prepare_f32_ [guard_prepare_f32_has_done_callback_] / effect_emit_prepare_f32_done_ + state_done_callback_decision --> state_done : completion_prepare_f32_ [guard_prepare_f32_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_prepare_f32_ [guard_prepare_f32_has_error_callback_] / effect_emit_prepare_f32_error_ + state_error_callback_decision --> state_errored : completion_prepare_f32_ [guard_prepare_f32_no_error_callback_] / none + state_done_callback_decision --> state_done : completion_prepare_f16_ [guard_prepare_f16_has_done_callback_] / effect_emit_prepare_f16_done_ + state_done_callback_decision --> state_done : completion_prepare_f16_ [guard_prepare_f16_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_prepare_f16_ [guard_prepare_f16_has_error_callback_] / effect_emit_prepare_f16_error_ + state_error_callback_decision --> state_errored : completion_prepare_f16_ [guard_prepare_f16_no_error_callback_] / none + state_done_callback_decision --> state_done : completion_execute_reference_ [guard_execute_reference_has_done_callback_] / effect_emit_execute_reference_done_ + state_done_callback_decision --> state_done : completion_execute_reference_ [guard_execute_reference_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_execute_reference_ [guard_execute_reference_has_error_callback_] / effect_emit_execute_reference_error_ + state_error_callback_decision --> state_errored : completion_execute_reference_ [guard_execute_reference_no_error_callback_] / none + state_done_callback_decision --> state_done : completion_execute_exact_x4_ [guard_execute_exact_x4_has_done_callback_] / effect_emit_execute_exact_x4_done_ + state_done_callback_decision --> state_done : completion_execute_exact_x4_ [guard_execute_exact_x4_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_execute_exact_x4_ [guard_execute_exact_x4_has_error_callback_] / effect_emit_execute_exact_x4_error_ + state_error_callback_decision --> state_errored : completion_execute_exact_x4_ [guard_execute_exact_x4_no_error_callback_] / none + state_done --> state_ready : completion_prepare_f32_ [always] / none + state_done --> state_ready : completion_prepare_f16_ [always] / none + state_done --> state_ready : completion_execute_reference_ [always] / none + state_done --> state_ready : completion_execute_exact_x4_ [always] / none + state_errored --> state_ready : completion_prepare_f32_ [always] / none + state_errored --> state_ready : completion_prepare_f16_ [always] / none + state_errored --> state_ready : completion_execute_reference_ [always] / none + state_errored --> state_ready : completion_execute_exact_x4_ [always] / none + state_ready --> state_ready : capture_diagnostics [always] / effect_capture_diagnostics_ + state_ready --> state_ready : _ [always] / effect_on_unexpected_ + state_done_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_error_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_done --> state_ready : _ [always] / effect_on_unexpected_ + state_errored --> state_ready : _ [always] / effect_on_unexpected_ diff --git a/.planning/architecture/mermaid/kernel_matmul.mmd b/.planning/architecture/mermaid/kernel_matmul.mmd new file mode 100644 index 00000000..cea1a515 --- /dev/null +++ b/.planning/architecture/mermaid/kernel_matmul.mmd @@ -0,0 +1,42 @@ +stateDiagram-v2 + direction TB + [*] --> state_ready + state_ready --> state_ready : configure_kernel_kind [always] / effect_configure_kernel_kind_ + state_ready --> state_serial_result_decision : execute_serial [always] / effect_execute_serial_ + state_serial_result_decision --> state_done_callback_decision : completion_execute_serial_ [guard_serial_accepted_] / effect_accept_serial_execution_ + state_serial_result_decision --> state_error_callback_decision : completion_execute_serial_ [guard_serial_rejected_] / effect_reject_serial_execution_ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_8__8__] / effect_execute_parallel_8__8__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_8__4__] / effect_execute_parallel_8__4__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_8__1__] / effect_execute_parallel_8__1__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_4__8__] / effect_execute_parallel_4__8__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_4__4__] / effect_execute_parallel_4__4__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_4__1__] / effect_execute_parallel_4__1__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_2__8__] / effect_execute_parallel_2__8__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_2__4__] / effect_execute_parallel_2__4__ + state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_ready_2__1__] / effect_execute_parallel_2__1__ + state_ready --> state_error_callback_decision : execute_parallel [guard_parallel_request_invalid_] / effect_reject_parallel_execution_ + state_ready --> state_error_callback_decision : execute_parallel [guard_parallel_unavailable_] / effect_reject_parallel_execution_ + state_ready --> state_error_callback_decision : execute_parallel [guard_parallel_no_lane_count_] / effect_reject_parallel_execution_ + state_parallel_result_decision --> state_error_callback_decision : completion_execute_parallel_ [guard_parallel_submission_failed_] / effect_reject_parallel_execution_ + state_parallel_result_decision --> state_error_callback_decision : completion_execute_parallel_ [guard_parallel_lane_rejected_] / effect_reject_parallel_execution_ + state_parallel_result_decision --> state_done_callback_decision : completion_execute_parallel_ [guard_parallel_all_lanes_accepted_] / effect_accept_parallel_execution_ + state_done_callback_decision --> state_done : completion_execute_serial_ [guard_serial_has_done_callback_] / effect_emit_serial_done_ + state_done_callback_decision --> state_done : completion_execute_serial_ [guard_serial_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_execute_serial_ [guard_serial_has_error_callback_] / effect_emit_serial_error_ + state_error_callback_decision --> state_errored : completion_execute_serial_ [guard_serial_no_error_callback_] / none + state_done_callback_decision --> state_done : completion_execute_parallel_ [guard_parallel_has_done_callback_] / effect_emit_parallel_done_ + state_done_callback_decision --> state_done : completion_execute_parallel_ [guard_parallel_no_done_callback_] / none + state_error_callback_decision --> state_errored : completion_execute_parallel_ [guard_parallel_has_error_callback_] / effect_emit_parallel_error_ + state_error_callback_decision --> state_errored : completion_execute_parallel_ [guard_parallel_no_error_callback_] / none + state_done --> state_ready : completion_execute_serial_ [always] / none + state_done --> state_ready : completion_execute_parallel_ [always] / none + state_errored --> state_ready : completion_execute_serial_ [always] / none + state_errored --> state_ready : completion_execute_parallel_ [always] / none + state_ready --> state_ready : capture_diagnostics [always] / effect_capture_diagnostics_ + state_ready --> state_ready : _ [always] / effect_on_unexpected_ + state_serial_result_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_parallel_result_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_done_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_error_callback_decision --> state_ready : _ [always] / effect_on_unexpected_ + state_done --> state_ready : _ [always] / effect_on_unexpected_ + state_errored --> state_ready : _ [always] / effect_on_unexpected_ diff --git a/.planning/architecture/mermaid/logits_sampler.mmd b/.planning/architecture/mermaid/logits_sampler.mmd index 5c230682..ad3ad6f3 100644 --- a/.planning/architecture/mermaid/logits_sampler.mmd +++ b/.planning/architecture/mermaid/logits_sampler.mmd @@ -26,7 +26,8 @@ stateDiagram-v2 state_temperature_top_k_scale --> state_temperature_top_k_probabilities : completion_sample_temperature_top_k_runtime_ [always] / compute_temperature_probabilities_ state_temperature_top_k_probabilities --> state_temperature_top_k_rank : completion_sample_temperature_top_k_runtime_ [always] / compute_temperature_top_k_ state_temperature_top_k_rank --> state_temperature_top_k_select : completion_sample_temperature_top_k_runtime_ [always] / select_temperature_top_k_ - state_temperature_top_k_select --> done : completion_sample_temperature_top_k_runtime_ [always] / none + state_temperature_top_k_select --> done : completion_sample_temperature_top_k_runtime_ [temperature_top_k_selected_token_valid_] / none + state_temperature_top_k_select --> errored : completion_sample_temperature_top_k_runtime_ [temperature_top_k_selected_token_invalid_] / mark_invalid_request_ done --> ready : completion_configure_runtime_ [always] / publish_done_ errored --> ready : completion_configure_runtime_ [always] / publish_error_ done --> ready : completion_sample_logits_runtime_ [always] / publish_done_ diff --git a/.planning/architecture/mermaid/speech_codec_mimi.mmd b/.planning/architecture/mermaid/speech_codec_mimi.mmd index 1c83cd79..6c15c83b 100644 --- a/.planning/architecture/mermaid/speech_codec_mimi.mmd +++ b/.planning/architecture/mermaid/speech_codec_mimi.mmd @@ -2,10 +2,16 @@ stateDiagram-v2 direction TB [*] --> state_uninitialized state_uninitialized --> state_bind_contract_decision : initialize_run [always] / none - state_bind_contract_decision --> state_bind_capacity_decision : completion_initialize_run_ [guard_bind_contract_valid_] / none + state_bind_contract_decision --> state_bind_f32_capacity_decision : completion_initialize_run_ [guard_bind_f32_contract_valid_] / none + state_bind_contract_decision --> state_bind_native_capacity_decision : completion_initialize_run_ [guard_bind_native_contract_valid_] / none + state_bind_contract_decision --> state_bind_q8_capacity_decision : completion_initialize_run_ [guard_bind_q8_contract_valid_] / none state_bind_contract_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_bind_contract_invalid_] / effect_mark_bind_failed_ - state_bind_capacity_decision --> state_binding : completion_initialize_run_ [guard_arena_capacity_valid_] / effect_bind_ - state_bind_capacity_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_arena_capacity_invalid_] / effect_mark_arena_capacity_invalid_ + state_bind_f32_capacity_decision --> state_binding : completion_initialize_run_ [guard_arena_capacity_valid_] / effect_bind_f32_ + state_bind_f32_capacity_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_arena_capacity_invalid_] / effect_mark_arena_capacity_invalid_ + state_bind_native_capacity_decision --> state_binding : completion_initialize_run_ [guard_arena_capacity_valid_] / effect_bind_native_ + state_bind_native_capacity_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_arena_capacity_invalid_] / effect_mark_arena_capacity_invalid_ + state_bind_q8_capacity_decision --> state_binding : completion_initialize_run_ [guard_arena_capacity_valid_] / effect_bind_q8_ + state_bind_q8_capacity_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_arena_capacity_invalid_] / effect_mark_arena_capacity_invalid_ state_binding --> state_init_error_out_decision : completion_initialize_run_ [always] / none state_init_error_out_decision --> state_init_callback_decision : completion_initialize_run_ [initialize_run__] / initialize_run__ state_init_error_out_decision --> state_init_callback_decision : completion_initialize_run_ [initialize_run__] / none @@ -44,6 +50,8 @@ stateDiagram-v2 state_decode_failed_callback_decision --> state_session_ready : completion_decode_frame_run_ [decode_frame_run__] / effect_emit_decode_error_ state_decode_failed_callback_decision --> state_session_ready : completion_decode_frame_run_ [decode_frame_run__] / none state_session_ready --> state_session_ready : reset_stream_run [always] / effect_reset_stream_ + state_uninitialized --> state_uninitialized : capture_diagnostics [always] / effect_capture_diagnostics_ + state_session_ready --> state_session_ready : capture_diagnostics [always] / effect_capture_diagnostics_ state_uninitialized --> state_uninit_encode_error_out_decision : encode_frame_run [always] / encode_frame_run__ state_uninit_encode_error_out_decision --> state_uninit_encode_callback_decision : completion_encode_frame_run_ [encode_frame_run__] / encode_frame_run__ state_uninit_encode_error_out_decision --> state_uninit_encode_callback_decision : completion_encode_frame_run_ [encode_frame_run__] / none @@ -90,8 +98,12 @@ stateDiagram-v2 state_decode_failed_callback_decision --> state_session_ready : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ state_bind_contract_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ state_bind_contract_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ - state_bind_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ - state_bind_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_bind_f32_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_bind_f32_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_bind_native_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_bind_native_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_bind_q8_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_bind_q8_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ state_binding --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ state_binding --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ state_init_error_out_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ diff --git a/.planning/architecture/mermaid/speech_predictor_moshi_executor.mmd b/.planning/architecture/mermaid/speech_predictor_moshi_executor.mmd index 4da0f26d..7294561f 100644 --- a/.planning/architecture/mermaid/speech_predictor_moshi_executor.mmd +++ b/.planning/architecture/mermaid/speech_predictor_moshi_executor.mmd @@ -77,7 +77,8 @@ stateDiagram-v2 state_temporal_layer_norm_result_decision --> state_temporal_layer_projection : completion_step_run_ [guard_temporal_layer_projection_supported_] / effect_bind_temporal_layer_projection_ state_temporal_layer_norm_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_projection_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_projection --> state_temporal_layer_projection_bind_result_decision : completion_step_run_ [always] / none - state_temporal_layer_projection_bind_result_decision --> state_temporal_layer_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_projection_ + state_temporal_layer_projection_bind_result_decision --> state_temporal_layer_projection_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_temporal_layer_projection_serial_ + state_temporal_layer_projection_bind_result_decision --> state_temporal_layer_projection_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_temporal_layer_projection_parallel_ state_temporal_layer_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_projection_run --> state_temporal_layer_projection_result_decision : completion_step_run_ [guard_temporal_layer_projection_succeeded_] / none state_temporal_layer_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_projection_failed_] / effect_mark_graph_execution_unsupported_ @@ -92,14 +93,19 @@ stateDiagram-v2 state_temporal_layer_key_rope_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_cache_write_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_cache_write --> state_temporal_layer_cache_write_result_decision : completion_step_run_ [guard_temporal_layer_cache_write_succeeded_] / none state_temporal_layer_cache_write --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_cache_write_failed_] / effect_mark_graph_execution_unsupported_ - state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_supported_] / effect_run_temporal_layer_attention_ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_route_1__] / effect_run_temporal_layer_attention_1__ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_route_2__] / effect_run_temporal_layer_attention_2__ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_route_4__] / effect_run_temporal_layer_attention_4__ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_route_8__] / effect_run_temporal_layer_attention_8__ state_temporal_layer_cache_write_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_cache_write_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_route_unavailable_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_attention --> state_temporal_layer_attention_result_decision : completion_step_run_ [guard_temporal_layer_attention_succeeded_] / none state_temporal_layer_attention --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_attention_result_decision --> state_temporal_layer_out_projection : completion_step_run_ [guard_temporal_layer_out_projection_supported_] / effect_bind_temporal_layer_out_projection_ state_temporal_layer_attention_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_out_projection_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_out_projection --> state_temporal_layer_out_projection_bind_result_decision : completion_step_run_ [always] / none - state_temporal_layer_out_projection_bind_result_decision --> state_temporal_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_out_projection_ + state_temporal_layer_out_projection_bind_result_decision --> state_temporal_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_temporal_layer_out_projection_serial_ + state_temporal_layer_out_projection_bind_result_decision --> state_temporal_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_temporal_layer_out_projection_parallel_ state_temporal_layer_out_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_out_projection_run --> state_temporal_layer_out_projection_result_decision : completion_step_run_ [guard_temporal_layer_out_projection_succeeded_] / none state_temporal_layer_out_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_out_projection_failed_] / effect_mark_graph_execution_unsupported_ @@ -116,7 +122,8 @@ stateDiagram-v2 state_temporal_layer_norm2_result_decision --> state_temporal_layer_gating_in : completion_step_run_ [guard_temporal_layer_gating_in_supported_] / effect_bind_temporal_layer_gating_in_ state_temporal_layer_norm2_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_in_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_gating_in --> state_temporal_layer_gating_in_bind_result_decision : completion_step_run_ [always] / none - state_temporal_layer_gating_in_bind_result_decision --> state_temporal_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_gating_in_ + state_temporal_layer_gating_in_bind_result_decision --> state_temporal_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_temporal_layer_gating_in_serial_ + state_temporal_layer_gating_in_bind_result_decision --> state_temporal_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_temporal_layer_gating_in_parallel_ state_temporal_layer_gating_in_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_gating_in_run --> state_temporal_layer_gating_in_result_decision : completion_step_run_ [guard_temporal_layer_gating_in_succeeded_] / none state_temporal_layer_gating_in_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_in_failed_] / effect_mark_graph_execution_unsupported_ @@ -130,7 +137,8 @@ stateDiagram-v2 state_temporal_layer_silu_gate_result_decision --> state_temporal_layer_gating_out : completion_step_run_ [guard_temporal_layer_gating_out_supported_] / effect_bind_temporal_layer_gating_out_ state_temporal_layer_silu_gate_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_out_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_gating_out --> state_temporal_layer_gating_out_bind_result_decision : completion_step_run_ [always] / none - state_temporal_layer_gating_out_bind_result_decision --> state_temporal_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_gating_out_ + state_temporal_layer_gating_out_bind_result_decision --> state_temporal_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_temporal_layer_gating_out_serial_ + state_temporal_layer_gating_out_bind_result_decision --> state_temporal_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_temporal_layer_gating_out_parallel_ state_temporal_layer_gating_out_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_gating_out_run --> state_temporal_layer_gating_out_result_decision : completion_step_run_ [guard_temporal_layer_gating_out_succeeded_] / none state_temporal_layer_gating_out_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_out_failed_] / effect_mark_graph_execution_unsupported_ @@ -154,11 +162,14 @@ stateDiagram-v2 state_sampling_ready --> state_text_logits : step_run [guard_sampling_step_valid_] / effect_restore_prediction_state_and_bind_text_logits_ state_sampling_ready --> state_step_error_out_decision : step_run [guard_sampling_step_invalid_] / effect_mark_request_shape_ state_text_logits --> state_text_logits_bind_result_decision : completion_step_run_ [always] / none - state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_forced_text_token_valid_and_sampling_consume_] / effect_compute_text_token_logits_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_forced_text_token_sampling_serial_] / effect_compute_text_token_logits_serial_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_forced_text_token_sampling_parallel_] / effect_compute_text_token_logits_parallel_ state_text_logits_bind_result_decision --> state_text_logits_result_decision : completion_step_run_ [guard_forced_text_token_valid_without_sampling_consume_] / effect_publish_forced_text_token_ state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_forced_text_sampling_config_invalid_] / effect_mark_graph_execution_unsupported_ + state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_forced_text_token_invalid_] / effect_mark_request_shape_ state_text_logits_bind_result_decision --> state_text_logits_run : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_argmax_] / effect_select_text_token_ - state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_sampling_] / effect_compute_text_token_logits_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_text_logits_sampling_serial_] / effect_compute_text_token_logits_serial_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_text_logits_sampling_parallel_] / effect_compute_text_token_logits_parallel_ state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid_] / effect_mark_graph_execution_unsupported_ state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_text_logits_sample_projection --> state_text_logits_sample_projection_result_decision : completion_step_run_ [always] / none @@ -185,7 +196,8 @@ stateDiagram-v2 state_depformer_weight_decision --> state_depformer_input : completion_step_run_ [guard_depformer_audio_input_supported_] / effect_bind_depformer_audio_input_projection_ state_depformer_weight_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_input --> state_depformer_input_projection_bind_result_decision : completion_step_run_ [always] / none - state_depformer_input_projection_bind_result_decision --> state_depformer_input_projection : completion_step_run_ [guard_depformer_input_projection_bound_] / effect_run_depformer_input_projection_ + state_depformer_input_projection_bind_result_decision --> state_depformer_input_projection : completion_step_run_ [guard_depformer_input_projection_serial_] / effect_run_depformer_input_projection_serial_ + state_depformer_input_projection_bind_result_decision --> state_depformer_input_projection : completion_step_run_ [guard_depformer_input_projection_parallel_] / effect_run_depformer_input_projection_parallel_ state_depformer_input_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_projection_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_input_projection --> state_depformer_input_projection_result_decision : completion_step_run_ [always] / none state_depformer_input_projection_result_decision --> state_depformer_input_embedding_bind : completion_step_run_ [guard_depformer_text_input_projection_succeeded_] / effect_bind_depformer_text_input_embedding_ @@ -210,7 +222,8 @@ stateDiagram-v2 state_depformer_layer_norm_result_decision --> state_depformer_layer_projection : completion_step_run_ [guard_depformer_layer_projection_supported_] / effect_bind_depformer_layer_projection_ state_depformer_layer_norm_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_projection_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_projection --> state_depformer_layer_projection_bind_result_decision : completion_step_run_ [always] / none - state_depformer_layer_projection_bind_result_decision --> state_depformer_layer_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_projection_ + state_depformer_layer_projection_bind_result_decision --> state_depformer_layer_projection_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_depformer_layer_projection_serial_ + state_depformer_layer_projection_bind_result_decision --> state_depformer_layer_projection_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_depformer_layer_projection_parallel_ state_depformer_layer_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_projection_run --> state_depformer_layer_projection_result_decision : completion_step_run_ [guard_depformer_layer_projection_succeeded_] / none state_depformer_layer_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_projection_failed_] / effect_mark_graph_execution_unsupported_ @@ -225,7 +238,8 @@ stateDiagram-v2 state_depformer_layer_attention_result_decision --> state_depformer_layer_out_projection : completion_step_run_ [guard_depformer_layer_out_projection_supported_] / effect_bind_depformer_layer_out_projection_ state_depformer_layer_attention_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_out_projection_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_out_projection --> state_depformer_layer_out_projection_bind_result_decision : completion_step_run_ [always] / none - state_depformer_layer_out_projection_bind_result_decision --> state_depformer_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_out_projection_ + state_depformer_layer_out_projection_bind_result_decision --> state_depformer_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_depformer_layer_out_projection_serial_ + state_depformer_layer_out_projection_bind_result_decision --> state_depformer_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_depformer_layer_out_projection_parallel_ state_depformer_layer_out_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_out_projection_run --> state_depformer_layer_out_projection_result_decision : completion_step_run_ [guard_depformer_layer_out_projection_succeeded_] / none state_depformer_layer_out_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_out_projection_failed_] / effect_mark_graph_execution_unsupported_ @@ -242,7 +256,8 @@ stateDiagram-v2 state_depformer_layer_norm2_result_decision --> state_depformer_layer_gating_in : completion_step_run_ [guard_depformer_layer_gating_in_supported_] / effect_bind_depformer_layer_gating_in_ state_depformer_layer_norm2_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_in_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_gating_in --> state_depformer_layer_gating_in_bind_result_decision : completion_step_run_ [always] / none - state_depformer_layer_gating_in_bind_result_decision --> state_depformer_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_gating_in_ + state_depformer_layer_gating_in_bind_result_decision --> state_depformer_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_depformer_layer_gating_in_serial_ + state_depformer_layer_gating_in_bind_result_decision --> state_depformer_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_depformer_layer_gating_in_parallel_ state_depformer_layer_gating_in_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_gating_in_run --> state_depformer_layer_gating_in_result_decision : completion_step_run_ [guard_depformer_layer_gating_in_succeeded_] / none state_depformer_layer_gating_in_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_in_failed_] / effect_mark_graph_execution_unsupported_ @@ -256,7 +271,8 @@ stateDiagram-v2 state_depformer_layer_silu_gate_result_decision --> state_depformer_layer_gating_out : completion_step_run_ [guard_depformer_layer_gating_out_supported_] / effect_bind_depformer_layer_gating_out_ state_depformer_layer_silu_gate_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_out_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_gating_out --> state_depformer_layer_gating_out_bind_result_decision : completion_step_run_ [always] / none - state_depformer_layer_gating_out_bind_result_decision --> state_depformer_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_gating_out_ + state_depformer_layer_gating_out_bind_result_decision --> state_depformer_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_depformer_layer_gating_out_serial_ + state_depformer_layer_gating_out_bind_result_decision --> state_depformer_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_depformer_layer_gating_out_parallel_ state_depformer_layer_gating_out_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_gating_out_run --> state_depformer_layer_gating_out_result_decision : completion_step_run_ [guard_depformer_layer_gating_out_succeeded_] / none state_depformer_layer_gating_out_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_out_failed_] / effect_mark_graph_execution_unsupported_ @@ -268,7 +284,8 @@ stateDiagram-v2 state_depformer_layer_ff_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_logits --> state_depformer_logits_bind_result_decision : completion_step_run_ [always] / none state_depformer_logits_bind_result_decision --> state_depformer_logits_run : completion_step_run_ [guard_depformer_logits_projection_bound_argmax_] / effect_select_depformer_token_ - state_depformer_logits_bind_result_decision --> state_depformer_logits_sample_projection : completion_step_run_ [guard_depformer_logits_projection_bound_sampling_] / effect_compute_depformer_token_logits_ + state_depformer_logits_bind_result_decision --> state_depformer_logits_sample_projection : completion_step_run_ [guard_depformer_logits_sampling_serial_] / effect_compute_depformer_token_logits_serial_ + state_depformer_logits_bind_result_decision --> state_depformer_logits_sample_projection : completion_step_run_ [guard_depformer_logits_sampling_parallel_] / effect_compute_depformer_token_logits_parallel_ state_depformer_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_projection_bound_sampling_invalid_] / effect_mark_graph_execution_unsupported_ state_depformer_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_logits_sample_projection --> state_depformer_logits_sample_projection_result_decision : completion_step_run_ [always] / none diff --git a/.planning/architecture/mermaid/speech_tokenizer_moshi.mmd b/.planning/architecture/mermaid/speech_tokenizer_moshi.mmd index 3281e4e8..85216ea6 100644 --- a/.planning/architecture/mermaid/speech_tokenizer_moshi.mmd +++ b/.planning/architecture/mermaid/speech_tokenizer_moshi.mmd @@ -19,13 +19,13 @@ stateDiagram-v2 state_prepared_tail --> state_prepared_tail : tokenize [always] / phase_order__ state_prepared_full --> state_commit_full_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ state_prepared_full --> state_commit_full_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ - state_prepared_tail --> state_commit_full_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ - state_prepared_tail --> state_commit_full_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ + state_prepared_tail --> state_commit_generated_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ + state_prepared_tail --> state_commit_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ state_prepared_generated --> state_commit_generated_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ state_prepared_generated --> state_commit_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ - state_prepared_full --> state_prepared_full : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ - state_prepared_generated --> state_prepared_generated : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ - state_prepared_tail --> state_prepared_tail : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ + state_prepared_full --> state_prepared_full : detokenize_run [guard_detokenize_request_invalid_] / request_shape__ + state_prepared_generated --> state_prepared_generated : detokenize_run [guard_detokenize_request_invalid_] / request_shape__ + state_prepared_tail --> state_prepared_tail : detokenize_run [guard_detokenize_request_invalid_] / request_shape__ state_prepared_full --> state_prepared_full : detokenize_run [guard_position_overflow_] / position_overflow__ state_prepared_generated --> state_prepared_generated : detokenize_run [guard_position_overflow_] / position_overflow__ state_prepared_tail --> state_prepared_tail : detokenize_run [guard_position_overflow_] / position_overflow__ diff --git a/.planning/architecture/mermaid/text_generator_matmul.mmd b/.planning/architecture/mermaid/text_generator_matmul.mmd deleted file mode 100644 index 8961b875..00000000 --- a/.planning/architecture/mermaid/text_generator_matmul.mmd +++ /dev/null @@ -1,18 +0,0 @@ -stateDiagram-v2 - direction TB - [*] --> state_ready - state_ready --> state_ready : configure_kernel_kind [always] / effect_configure_kernel_kind_ - state_ready --> state_serial_result_decision : execute_serial [always] / effect_execute_serial_ - state_serial_result_decision --> state_ready : completion_execute_serial_ [guard_serial_accepted_] / effect_accept_serial_execution_ - state_serial_result_decision --> state_ready : completion_execute_serial_ [guard_serial_rejected_] / effect_reject_serial_execution_ - state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_x8_ready_] / effect_execute_parallel_x8_ - state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_x4_ready_] / effect_execute_parallel_x4_ - state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_unit_ready_] / effect_execute_parallel_unit_ - state_ready --> state_ready : execute_parallel [guard_parallel_unavailable_] / effect_reject_parallel_execution_ - state_parallel_result_decision --> state_ready : completion_execute_parallel_ [guard_parallel_submission_failed_] / effect_reject_parallel_execution_ - state_parallel_result_decision --> state_ready : completion_execute_parallel_ [guard_parallel_join_failed_] / effect_reject_parallel_execution_ - state_parallel_result_decision --> state_ready : completion_execute_parallel_ [guard_parallel_lane_rejected_] / effect_reject_parallel_execution_ - state_parallel_result_decision --> state_ready : completion_execute_parallel_ [guard_parallel_all_lanes_accepted_] / effect_accept_parallel_execution_ - state_ready --> state_ready : _ [always] / effect_on_unexpected_ - state_serial_result_decision --> state_ready : _ [always] / effect_on_unexpected_ - state_parallel_result_decision --> state_ready : _ [always] / effect_on_unexpected_ diff --git a/.planning/architecture/speech_codec_mimi.md b/.planning/architecture/speech_codec_mimi.md index 8192c972..e24dcdf4 100644 --- a/.planning/architecture/speech_codec_mimi.md +++ b/.planning/architecture/speech_codec_mimi.md @@ -9,10 +9,16 @@ stateDiagram-v2 direction TB [*] --> state_uninitialized state_uninitialized --> state_bind_contract_decision : initialize_run [always] / none - state_bind_contract_decision --> state_bind_capacity_decision : completion_initialize_run_ [guard_bind_contract_valid_] / none + state_bind_contract_decision --> state_bind_f32_capacity_decision : completion_initialize_run_ [guard_bind_f32_contract_valid_] / none + state_bind_contract_decision --> state_bind_native_capacity_decision : completion_initialize_run_ [guard_bind_native_contract_valid_] / none + state_bind_contract_decision --> state_bind_q8_capacity_decision : completion_initialize_run_ [guard_bind_q8_contract_valid_] / none state_bind_contract_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_bind_contract_invalid_] / effect_mark_bind_failed_ - state_bind_capacity_decision --> state_binding : completion_initialize_run_ [guard_arena_capacity_valid_] / effect_bind_ - state_bind_capacity_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_arena_capacity_invalid_] / effect_mark_arena_capacity_invalid_ + state_bind_f32_capacity_decision --> state_binding : completion_initialize_run_ [guard_arena_capacity_valid_] / effect_bind_f32_ + state_bind_f32_capacity_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_arena_capacity_invalid_] / effect_mark_arena_capacity_invalid_ + state_bind_native_capacity_decision --> state_binding : completion_initialize_run_ [guard_arena_capacity_valid_] / effect_bind_native_ + state_bind_native_capacity_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_arena_capacity_invalid_] / effect_mark_arena_capacity_invalid_ + state_bind_q8_capacity_decision --> state_binding : completion_initialize_run_ [guard_arena_capacity_valid_] / effect_bind_q8_ + state_bind_q8_capacity_decision --> state_init_failed_error_out_decision : completion_initialize_run_ [guard_arena_capacity_invalid_] / effect_mark_arena_capacity_invalid_ state_binding --> state_init_error_out_decision : completion_initialize_run_ [always] / none state_init_error_out_decision --> state_init_callback_decision : completion_initialize_run_ [initialize_run__] / initialize_run__ state_init_error_out_decision --> state_init_callback_decision : completion_initialize_run_ [initialize_run__] / none @@ -51,6 +57,8 @@ stateDiagram-v2 state_decode_failed_callback_decision --> state_session_ready : completion_decode_frame_run_ [decode_frame_run__] / effect_emit_decode_error_ state_decode_failed_callback_decision --> state_session_ready : completion_decode_frame_run_ [decode_frame_run__] / none state_session_ready --> state_session_ready : reset_stream_run [always] / effect_reset_stream_ + state_uninitialized --> state_uninitialized : capture_diagnostics [always] / effect_capture_diagnostics_ + state_session_ready --> state_session_ready : capture_diagnostics [always] / effect_capture_diagnostics_ state_uninitialized --> state_uninit_encode_error_out_decision : encode_frame_run [always] / encode_frame_run__ state_uninit_encode_error_out_decision --> state_uninit_encode_callback_decision : completion_encode_frame_run_ [encode_frame_run__] / encode_frame_run__ state_uninit_encode_error_out_decision --> state_uninit_encode_callback_decision : completion_encode_frame_run_ [encode_frame_run__] / none @@ -97,8 +105,12 @@ stateDiagram-v2 state_decode_failed_callback_decision --> state_session_ready : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ state_bind_contract_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ state_bind_contract_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ - state_bind_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ - state_bind_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_bind_f32_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_bind_f32_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_bind_native_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_bind_native_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ + state_bind_q8_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ + state_bind_q8_capacity_decision --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ state_binding --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ state_binding --> state_uninitialized : _ [guard_unexpected_error_out_absent_] / effect_mark_unexpected_ state_init_error_out_decision --> state_uninitialized : _ [guard_unexpected_error_out_present_] / effect_mark_unexpected_and_store_ @@ -124,10 +136,16 @@ stateDiagram-v2 | Source | Event | Guard | Action | Target | | --- | --- | --- | --- | --- | | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`initialize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | -| [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_bind_contract_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_bind_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_bind_f32_contract_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_bind_f32_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_bind_native_contract_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_bind_native_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_bind_q8_contract_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_bind_q8_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_bind_contract_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | -| [`state_bind_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_arena_capacity_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_bind>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_binding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | -| [`state_bind_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_arena_capacity_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_arena_capacity_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_f32_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_arena_capacity_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_bind_f32>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_binding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_f32_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_arena_capacity_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_arena_capacity_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_native_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_arena_capacity_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_bind_native>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_binding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_native_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_arena_capacity_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_arena_capacity_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_q8_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_arena_capacity_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_bind_q8>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_binding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_q8_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_arena_capacity_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_arena_capacity_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_init_failed_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_binding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_init_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_init_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_init_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_init_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`initialize_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_init_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | @@ -166,6 +184,8 @@ stateDiagram-v2 | [`state_decode_failed_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`decode_frame_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_emit_decode_error>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_session_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_decode_failed_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`decode_frame_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_session_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_session_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`reset_stream_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_reset_stream>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_session_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`capture_diagnostics`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_capture_diagnostics>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_session_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`capture_diagnostics`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_capture_diagnostics>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_session_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`encode_frame_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`encode_frame_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninit_encode_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_uninit_encode_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`encode_frame_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`encode_frame_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninit_encode_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_uninit_encode_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`encode_frame_run>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninit_encode_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | @@ -212,8 +232,12 @@ stateDiagram-v2 | [`state_decode_failed_callback_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_session_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_bind_contract_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | -| [`state_bind_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | -| [`state_bind_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_f32_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_f32_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_native_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_native_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_q8_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | +| [`state_bind_q8_capacity_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_binding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_binding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_absent>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | | [`state_init_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`guard_unexpected_error_out_present>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`effect_mark_unexpected_and_store>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | [`state_uninitialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/codec/mimi/sm.hpp) | diff --git a/.planning/architecture/speech_predictor_moshi_executor.md b/.planning/architecture/speech_predictor_moshi_executor.md index 6008b89d..3e519b4e 100644 --- a/.planning/architecture/speech_predictor_moshi_executor.md +++ b/.planning/architecture/speech_predictor_moshi_executor.md @@ -84,7 +84,8 @@ stateDiagram-v2 state_temporal_layer_norm_result_decision --> state_temporal_layer_projection : completion_step_run_ [guard_temporal_layer_projection_supported_] / effect_bind_temporal_layer_projection_ state_temporal_layer_norm_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_projection_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_projection --> state_temporal_layer_projection_bind_result_decision : completion_step_run_ [always] / none - state_temporal_layer_projection_bind_result_decision --> state_temporal_layer_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_projection_ + state_temporal_layer_projection_bind_result_decision --> state_temporal_layer_projection_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_temporal_layer_projection_serial_ + state_temporal_layer_projection_bind_result_decision --> state_temporal_layer_projection_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_temporal_layer_projection_parallel_ state_temporal_layer_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_projection_run --> state_temporal_layer_projection_result_decision : completion_step_run_ [guard_temporal_layer_projection_succeeded_] / none state_temporal_layer_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_projection_failed_] / effect_mark_graph_execution_unsupported_ @@ -99,14 +100,19 @@ stateDiagram-v2 state_temporal_layer_key_rope_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_cache_write_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_cache_write --> state_temporal_layer_cache_write_result_decision : completion_step_run_ [guard_temporal_layer_cache_write_succeeded_] / none state_temporal_layer_cache_write --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_cache_write_failed_] / effect_mark_graph_execution_unsupported_ - state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_supported_] / effect_run_temporal_layer_attention_ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_route_1__] / effect_run_temporal_layer_attention_1__ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_route_2__] / effect_run_temporal_layer_attention_2__ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_route_4__] / effect_run_temporal_layer_attention_4__ + state_temporal_layer_cache_write_result_decision --> state_temporal_layer_attention : completion_step_run_ [guard_temporal_layer_attention_route_8__] / effect_run_temporal_layer_attention_8__ state_temporal_layer_cache_write_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_unsupported_] / effect_mark_graph_execution_unsupported_ + state_temporal_layer_cache_write_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_route_unavailable_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_attention --> state_temporal_layer_attention_result_decision : completion_step_run_ [guard_temporal_layer_attention_succeeded_] / none state_temporal_layer_attention --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_attention_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_attention_result_decision --> state_temporal_layer_out_projection : completion_step_run_ [guard_temporal_layer_out_projection_supported_] / effect_bind_temporal_layer_out_projection_ state_temporal_layer_attention_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_out_projection_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_out_projection --> state_temporal_layer_out_projection_bind_result_decision : completion_step_run_ [always] / none - state_temporal_layer_out_projection_bind_result_decision --> state_temporal_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_out_projection_ + state_temporal_layer_out_projection_bind_result_decision --> state_temporal_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_temporal_layer_out_projection_serial_ + state_temporal_layer_out_projection_bind_result_decision --> state_temporal_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_temporal_layer_out_projection_parallel_ state_temporal_layer_out_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_out_projection_run --> state_temporal_layer_out_projection_result_decision : completion_step_run_ [guard_temporal_layer_out_projection_succeeded_] / none state_temporal_layer_out_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_out_projection_failed_] / effect_mark_graph_execution_unsupported_ @@ -123,7 +129,8 @@ stateDiagram-v2 state_temporal_layer_norm2_result_decision --> state_temporal_layer_gating_in : completion_step_run_ [guard_temporal_layer_gating_in_supported_] / effect_bind_temporal_layer_gating_in_ state_temporal_layer_norm2_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_in_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_gating_in --> state_temporal_layer_gating_in_bind_result_decision : completion_step_run_ [always] / none - state_temporal_layer_gating_in_bind_result_decision --> state_temporal_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_gating_in_ + state_temporal_layer_gating_in_bind_result_decision --> state_temporal_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_temporal_layer_gating_in_serial_ + state_temporal_layer_gating_in_bind_result_decision --> state_temporal_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_temporal_layer_gating_in_parallel_ state_temporal_layer_gating_in_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_gating_in_run --> state_temporal_layer_gating_in_result_decision : completion_step_run_ [guard_temporal_layer_gating_in_succeeded_] / none state_temporal_layer_gating_in_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_in_failed_] / effect_mark_graph_execution_unsupported_ @@ -137,7 +144,8 @@ stateDiagram-v2 state_temporal_layer_silu_gate_result_decision --> state_temporal_layer_gating_out : completion_step_run_ [guard_temporal_layer_gating_out_supported_] / effect_bind_temporal_layer_gating_out_ state_temporal_layer_silu_gate_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_out_unsupported_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_gating_out --> state_temporal_layer_gating_out_bind_result_decision : completion_step_run_ [always] / none - state_temporal_layer_gating_out_bind_result_decision --> state_temporal_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_temporal_layer_gating_out_ + state_temporal_layer_gating_out_bind_result_decision --> state_temporal_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_temporal_layer_gating_out_serial_ + state_temporal_layer_gating_out_bind_result_decision --> state_temporal_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_temporal_layer_gating_out_parallel_ state_temporal_layer_gating_out_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_temporal_layer_gating_out_run --> state_temporal_layer_gating_out_result_decision : completion_step_run_ [guard_temporal_layer_gating_out_succeeded_] / none state_temporal_layer_gating_out_run --> state_step_error_out_decision : completion_step_run_ [guard_temporal_layer_gating_out_failed_] / effect_mark_graph_execution_unsupported_ @@ -161,11 +169,14 @@ stateDiagram-v2 state_sampling_ready --> state_text_logits : step_run [guard_sampling_step_valid_] / effect_restore_prediction_state_and_bind_text_logits_ state_sampling_ready --> state_step_error_out_decision : step_run [guard_sampling_step_invalid_] / effect_mark_request_shape_ state_text_logits --> state_text_logits_bind_result_decision : completion_step_run_ [always] / none - state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_forced_text_token_valid_and_sampling_consume_] / effect_compute_text_token_logits_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_forced_text_token_sampling_serial_] / effect_compute_text_token_logits_serial_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_forced_text_token_sampling_parallel_] / effect_compute_text_token_logits_parallel_ state_text_logits_bind_result_decision --> state_text_logits_result_decision : completion_step_run_ [guard_forced_text_token_valid_without_sampling_consume_] / effect_publish_forced_text_token_ state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_forced_text_sampling_config_invalid_] / effect_mark_graph_execution_unsupported_ + state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_forced_text_token_invalid_] / effect_mark_request_shape_ state_text_logits_bind_result_decision --> state_text_logits_run : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_argmax_] / effect_select_text_token_ - state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_sampling_] / effect_compute_text_token_logits_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_text_logits_sampling_serial_] / effect_compute_text_token_logits_serial_ + state_text_logits_bind_result_decision --> state_text_logits_sample_projection : completion_step_run_ [guard_text_logits_sampling_parallel_] / effect_compute_text_token_logits_parallel_ state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid_] / effect_mark_graph_execution_unsupported_ state_text_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_text_logits_sample_projection --> state_text_logits_sample_projection_result_decision : completion_step_run_ [always] / none @@ -192,7 +203,8 @@ stateDiagram-v2 state_depformer_weight_decision --> state_depformer_input : completion_step_run_ [guard_depformer_audio_input_supported_] / effect_bind_depformer_audio_input_projection_ state_depformer_weight_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_input --> state_depformer_input_projection_bind_result_decision : completion_step_run_ [always] / none - state_depformer_input_projection_bind_result_decision --> state_depformer_input_projection : completion_step_run_ [guard_depformer_input_projection_bound_] / effect_run_depformer_input_projection_ + state_depformer_input_projection_bind_result_decision --> state_depformer_input_projection : completion_step_run_ [guard_depformer_input_projection_serial_] / effect_run_depformer_input_projection_serial_ + state_depformer_input_projection_bind_result_decision --> state_depformer_input_projection : completion_step_run_ [guard_depformer_input_projection_parallel_] / effect_run_depformer_input_projection_parallel_ state_depformer_input_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_input_projection_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_input_projection --> state_depformer_input_projection_result_decision : completion_step_run_ [always] / none state_depformer_input_projection_result_decision --> state_depformer_input_embedding_bind : completion_step_run_ [guard_depformer_text_input_projection_succeeded_] / effect_bind_depformer_text_input_embedding_ @@ -217,7 +229,8 @@ stateDiagram-v2 state_depformer_layer_norm_result_decision --> state_depformer_layer_projection : completion_step_run_ [guard_depformer_layer_projection_supported_] / effect_bind_depformer_layer_projection_ state_depformer_layer_norm_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_projection_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_projection --> state_depformer_layer_projection_bind_result_decision : completion_step_run_ [always] / none - state_depformer_layer_projection_bind_result_decision --> state_depformer_layer_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_projection_ + state_depformer_layer_projection_bind_result_decision --> state_depformer_layer_projection_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_depformer_layer_projection_serial_ + state_depformer_layer_projection_bind_result_decision --> state_depformer_layer_projection_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_depformer_layer_projection_parallel_ state_depformer_layer_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_projection_run --> state_depformer_layer_projection_result_decision : completion_step_run_ [guard_depformer_layer_projection_succeeded_] / none state_depformer_layer_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_projection_failed_] / effect_mark_graph_execution_unsupported_ @@ -232,7 +245,8 @@ stateDiagram-v2 state_depformer_layer_attention_result_decision --> state_depformer_layer_out_projection : completion_step_run_ [guard_depformer_layer_out_projection_supported_] / effect_bind_depformer_layer_out_projection_ state_depformer_layer_attention_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_out_projection_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_out_projection --> state_depformer_layer_out_projection_bind_result_decision : completion_step_run_ [always] / none - state_depformer_layer_out_projection_bind_result_decision --> state_depformer_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_out_projection_ + state_depformer_layer_out_projection_bind_result_decision --> state_depformer_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_depformer_layer_out_projection_serial_ + state_depformer_layer_out_projection_bind_result_decision --> state_depformer_layer_out_projection_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_depformer_layer_out_projection_parallel_ state_depformer_layer_out_projection_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_out_projection_run --> state_depformer_layer_out_projection_result_decision : completion_step_run_ [guard_depformer_layer_out_projection_succeeded_] / none state_depformer_layer_out_projection_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_out_projection_failed_] / effect_mark_graph_execution_unsupported_ @@ -249,7 +263,8 @@ stateDiagram-v2 state_depformer_layer_norm2_result_decision --> state_depformer_layer_gating_in : completion_step_run_ [guard_depformer_layer_gating_in_supported_] / effect_bind_depformer_layer_gating_in_ state_depformer_layer_norm2_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_in_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_gating_in --> state_depformer_layer_gating_in_bind_result_decision : completion_step_run_ [always] / none - state_depformer_layer_gating_in_bind_result_decision --> state_depformer_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_gating_in_ + state_depformer_layer_gating_in_bind_result_decision --> state_depformer_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_depformer_layer_gating_in_serial_ + state_depformer_layer_gating_in_bind_result_decision --> state_depformer_layer_gating_in_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_depformer_layer_gating_in_parallel_ state_depformer_layer_gating_in_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_gating_in_run --> state_depformer_layer_gating_in_result_decision : completion_step_run_ [guard_depformer_layer_gating_in_succeeded_] / none state_depformer_layer_gating_in_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_in_failed_] / effect_mark_graph_execution_unsupported_ @@ -263,7 +278,8 @@ stateDiagram-v2 state_depformer_layer_silu_gate_result_decision --> state_depformer_layer_gating_out : completion_step_run_ [guard_depformer_layer_gating_out_supported_] / effect_bind_depformer_layer_gating_out_ state_depformer_layer_silu_gate_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_out_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_gating_out --> state_depformer_layer_gating_out_bind_result_decision : completion_step_run_ [always] / none - state_depformer_layer_gating_out_bind_result_decision --> state_depformer_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_] / effect_run_depformer_layer_gating_out_ + state_depformer_layer_gating_out_bind_result_decision --> state_depformer_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_serial_] / effect_run_depformer_layer_gating_out_serial_ + state_depformer_layer_gating_out_bind_result_decision --> state_depformer_layer_gating_out_run : completion_step_run_ [guard_projection_view_bound_parallel_] / effect_run_depformer_layer_gating_out_parallel_ state_depformer_layer_gating_out_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_layer_gating_out_run --> state_depformer_layer_gating_out_result_decision : completion_step_run_ [guard_depformer_layer_gating_out_succeeded_] / none state_depformer_layer_gating_out_run --> state_step_error_out_decision : completion_step_run_ [guard_depformer_layer_gating_out_failed_] / effect_mark_graph_execution_unsupported_ @@ -275,7 +291,8 @@ stateDiagram-v2 state_depformer_layer_ff_residual_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_unsupported_] / effect_mark_graph_execution_unsupported_ state_depformer_logits --> state_depformer_logits_bind_result_decision : completion_step_run_ [always] / none state_depformer_logits_bind_result_decision --> state_depformer_logits_run : completion_step_run_ [guard_depformer_logits_projection_bound_argmax_] / effect_select_depformer_token_ - state_depformer_logits_bind_result_decision --> state_depformer_logits_sample_projection : completion_step_run_ [guard_depformer_logits_projection_bound_sampling_] / effect_compute_depformer_token_logits_ + state_depformer_logits_bind_result_decision --> state_depformer_logits_sample_projection : completion_step_run_ [guard_depformer_logits_sampling_serial_] / effect_compute_depformer_token_logits_serial_ + state_depformer_logits_bind_result_decision --> state_depformer_logits_sample_projection : completion_step_run_ [guard_depformer_logits_sampling_parallel_] / effect_compute_depformer_token_logits_parallel_ state_depformer_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_depformer_logits_projection_bound_sampling_invalid_] / effect_mark_graph_execution_unsupported_ state_depformer_logits_bind_result_decision --> state_step_error_out_decision : completion_step_run_ [guard_projection_view_bind_failed_] / effect_mark_graph_execution_unsupported_ state_depformer_logits_sample_projection --> state_depformer_logits_sample_projection_result_decision : completion_step_run_ [always] / none @@ -395,7 +412,8 @@ stateDiagram-v2 | [`state_temporal_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_projection_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_layer_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_projection_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_temporal_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_projection_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_projection_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -410,14 +428,19 @@ stateDiagram-v2 | [`state_temporal_layer_key_rope_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_cache_write_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_cache_write`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_cache_write_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_cache_write`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_cache_write_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_attention>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_route<1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_attention<1>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_route<2>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_attention<2>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_route<4>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_attention<4>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_route<8>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_attention<8>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_cache_write_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_route_unavailable>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_attention`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_attention_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_out_projection_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_layer_out_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_out_projection_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_out_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_temporal_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_out_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_out_projection_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_out_projection_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_out_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_out_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_out_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -434,7 +457,8 @@ stateDiagram-v2 | [`state_temporal_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_in_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_layer_gating_in>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_in_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_gating_in`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_temporal_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_gating_in>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_gating_in_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_gating_in_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_in_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_in_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_in_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -448,7 +472,8 @@ stateDiagram-v2 | [`state_temporal_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_out_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_temporal_layer_gating_out>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_out_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_gating_out`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_temporal_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_gating_out>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_gating_out_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_temporal_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_temporal_layer_gating_out_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_out_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_temporal_layer_gating_out_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_temporal_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_temporal_layer_gating_out_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -472,11 +497,14 @@ stateDiagram-v2 | [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_sampling_step_valid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_restore_prediction_state_and_bind_text_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_sampling_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`step_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_sampling_step_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_request_shape>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_text_logits`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_token_valid_and_sampling_consume>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_text_token_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_token_sampling_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_text_token_logits_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_token_sampling_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_text_token_logits_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_token_valid_without_sampling_consume>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_publish_forced_text_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_sampling_config_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_forced_text_token_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_request_shape>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_projection_bound_and_no_forced_token_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_select_text_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_projection_bound_and_no_forced_token_sampling>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_text_token_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_sampling_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_text_token_logits_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_sampling_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_text_token_logits_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_text_logits_projection_bound_and_no_forced_token_sampling_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_text_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_text_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_text_logits_sample_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -503,7 +531,8 @@ stateDiagram-v2 | [`state_depformer_weight_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_audio_input_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_audio_input_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_weight_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_input`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_depformer_input_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_projection_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_input_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_projection_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_input_projection_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_input_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_projection_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_input_projection_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_input_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_input_projection_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_input_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_input_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_text_input_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_text_input_embedding>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_input_embedding_bind`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -528,7 +557,8 @@ stateDiagram-v2 | [`state_depformer_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_projection_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_layer_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_norm_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_projection_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_depformer_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_projection_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_projection_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -543,7 +573,8 @@ stateDiagram-v2 | [`state_depformer_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_out_projection_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_layer_out_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_attention_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_out_projection_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_out_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_depformer_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_out_projection>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_out_projection_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_out_projection_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_out_projection_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_out_projection_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_out_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_out_projection_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_out_projection_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -560,7 +591,8 @@ stateDiagram-v2 | [`state_depformer_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_in_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_layer_gating_in>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_norm2_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_in_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_gating_in`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_depformer_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_gating_in>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_gating_in_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_gating_in_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_gating_in_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_in_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_in_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_gating_in_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_in_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -574,7 +606,8 @@ stateDiagram-v2 | [`state_depformer_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_out_supported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_bind_depformer_layer_gating_out>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_silu_gate_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_out_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_gating_out`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_depformer_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_gating_out>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_gating_out_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bound_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_run_depformer_layer_gating_out_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_gating_out_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_out_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_layer_gating_out_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_layer_gating_out_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_layer_gating_out_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | @@ -586,7 +619,8 @@ stateDiagram-v2 | [`state_depformer_layer_ff_residual_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_logits`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_projection_bound_argmax>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_select_depformer_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | -| [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_projection_bound_sampling>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_depformer_token_logits>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_sampling_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_depformer_token_logits_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | +| [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_sampling_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_compute_depformer_token_logits_parallel>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_depformer_logits_projection_bound_sampling_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_logits_bind_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`guard_projection_view_bind_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`effect_mark_graph_execution_unsupported>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_step_error_out_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | | [`state_depformer_logits_sample_projection`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | [`state_depformer_logits_sample_projection_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/predictor/moshi/executor/sm.hpp) | diff --git a/.planning/architecture/speech_tokenizer_moshi.md b/.planning/architecture/speech_tokenizer_moshi.md index 4d936f23..620d12b2 100644 --- a/.planning/architecture/speech_tokenizer_moshi.md +++ b/.planning/architecture/speech_tokenizer_moshi.md @@ -26,13 +26,13 @@ stateDiagram-v2 state_prepared_tail --> state_prepared_tail : tokenize [always] / phase_order__ state_prepared_full --> state_commit_full_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ state_prepared_full --> state_commit_full_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ - state_prepared_tail --> state_commit_full_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ - state_prepared_tail --> state_commit_full_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ + state_prepared_tail --> state_commit_generated_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ + state_prepared_tail --> state_commit_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ state_prepared_generated --> state_commit_generated_zero : detokenize_run [guard_detokenize_valid_replace_] / effect_begin_detokenize_ state_prepared_generated --> state_commit_generated : detokenize_run [guard_detokenize_valid_generated_] / effect_begin_detokenize_ - state_prepared_full --> state_prepared_full : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ - state_prepared_generated --> state_prepared_generated : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ - state_prepared_tail --> state_prepared_tail : detokenize_run [guard_detokenize_shape_invalid_] / request_shape__ + state_prepared_full --> state_prepared_full : detokenize_run [guard_detokenize_request_invalid_] / request_shape__ + state_prepared_generated --> state_prepared_generated : detokenize_run [guard_detokenize_request_invalid_] / request_shape__ + state_prepared_tail --> state_prepared_tail : detokenize_run [guard_detokenize_request_invalid_] / request_shape__ state_prepared_full --> state_prepared_full : detokenize_run [guard_position_overflow_] / position_overflow__ state_prepared_generated --> state_prepared_generated : detokenize_run [guard_position_overflow_] / position_overflow__ state_prepared_tail --> state_prepared_tail : detokenize_run [guard_position_overflow_] / position_overflow__ @@ -99,13 +99,13 @@ stateDiagram-v2 | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`tokenize`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`phase_order>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_replace>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_full_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_generated>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_full_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | -| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_replace>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_full_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | -| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_generated>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_full_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_replace>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_generated_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_generated>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_replace>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_generated_zero`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_valid_generated>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`effect_begin_detokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_commit_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | -| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_shape_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | -| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_shape_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | -| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_shape_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_request_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_request_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | +| [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_detokenize_request_invalid>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`request_shape>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_full`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_generated`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`detokenize_run`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`guard_position_overflow>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`position_overflow>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | [`state_prepared_tail`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/speech/tokenizer/moshi/sm.hpp) | diff --git a/.planning/architecture/text_generator_matmul.md b/.planning/architecture/text_generator_matmul.md deleted file mode 100644 index 2ea05369..00000000 --- a/.planning/architecture/text_generator_matmul.md +++ /dev/null @@ -1,46 +0,0 @@ -# text_generator_matmul - -Source: [`emel/text/generator/matmul/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) - -## Mermaid - -```mermaid -stateDiagram-v2 - direction TB - [*] --> state_ready - state_ready --> state_ready : configure_kernel_kind [always] / effect_configure_kernel_kind_ - state_ready --> state_serial_result_decision : execute_serial [always] / effect_execute_serial_ - state_serial_result_decision --> state_ready : completion_execute_serial_ [guard_serial_accepted_] / effect_accept_serial_execution_ - state_serial_result_decision --> state_ready : completion_execute_serial_ [guard_serial_rejected_] / effect_reject_serial_execution_ - state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_x8_ready_] / effect_execute_parallel_x8_ - state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_x4_ready_] / effect_execute_parallel_x4_ - state_ready --> state_parallel_result_decision : execute_parallel [guard_parallel_unit_ready_] / effect_execute_parallel_unit_ - state_ready --> state_ready : execute_parallel [guard_parallel_unavailable_] / effect_reject_parallel_execution_ - state_parallel_result_decision --> state_ready : completion_execute_parallel_ [guard_parallel_submission_failed_] / effect_reject_parallel_execution_ - state_parallel_result_decision --> state_ready : completion_execute_parallel_ [guard_parallel_join_failed_] / effect_reject_parallel_execution_ - state_parallel_result_decision --> state_ready : completion_execute_parallel_ [guard_parallel_lane_rejected_] / effect_reject_parallel_execution_ - state_parallel_result_decision --> state_ready : completion_execute_parallel_ [guard_parallel_all_lanes_accepted_] / effect_accept_parallel_execution_ - state_ready --> state_ready : _ [always] / effect_on_unexpected_ - state_serial_result_decision --> state_ready : _ [always] / effect_on_unexpected_ - state_parallel_result_decision --> state_ready : _ [always] / effect_on_unexpected_ -``` - -## Transitions - -| Source | Event | Guard | Action | Target | -| --- | --- | --- | --- | --- | -| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`configure_kernel_kind`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_configure_kernel_kind>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`execute_serial`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_execute_serial>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_serial_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_serial_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_serial_accepted>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_accept_serial_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_serial_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_serial_rejected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_reject_serial_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_parallel_x8_ready>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_execute_parallel_x8>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_parallel_x4_ready>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_execute_parallel_x4>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_parallel_unit_ready>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_execute_parallel_unit>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`execute_parallel`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_parallel_unavailable>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_parallel_submission_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_parallel_join_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_parallel_lane_rejected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_reject_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`guard_parallel_all_lanes_accepted>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_accept_parallel_execution>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_serial_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | -| [`state_parallel_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`effect_on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | [`state_ready`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/generator/matmul/sm.hpp) | diff --git a/scripts/bench_moshi_lm_compare.sh b/scripts/bench_moshi_lm_compare.sh index 599951d1..8d381a5a 100755 --- a/scripts/bench_moshi_lm_compare.sh +++ b/scripts/bench_moshi_lm_compare.sh @@ -8,8 +8,6 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/build_jobs.sh source "$ROOT_DIR/scripts/build_jobs.sh" -# shellcheck source=scripts/zig_toolchain.sh -source "$ROOT_DIR/scripts/zig_toolchain.sh" BUILD_DIR_OVERRIDE="${EMEL_MOSHI_LM_COMPARE_BUILD_DIR:-}" BUILD_DIR="" @@ -120,12 +118,17 @@ for tool in cmake ninja git; do fi done +if ! $RUN_ONLY && $USE_ZIG; then + # shellcheck source=scripts/zig_toolchain.sh + source "$ROOT_DIR/scripts/zig_toolchain.sh" +fi + cmake_args=(-S "$ROOT_DIR/tools/bench" -B "$BUILD_DIR" -G Ninja -DCMAKE_BUILD_TYPE=Release -DEMEL_BENCH_SUITE_FILTER=speech_lm_moshi -DEMEL_BENCH_SKIP_MOSHI_REFERENCE=ON) -if $USE_ZIG; then +if ! $RUN_ONLY && $USE_ZIG; then if ! command -v zig >/dev/null 2>&1; then echo "error: zig not found (use --system to use system compilers)" >&2 exit 1 diff --git a/snapshots/quality_gates/timing.txt b/snapshots/quality_gates/timing.txt index 6e41ef50..5f90c7f8 100644 --- a/snapshots/quality_gates/timing.txt +++ b/snapshots/quality_gates/timing.txt @@ -1,12 +1,12 @@ # quality_gates timing (seconds) domain_boundaries 1 legacy_sml_surface 0 -build_with_zig 25 -bench_snapshot 38 -test_with_coverage 69 -paritychecker 2 +build_with_zig 6 +bench_snapshot 9 +test_with_coverage 0 +paritychecker 0 fuzz_smoke 0 -determinism_check 2 -lint_snapshot 12 +determinism_check 0 +lint_snapshot 11 generate_docs 0 -total 108 +total 27 diff --git a/tests/speech/predictor/moshi_lifecycle_tests.cpp b/tests/speech/predictor/moshi_lifecycle_tests.cpp index 98f7a220..8099115e 100644 --- a/tests/speech/predictor/moshi_lifecycle_tests.cpp +++ b/tests/speech/predictor/moshi_lifecycle_tests.cpp @@ -1713,6 +1713,9 @@ TEST_CASE("speech Moshi attention benchmark binds temporal and depformer KV") { CHECK(source.find("depformer_key_cache") != std::string::npos); CHECK(source.find("depformer_value_cache") != std::string::npos); CHECK(source.find(".depformer =") != std::string::npos); + CHECK(source.find("attention_lanes.emplace(lane_count - 1u)") != + std::string::npos); + CHECK(source.find("attention_lanes.emplace()") == std::string::npos); } TEST_CASE("speech_moshi_generator_graph_outputs_use_explicit_transitions") { diff --git a/tools/bench/bench_runner_tests.cpp b/tools/bench/bench_runner_tests.cpp index e9cedffe..f6bef1f9 100644 --- a/tools/bench/bench_runner_tests.cpp +++ b/tools/bench/bench_runner_tests.cpp @@ -1245,6 +1245,9 @@ TEST_CASE("moshi lm wrapper gives --model priority over inherited env") { write_file(tool_path, "#!/bin/sh\nexit 0\n"); make_executable(tool_path); } + const std::filesystem::path xcrun_path = fake_bin_dir / "xcrun"; + write_file(xcrun_path, "#!/bin/sh\nexit 99\n"); + make_executable(xcrun_path); write_file(fake_runner, "#!/bin/sh\n" "printf 'personaplex=%s\\nmoshi=%s\\nbench=%s\\nargs=%s\\n' " @@ -1304,6 +1307,18 @@ TEST_CASE("moshi lm wrapper keeps build-only model-free") { CHECK(tool_check < build_only_exit); } +TEST_CASE("moshi lm wrapper defers Zig setup outside system run-only") { + const std::string script = read_file(bench_moshi_lm_compare_wrapper_path()); + const std::size_t source_zig = + script.find("source \"$ROOT_DIR/scripts/zig_toolchain.sh\""); + const std::size_t build_guard = + script.find("if ! $RUN_ONLY && $USE_ZIG; then"); + + REQUIRE(source_zig != std::string::npos); + REQUIRE(build_guard != std::string::npos); + CHECK(build_guard < source_zig); +} + TEST_CASE("moshi lm wrapper and runner search common companion layouts") { const std::string script = read_file(bench_moshi_lm_compare_wrapper_path()); CHECK(script.find("$ROOT_DIR/../companion/zig-out/" diff --git a/tools/bench/speech/lm_moshi_bench.cpp b/tools/bench/speech/lm_moshi_bench.cpp index f09c8df9..83877e6d 100644 --- a/tools/bench/speech/lm_moshi_bench.cpp +++ b/tools/bench/speech/lm_moshi_bench.cpp @@ -216,7 +216,7 @@ struct attention_executor_bench_fixture { depformer_layer_offsets[index] = index * depformer_per_layer_cache; } if (lane_count > 1u) { - attention_lanes.emplace(); + attention_lanes.emplace(lane_count - 1u); } for (std::size_t index = 0u; index < input_embedding.size(); ++index) { input_embedding[index] = From 88757eb6b9e1a90733dda107ad25fb5ed32536f4 Mon Sep 17 00:00:00 2001 From: gabewillen Date: Fri, 24 Jul 2026 04:23:50 -0500 Subject: [PATCH 6/6] fix PersonaPlex CPU review findings --- scripts/bench_personaplex_compare.sh | 4 +- scripts/setup_personaplex_mlx_reference.sh | 15 ++++-- snapshots/lint/clang_format.txt | 1 - .../predictor/moshi/executor/guards.hpp | 21 +++++---- .../predictor/moshi_lifecycle_tests.cpp | 47 ++++++++++++++++++- tools/bench/personaplex_compare.py | 4 +- tools/bench/personaplex_compare_tests.cpp | 22 ++++++++- .../bench/speech/personaplex_emel_runner.cpp | 2 +- 8 files changed, 95 insertions(+), 21 deletions(-) diff --git a/scripts/bench_personaplex_compare.sh b/scripts/bench_personaplex_compare.sh index 206bbc55..69204b85 100755 --- a/scripts/bench_personaplex_compare.sh +++ b/scripts/bench_personaplex_compare.sh @@ -5,8 +5,6 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=scripts/build_jobs.sh source "$ROOT_DIR/scripts/build_jobs.sh" -# shellcheck source=scripts/zig_toolchain.sh -source "$ROOT_DIR/scripts/zig_toolchain.sh" BUILD_DIR="${EMEL_PERSONAPLEX_COMPARE_BUILD_DIR:-$ROOT_DIR/build/personaplex_compare_tools_zig}" OUTPUT_DIR="${EMEL_PERSONAPLEX_COMPARE_OUTPUT_DIR:-$ROOT_DIR/build/personaplex_compare}" @@ -90,6 +88,8 @@ if [[ -z "$PYTHON_BIN" ]] || fi if ! $RUN_ONLY; then + # shellcheck source=scripts/zig_toolchain.sh + source "$ROOT_DIR/scripts/zig_toolchain.sh" for tool in cmake ninja git zig; do if ! command -v "$tool" >/dev/null 2>&1; then echo "error: required build tool missing: $tool" >&2 diff --git a/scripts/setup_personaplex_mlx_reference.sh b/scripts/setup_personaplex_mlx_reference.sh index b1f09d35..ff7c5f07 100755 --- a/scripts/setup_personaplex_mlx_reference.sh +++ b/scripts/setup_personaplex_mlx_reference.sh @@ -15,6 +15,7 @@ ARTIFACT_DIR="${EMEL_PERSONAPLEX_MLX_ARTIFACT_DIR:-$ROOT_DIR/build/personaplex_m MOSHI_ARTIFACT_DIR="${EMEL_MOSHI_REFERENCE_ARTIFACT_DIR:-$ROOT_DIR/build/moshi_reference}" SRC_DIR="$ARTIFACT_DIR/src" VENV_DIR="$ARTIFACT_DIR/venv" +INSTALLED_REF_FILE="$VENV_DIR/.personaplex_mlx_ref" DRIVER_SHIM="$ARTIFACT_DIR/mimi_driver" PERSONAPLEX_MLX_REPOSITORY="https://github.com/mu-hashmi/personaplex-mlx" @@ -129,7 +130,16 @@ probe_personaplex_mlx() { return "$status" } -if probe_personaplex_mlx; then +install_personaplex_mlx() { + "$VENV_DIR/bin/pip" install --quiet --upgrade pip + "$VENV_DIR/bin/pip" install --quiet -e "$SRC_DIR" + printf '%s\n' "$resolved_ref" > "$INSTALLED_REF_FILE" +} + +if [[ ! -f "$INSTALLED_REF_FILE" || + "$(<"$INSTALLED_REF_FILE")" != "$resolved_ref" ]]; then + install_personaplex_mlx +elif probe_personaplex_mlx; then : else probe_status=$? @@ -137,8 +147,7 @@ else echo "error: personaplex_mlx import probe was repeatedly killed" >&2 exit "$probe_status" fi - "$VENV_DIR/bin/pip" install --quiet --upgrade pip - "$VENV_DIR/bin/pip" install --quiet -e "$SRC_DIR" + install_personaplex_mlx fi probe_personaplex_mlx \ || { echo "error: personaplex_mlx import check failed" >&2; exit 1; } diff --git a/snapshots/lint/clang_format.txt b/snapshots/lint/clang_format.txt index 4f967b74..10c63f31 100644 --- a/snapshots/lint/clang_format.txt +++ b/snapshots/lint/clang_format.txt @@ -360,7 +360,6 @@ src/emel/speech/encoder/whisper/errors.hpp src/emel/speech/encoder/whisper/guards.hpp src/emel/speech/predictor/moshi/events.hpp src/emel/speech/predictor/moshi/executor/actions.hpp -src/emel/speech/predictor/moshi/executor/guards.hpp src/emel/tensor/view/actions.hpp src/emel/tensor/view/context.hpp src/emel/tensor/view/detail.hpp diff --git a/src/emel/speech/predictor/moshi/executor/guards.hpp b/src/emel/speech/predictor/moshi/executor/guards.hpp index 66cc2d69..0d689d0b 100644 --- a/src/emel/speech/predictor/moshi/executor/guards.hpp +++ b/src/emel/speech/predictor/moshi/executor/guards.hpp @@ -1035,22 +1035,27 @@ struct guard_temporal_layer_attention_unsupported { } }; -template -struct guard_temporal_layer_attention_route { +template struct guard_temporal_layer_attention_route { bool operator()(const event::step_run &runtime_ev, const action::context &ctx) const noexcept { static_assert(lane_count == 1u || lane_count == 2u || lane_count == 4u || lane_count == 8u); - const bool route_selected = ctx.active_attention_lanes == lane_count; - const bool enough_heads = - runtime_ev.request.model.moshi_lm.num_heads >= - static_cast(lane_count); + const bool configured_lane_count_supported = + ctx.active_attention_lanes == 1u || ctx.active_attention_lanes == 2u || + ctx.active_attention_lanes == 4u || ctx.active_attention_lanes == 8u; + const std::size_t available_lanes = + std::min(ctx.active_attention_lanes, + static_cast( + std::max(runtime_ev.request.model.moshi_lm.num_heads, 0))); + const bool route_selected = configured_lane_count_supported && + available_lanes >= lane_count && + available_lanes < lane_count * 2u; if constexpr (lane_count == 1u) { return guard_temporal_layer_attention_supported{}(runtime_ev, ctx) && - route_selected && enough_heads; + route_selected; } else { return guard_temporal_layer_attention_supported{}(runtime_ev, ctx) && - route_selected && enough_heads && ctx.attention_lanes != nullptr; + route_selected && ctx.attention_lanes != nullptr; } } }; diff --git a/tests/speech/predictor/moshi_lifecycle_tests.cpp b/tests/speech/predictor/moshi_lifecycle_tests.cpp index 8099115e..9004cae8 100644 --- a/tests/speech/predictor/moshi_lifecycle_tests.cpp +++ b/tests/speech/predictor/moshi_lifecycle_tests.cpp @@ -2007,6 +2007,14 @@ TEST_CASE("speech_moshi_generator_and_executor_cover_explicit_error_guards") { std::span{embedding.data(), static_cast(lm.dim)}; moshi_executor::event::step_ctx step_ctx{}; moshi_executor::event::step_run step_run{step, step_ctx}; + emel::kernel::sm parallel_guard_kernel{}; + parallel_matmul_fixture parallel_guard_matmul{}; + moshi_executor::action::context parallel_executor_ctx{ + make_executor_dependencies(parallel_guard_kernel, + parallel_guard_matmul.actor, {}, nullptr, + emel::kernel::matmul::lane_mode::parallel)}; + CHECK(moshi_executor::guard::guard_parallel_matmul_route{}( + step_run, parallel_executor_ctx)); executor_ctx.session.model = fixture.model.get(); executor_ctx.session.codebook_count = lm.n_q + 1; executor_ctx.session.dep_q = lm.dep_q; @@ -2604,6 +2612,14 @@ TEST_CASE( auto step_ctx = std::make_unique(); moshi_executor::event::step_run step_run{step, *step_ctx}; + CHECK(moshi_executor::guard::guard_unexpected_error_out_absent{}( + step_run, executor_ctx)); + emel::error::type unexpected_err = k_no_error; + step.error_out = &unexpected_err; + CHECK(moshi_executor::guard::guard_unexpected_error_out_present{}( + step_run, executor_ctx)); + step.error_out = nullptr; + step.forced_text_token = lm.text_card; CHECK_FALSE(moshi_executor::guard::guard_forced_text_token_valid{}( step_run, executor_ctx)); @@ -2762,14 +2778,15 @@ struct attention_parent_fixture { }; std::unique_ptr -make_attention_parent_fixture(const std::size_t lanes) { +make_attention_parent_fixture(const std::size_t lanes, + const int32_t num_heads = 8) { auto fixture = std::make_unique(); fixture->loaded = load_fixture_or_skip("moshi-tiny-lm.gguf"); if (fixture->loaded.model == nullptr) { return fixture; } auto &model = *fixture->loaded.model; - model.moshi_lm.num_heads = 8; + model.moshi_lm.num_heads = num_heads; fixture->temporal = std::make_unique(); fixture->temporal_positions = std::make_unique( memory_streaming::dependencies{.capacity = model.moshi_lm.context}); @@ -2899,6 +2916,32 @@ TEST_CASE("speech Moshi unsupported attention lane count fails explicitly") { emel::error::cast(moshi_executor::error::graph_execution_unsupported)); } +TEST_CASE("speech Moshi attention falls back to the largest lane count bounded " + "by model heads") { + for (const int32_t num_heads : {1, 2, 4}) { + auto bounded = make_attention_parent_fixture( + static_cast(num_heads), num_heads); + auto eight_requested = make_attention_parent_fixture(8u, num_heads); + if (bounded->loaded.model == nullptr || + eight_requested->loaded.model == nullptr) { + return; + } + + std::size_t bounded_allocations = 0u; + std::size_t eight_allocations = 0u; + REQUIRE(run_attention_parent_prediction(*bounded, bounded_allocations)); + REQUIRE( + run_attention_parent_prediction(*eight_requested, eight_allocations)); + CHECK(bounded_allocations == 0u); + CHECK(eight_allocations == 0u); + CHECK(eight_requested->temporal_state_with_canaries == + bounded->temporal_state_with_canaries); + CHECK(eight_requested->temporal->key_cache == bounded->temporal->key_cache); + CHECK(eight_requested->temporal->value_cache == + bounded->temporal->value_cache); + } +} + TEST_CASE( "speech Moshi attention partial submission drains accepted children") { auto fixture = make_attention_parent_fixture(8u); diff --git a/tools/bench/personaplex_compare.py b/tools/bench/personaplex_compare.py index 708ae09d..c46b8235 100755 --- a/tools/bench/personaplex_compare.py +++ b/tools/bench/personaplex_compare.py @@ -123,8 +123,8 @@ def thread_contract_reasons( if owner_threads != 1: reasons.append("EMEL owner-thread count must be exactly one") accounted_concurrency = owner_threads + stage_workers + matmul_workers - if accounted_concurrency > requested_total: - reasons.append("EMEL runnable concurrency exceeds total CPU budget") + if accounted_concurrency != requested_total: + reasons.append("EMEL runnable concurrency does not match total CPU budget") if stage_workers not in (0, 2): reasons.append("EMEL stage worker count is unsupported") if ((stage_workers == 0 and stage_mode != "serial") or diff --git a/tools/bench/personaplex_compare_tests.cpp b/tools/bench/personaplex_compare_tests.cpp index cbd21c0a..176849a5 100644 --- a/tools/bench/personaplex_compare_tests.cpp +++ b/tools/bench/personaplex_compare_tests.cpp @@ -111,9 +111,12 @@ TEST_CASE( "matmul_mode=parallel\\n'); " "contract=m.parse_emel_threads(p); " "assert contract == (8,1,2,3,4,'parallel','parallel'); " - "assert m.thread_contract_reasons(contract, 8) == []; " + "assert m.thread_contract_reasons(contract, 8) == " + "['EMEL runnable concurrency does not match total CPU budget']; " + "assert m.thread_contract_reasons((8,1,0,7,8,'serial','parallel'), 8) " + "== []; " "assert m.thread_contract_reasons((8,1,2,7,8,'parallel','parallel'), 8) " - "== ['EMEL runnable concurrency exceeds total CPU budget']; " + "== ['EMEL runnable concurrency does not match total CPU budget']; " "p.unlink()"; CHECK(command_exit_code("python3 -B -c " + quote_arg_posix(program)) == 0); } @@ -177,6 +180,8 @@ TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { "personaplex_emel_runner.cpp"); const std::string setup_source = read_file(repo_root() / "scripts" / "setup_moshi_cpp_reference.sh"); + const std::string mlx_setup_source = + read_file(repo_root() / "scripts" / "setup_personaplex_mlx_reference.sh"); const std::string converter_source = read_file(repo_root() / "tools" / "bench" / "moshi_gguf_convert.py"); const std::string inference_source = @@ -213,6 +218,19 @@ TEST_CASE("personaplex compare keeps CPU and lane isolation explicit") { CHECK(wrapper_source.find( "bash \"$ROOT_DIR/scripts/setup_moshi_cpp_reference.sh\"") != std::string::npos); + const std::size_t run_only_branch = + wrapper_source.find("if ! $RUN_ONLY; then"); + const std::size_t zig_toolchain_source = + wrapper_source.find("source \"$ROOT_DIR/scripts/zig_toolchain.sh\""); + REQUIRE(run_only_branch != std::string::npos); + REQUIRE(zig_toolchain_source != std::string::npos); + CHECK(zig_toolchain_source > run_only_branch); + CHECK(mlx_setup_source.find("INSTALLED_REF_FILE=") != std::string::npos); + CHECK(mlx_setup_source.find("[[ ! -f \"$INSTALLED_REF_FILE\" ||") != + std::string::npos); + CHECK(mlx_setup_source.find( + "printf '%s\\n' \"$resolved_ref\" > \"$INSTALLED_REF_FILE\"") != + std::string::npos); CHECK(wrapper_source.find("--emel-lm PATH") != std::string::npos); CHECK(wrapper_source.find("--emel-lm) EMEL_LM=") != std::string::npos); CHECK(wrapper_source.find("--emel-mimi PATH") != std::string::npos); diff --git a/tools/bench/speech/personaplex_emel_runner.cpp b/tools/bench/speech/personaplex_emel_runner.cpp index c18a0de0..9f7bc880 100644 --- a/tools/bench/speech/personaplex_emel_runner.cpp +++ b/tools/bench/speech/personaplex_emel_runner.cpp @@ -609,7 +609,7 @@ int main(int argc, char **argv) { }; emel::kernel::sm prediction_kernel{}; const size_t stage_worker_count = - config.cpu_threads >= 4u + config.cpu_threads == 4u ? generator::action::wavefront_stage_pool::static_worker_count : 0u; const size_t available_matmul_lanes = config.cpu_threads - stage_worker_count;