From 292fb158337357d4e88df41f1c514ac8ffdb8747 Mon Sep 17 00:00:00 2001
From: neil couture <neilcouture@gmail.com>
Date: Sun, 2 Aug 2026 20:40:09 +0000
Subject: [PATCH] Kimi K3 CPU decode optimizations: MXFP4 kernels, parallel
 expert streaming, NUMA placement

Squashed port of the k3-optimize work:
- MXFP4 matmul: scalar kernel hygiene (bit-exact) plus an AVX-512 tier
  (within-tolerance), benchmarked in benches/kernels.rs
- Expert streaming: parallel miss loads, preload, skip pinning at full
  residency, honest logging
- latent_moe: across-expert parallelism (bit-identical v1)
- NUMA: src/numa.rs topology + placement primitives, --numa flag,
  pinned per-node pools, expert home nodes, per-socket pools with
  flattened domain dispatch, attention split
- QTSharded/DenseQT row-sharding (lm_head kept; KDA projections
  measured and reverted)
- K3 output filter stripping chat-envelope close markup
- Synthetic K3-at-scale fixture generator (src/bin/gen_k3_synth.rs)
  and target-box baseline; PERFORMANCE.md updated with measurements,
  including reverted attempts

Co-Authored-By: Claude <noreply@anthropic.com>
---
 CLAUDE.md                               | 179 +++++
 Cargo.toml                              |  12 +
 K3_OPTIMIZE_BRIEF.md                    | 515 ++++++++++++
 NUMA_AMX_BRIEF.md                       | 456 +++++++++++
 PERFORMANCE.md                          | 749 ++++++++++++++++++
 benches/kernels.rs                      |  32 +-
 examples/numa_pool_bench.rs             | 177 +++++
 examples/teacher_forced_decode_bench.rs |  50 +-
 src/bin/gen_k3_synth.rs                 | 323 ++++++++
 src/chat.rs                             | 101 ++-
 src/expert_cache.rs                     | 219 +++++-
 src/generate.rs                         |  43 +-
 src/glm52/model.rs                      |  13 +
 src/glm52/moe.rs                        |  31 +-
 src/kernels.rs                          | 991 ++++++++++++++++++++++--
 src/kimi_k3/chat_template.rs            | 122 ++-
 src/kimi_k3/generate.rs                 |  73 +-
 src/kimi_k3/model.rs                    |  74 +-
 src/kimi_k3/moe.rs                      | 488 +++++++++++-
 src/kimi_linear/generate.rs             |  18 +-
 src/kimi_linear/model.rs                |  13 +
 src/lib.rs                              |   1 +
 src/main.rs                             |  70 +-
 src/model.rs                            |  23 +
 src/numa.rs                             | 582 ++++++++++++++
 src/quant.rs                            |  33 +
 src/server.rs                           |  14 +-
 27 files changed, 5237 insertions(+), 165 deletions(-)
 create mode 100644 CLAUDE.md
 create mode 100644 K3_OPTIMIZE_BRIEF.md
 create mode 100644 NUMA_AMX_BRIEF.md
 create mode 100644 examples/numa_pool_bench.rs
 create mode 100644 src/bin/gen_k3_synth.rs
 create mode 100644 src/numa.rs

diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..80a2fb7
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,179 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this is
+
+rabbit is a Rust CPU inference engine for frontier open-weight MoE models on a single machine.
+The dense part of the model (attention, shared experts, embeddings) stays resident in RAM,
+quantized; the thousands of routed experts live on disk and stream on demand. It began as a port
+of **colibrì** (a C, GLM-5.2-only engine — referenced constantly in comments as "the reference
+implementation", "the C", `glm.c`, `st.h`) and has since generalized into three architectures.
+
+No GPU. No async runtime. `pread`, never `mmap` (mmap leaves pages resident and corrupts the
+peak-RSS accounting the streaming design depends on — see `src/safetensors.rs`'s module doc).
+
+## Build, test, run
+
+```bash
+cargo build --release
+cargo test                     # unit + integration; fixture-dependent tests SKIP when absent
+cargo test --test teacher_forcing        # one integration test file (GLM-5.2 oracle)
+cargo test <name_substring>              # one unit test
+cargo bench --bench kernels -- "qt_matvec_rows_i4"   # criterion, filtered
+```
+
+```bash
+./target/release/rabbit --model <checkpoint-dir> --prompt "..." --max-tokens 40
+./target/release/rabbit --model <dir> --chat --session ~/.rabbit_session
+./target/release/rabbit --model <dir> --serve --port 8000
+cargo run --release --bin convert -- --indir <dir> --outdir <dir> --classify-only
+cargo run --release --example k3_smoke -- --model <dir> --max-tokens 5
+cargo run --release --example teacher_forced_decode_bench -- --model <dir> --steps 30
+```
+
+`--model` auto-detects the architecture from `config.json`'s `model_type`. Kimi K3 (`kimi_k3`)
+and Kimi Linear (`kimi_linear`) read Moonshot's published safetensors directly, no conversion.
+GLM-5.2 (`glm_moe_dsa`) needs colibrì's converted layout.
+
+There are no lint/format configs in the repo; the codebase is `rustfmt`-shaped but with a wide
+line budget (~150 cols) and single-line `match` arms — match the surrounding file, don't
+reformat.
+
+### The oracle fixtures
+
+Every architecture is validated **token-exact against a tiny random-weight model built from that
+model family's own real reference code**, vendored under `tests/oracle/*_pkg/`. The fixtures are
+gitignored and must be generated locally with Python (torch/transformers/safetensors, often
+Docker — read each script's own docstring, they document the exact invocation and the patches
+they apply):
+
+```bash
+cd tests/oracle && python3 make_glm_oracle.py    # -> glm_tiny/  + ref_glm.json
+cd tests/oracle && python3 make_kimi_oracle.py   # -> kimi_tiny/ + ref_kimi.json
+cd tests/oracle && python3 make_k3_oracle.py     # -> k3_tiny/   + ref_k3.json
+cd tests/oracle && python3 make_convert_oracle.py  # -> convert_src/, convert_ref_{row,grouped}/
+python3 tools/fetch_tokenizer_fixture.py         # -> tests/fixtures/ (real tokenizer, no weights)
+python3 tools/gen_tokenizer_cases.py             # ground truth from the Python `tokenizers` lib
+```
+
+**Fixture-dependent tests skip rather than fail when the fixture is missing** — a green
+`cargo test` does not mean the oracles ran. Check the output for `SKIP` lines before claiming
+correctness. Affected: `teacher_forcing{,_kimi,_k3}.rs`, `convert_oracle.rs`,
+`convert_*_end_to_end.rs`, `*tokenizer_fixture.rs`.
+
+## Architecture
+
+### Family dispatch
+
+`src/model.rs` is the single dispatch point: `Model` / `KvState` / `ExpertCaches` / `Tokenizer`
+are each a three-variant **enum** (not a trait object — the `KvState` shapes genuinely differ:
+GLM's growing `KvCache`/`DsaCache` vs. Kimi's fixed-size `KdaLayerState` + conv FIFOs). `step` /
+`step_all` / `step_profiled` match on the *triple* `(model, caches, kv)`; the `unreachable!` arms
+only fire if a caller mixes state from two different `Model::load` calls.
+
+`src/glm52/` is both GLM-5.2's implementation **and** the shared substrate. The two Kimi modules
+reuse it unchanged:
+- `glm52::moe::{moe, dense_mlp, RouteConfig}` — all three architectures route through this
+- `glm52::attention::{attention, rmsnorm, OutputGate, Absorb}` — MLA for all three
+- `crate::generate::{StepProfile, Phases, argmax, Rng, SamplingConfig}`
+
+Kimi K3 wraps Kimi Linear rather than reimplementing it: `kimi_k3::config::Cfg` embeds
+`kimi_linear::config::Cfg` as `.base` (K3's real `config.json` nests the whole Kimi Linear config
+under `text_config`), and only K3's genuinely new pieces are new code — `ops::situ_and_mul`,
+`moe::latent_moe` (routed experts compute in a narrower latent width), `attn_res.rs` (Attention
+Residuals, transient per forward call, so nothing to persist in a session), MLA/KDA output gates.
+
+So an architecture module is: `config.rs` (raw-JSON-first `Cfg::load`), `model.rs` (tensor
+names + `ModelError`), `generate.rs` (`step`/`step_all`/`step_profiled`, `KvState`,
+`ExpertCaches`), `kv_session.rs` (own on-disk format + magic bytes), `chat_template.rs`,
+`tokenizer.rs`. Adding a fourth means those files plus five arms in `src/model.rs` plus an
+`ExpertNaming` variant.
+
+### The streaming path
+
+`generate::step` → per-layer `layer_forward` → `glm52::moe::moe()` → `ExpertCache::get_or_load`.
+On a miss, `src/expert_cache.rs` batches the reads: one `io_uring` submission on Linux, a
+sequential-`pread` fallback elsewhere and for MXFP4 (which reads a packed+scale tensor *pair* the
+ring doesn't model — deliberate, documented tradeoff). `ExpertNaming` (in `expert_cache.rs`) is
+what maps a logical `(layer, expert)` to real on-disk tensor names per family, including the
+fused-`gate_up` and `language_model.`-prefixed MXFP4 variants.
+
+Tiers, cheapest first: per-layer LRU (`--expert-cache N` slots) → lazily-promoted pins from
+`.rabbit_usage` (`src/usage_cache.rs`, a plain-text histogram learned across runs, permissively
+parsed — a corrupt file must never block generation) → OS page cache → disk.
+
+`moe.rs`'s **per-expert early drain** applies each miss's contribution the moment its read
+completes, so float summation order depends on real disk timing. Free-running greedy decode is
+therefore **not reproducible run-to-run** — this is why timing comparisons use
+`examples/teacher_forced_decode_bench.rs` (fixed token sequence fed at each step) rather than
+"generate N tokens and time it". Read that file's doc before benchmarking anything.
+
+### Quantization and kernels
+
+`src/quant.rs` is the `QT` container: f32 passthrough, int8, packed int4, packed int2, plus
+grouped-scale int4, native FP8 E4M3 (block-scale, resolved via a `{name}_scale_inv` sibling
+tensor) and OCP MXFP4 read straight off disk. `bits` and storage tier are independent (`bits=3`
+lives in the int4 tier). Rounding uses `round_ties_even`, matching C's `lrintf` under the default
+IEEE mode — required for token-exact parity, not a style choice.
+
+`src/kernels.rs` dispatches at runtime via `is_x86_feature_detected!`. Parity expectations differ
+per kernel and are load-bearing for tests: the integer IDOT path (`dot_i8i8`/`dot_i4i8`) must
+agree **bit-for-bit** across scalar/AVX2/AVX-512-VNNI, while `matmul_i4`'s AVX-512 tier
+reassociates and is only within-tolerance (and is in fact *more* accurate than scalar). MXFP4's
+matmul is scalar-only today — the single biggest known K3 perf lever.
+
+### CLI / server
+
+`src/main.rs` is argument parsing only. `src/chat.rs` holds all session/generation/template logic
+shared by `--chat` and `--serve`. `src/server.rs` is a single-threaded blocking `tiny_http`
+accept loop with no server-side conversation state; every handler consumes its `Request` exactly
+once. `/profile` exposes a rolling window of `PROFILE_TURNS` per-turn phase timings.
+
+## Conventions that matter here
+
+- **Module docs carry the reasoning.** Nearly every file opens with a long `//!` block explaining
+  *why* — what colibrì did, what was deliberately not ported and why it doesn't affect
+  correctness, what was measured, what was tried and reverted. This is the primary design record.
+  Read the module doc before editing a module; extend it when you change a decision.
+- **Provenance claims are explicit and literal.** Comments distinguish "confirmed against the
+  real checkpoint's `config.json`, fetched 2026-07-27" from a guess, and name the reference source
+  (`KimiBlockSparseMLP`'s own code comments, `fla.ops.kda.gate`'s installed source). Do not write
+  such a claim unless you actually verified it, and say which artifact you read.
+- **Correctness is validated against real reference code, never against intuition.** Reference
+  Python is vendored verbatim (`tests/oracle/*_pkg/`, `colibri_convert_fp8_to_int4.py`), not
+  reimplemented. New architecture math needs an oracle before it needs optimizing.
+- Test names are full sentences: `load_rejects_an_unrecognized_model_type_without_touching_either_loader`.
+- Errors are per-family enums with `Display`; the dispatch layer wraps rather than flattens them.
+- Some prose (`rabbit-plan.md`, older comment fragments) is in Spanish. Code and new comments are
+  English.
+
+## Docs in the repo
+
+- `README.md` — capabilities, measured numbers, what is explicitly *not* built yet.
+- `PERFORMANCE.md` — chronological log of every perf technique, **including reverted ones**, plus
+  the exact commands and hardware for reproducing each number. Append here after perf work;
+  record failures too.
+- `rabbit-plan.md` — phase-by-phase development history and the "out of scope" list (MTP /
+  speculative decoding, prefetch instrumentation, GPU).
+- `DASHBOARD_BRIEF.md` — build brief for a web UI over `/profile`; not implemented.
+- `docs/k3-architecture.html` — browsable Kimi K3 architecture explorer (open it directly, no
+  server). Layer stack → per-layer dataflow → KDA/MLA/LatentMoE internals, every tensor shape
+  derived from a config preset so the real 2.8T checkpoint and the tiny oracle render the same
+  diagrams. Its **Matmuls** tab enumerates every matmul in the model with `[N,K]` operands and
+  MACs/token, self-checked against the parameter count (they reconcile exactly, so a missing or
+  double-counted row shows up immediately). Model architecture only — no streaming or kernel
+  material. Two things to know before editing:
+  - **Its numbers are a second copy of the config.** If you change `src/kimi_k3/config.rs`'s
+    real-shape fixture or the oracle generator, update the `PRESETS` object in its `<script>`.
+  - It records the real `kda_layers`/`full_attn_layers` arrays, which **nothing in `src/` or the
+    test fixtures does** — the fixture scales to 4 layers and so cannot express them. The real
+    layout is every 4th layer plus layer 93, i.e. 69 KDA / 24 MLA, with 0-indexed layers 91 and
+    92 both MLA. `K3_OPTIMIZE_BRIEF.md:79`'s "≈ 3 : 1" is that, rounded.
+
+## Versioning
+
+Pre-1.0 `0.MINOR.PATCH`, tracked by git tags and `release/vX.Y.Z` branches. `Cargo.toml`'s
+`version` field is not the source of truth. MINOR bumps end a development phase; the release
+commit subject describes the phase (`v0.23.0: Kimi K3 as a third architecture, validated against
+the real 2.8T checkpoint`).
diff --git a/Cargo.toml b/Cargo.toml
index c416a3a..ea54816 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,3 +37,15 @@ harness = false
 [[bench]]
 name = "expert_load"
 harness = false
+
+# Added 2026-07-31 for the Kimi K3 target-box performance work (see PERFORMANCE.md's Phase 0
+# section). thin LTO + a single codegen unit let the optimizer inline across the hand-written
+# `is_x86_feature_detected!` kernel dispatchers and their `#[target_feature]` inner fns, which
+# 16 codegen units + no LTO (the compiler defaults this crate shipped with) leave on the table.
+# Deliberately NOT set: `panic = "abort"` (the `--serve` accept loop relies on unwinding a
+# panicked request thread without killing the process) and `target-cpu=native` (open decision D1:
+# kept out of the committed build so binaries stay portable across the owner's machines; measured
+# separately via RUSTFLAGS, see PERFORMANCE.md).
+[profile.release]
+lto = "thin"
+codegen-units = 1
diff --git a/K3_OPTIMIZE_BRIEF.md b/K3_OPTIMIZE_BRIEF.md
new file mode 100644
index 0000000..fd9db95
--- /dev/null
+++ b/K3_OPTIMIZE_BRIEF.md
@@ -0,0 +1,515 @@
+# Brief: Kimi K3 performance work (decode kernel, expert loading, parallelism, warmup)
+
+This is an implementation brief for an AI (or engineer) with **no prior context** on this
+project. It was written 2026-07-31 against commit `5853baa` (v0.23.0) after a live diagnosis
+session on the target machine. Read it fully before writing any code. Companion documents:
+`CLAUDE.md` (repo conventions — binding), `PERFORMANCE.md` (measurement history and method),
+`README.md` (capabilities), `DASHBOARD_BRIEF.md` (unrelated scope, but the genre precedent for
+this document).
+
+Every `file:line` anchor below was verified at `5853baa`. Line numbers drift; anchors are given
+as `path:line` plus a searchable symbol name. **If an anchor doesn't match what you find, stop
+and re-read the surrounding module doc before proceeding — do not guess.**
+
+---
+
+## STATUS UPDATE (2026-08-01) — this brief has been EXECUTED; read this before anything else
+
+All phases ran on the owner's instance (commits `a26e5bf`..`62238a1`, **local-only** — that box
+cannot push to the GitHub remote). This document is now the historical record of what was
+planned; `PERFORMANCE.md`'s "Kimi K3 on the target box" section is the record of what actually
+happened. Active follow-on work lives in `NUMA_AMX_BRIEF.md` (rev 2), which supersedes §8's
+Phase 6 and absorbs Phase 5's v2. What a new reader must know:
+
+- **Result: 36.8 → 16.9 s/token (2.18×)** on the real checkpoint at default placement/threads,
+  and **~1.42 s/token in live serving** once launched correctly (`numactl --interleave=all`,
+  `--expert-cache 896 --preload-experts --threads 48`) — the serve-config change alone was
+  worth ~4× over a default-cache launch.
+- **Phase 3's key finding supersedes this brief's own framing:** the AVX-512 kernel is 3–5× at
+  the kernel and ~0% end-to-end, because s=1 matmuls fork ~3072 one-row tasks and hit a
+  ~4.8 ms rayon fork/join floor. Decode is **scheduling-overhead-bound first**, bandwidth-bound
+  second. Phase 5 v1 only overlaps that floor across 16 experts; **v2 (D5) is now mandatory**
+  and is fused into `NUMA_AMX_BRIEF.md` Phase N3.
+- **Phase 6 ran as `numactl --interleave=all` only** (measured 1.4× + variance collapse on the
+  synthetic fixture) — the zero-code outcome this brief preferred. Compute affinity, the larger
+  remaining win, is `NUMA_AMX_BRIEF.md` N1–N3.
+- **Errata:** K5's "`get` takes `&mut self`" is wrong — `get` is `&self`; LRU stamping happens
+  in `begin_loading` (confirmed during Phase 5). 4d (`--drop-os-cache`) was deliberately
+  deferred, not built.
+
+---
+
+## 1. What this is about
+
+**rabbit** runs frontier open-weight MoE models on a single machine: the dense part stays
+resident in RAM (quantized), and the routed experts stream from disk through a per-layer LRU
+cache (`src/expert_cache.rs`). **Kimi K3** (`moonshotai/Kimi-K3`, 2.8T params, 896 experts/layer,
+16 active) is the third and newest supported architecture. Its bring-up was correctness-first
+(token-exact against Moonshot's own reference code, see `tests/teacher_forcing_k3.rs`); **zero
+performance work has landed for it**. The precedent: GLM-5.2 started from a similarly slow floor
+and got 3.5× faster across eight measured versions (`PERFORMANCE.md`).
+
+### The two machines
+
+| | dev laptop (PERFORMANCE.md's numbers) | **target box (this brief's goal)** |
+|---|---|---|
+| CPU | AMD Ryzen AI 9 HX 370, 12c/24t, AVX-512/VNNI | 2× Intel Xeon 6975P-C (Granite Rapids-AP, custom cloud SKU): 192 physical cores / 384 threads (SMT on), **6 NUMA nodes** (SNC3), full AVX-512 + **AMX** (tile/int8/bf16), KVM guest — details in Phase 0a |
+| RAM | 123 GB | **3 TB** (6 NUMA nodes × ~507 GB) |
+| K3 checkpoint | (validated, slow) | `/data/hf/hub/kimi-k3` (1.56 TB, 96 shards) |
+
+The target box changes the economics completely: the **entire routed-expert set (~1.32 TiB) fits
+in RAM**, so disk streaming — rabbit's whole reason to exist on the laptop — becomes a
+warmup-only concern, and the bottleneck moves to compute and memory bandwidth.
+
+### Observed baseline (target box, 2026-07-31, real checkpoint)
+
+```
+token 112/200 in 26.8s (5.3s in disk I/O [0.0s actual disk wait] this step;
+                        expert cache totals: 95825 hits, 75004 misses)
+```
+
+Run: `rabbit --model ... --prompt ... --expert-cache 64` (the default capacity). Breakdown:
+
+- **~21–22 s/token compute**, dominated by the scalar MXFP4 matmul kernel (§3, item K1).
+- **~4–6 s/token "disk I/O"** — actually `load_nanos`: sequential, single-threaded expert
+  loads (§3, item K4). The bracketed `[0.0s actual disk wait]` is **structurally always zero
+  for K3** and means nothing (§3, item K3). Do not interpret it.
+- Hit rate 56% at capacity 64. Hits+misses grow by **exactly 1472/token** = 16 experts × 92
+  MoE layers (shared experts never touch the cache — they're resident dense weights).
+- Effective compute rate: ~170 GFLOP/token ÷ 21 s ≈ **8 GFLOP/s aggregate across 192 cores**
+  (~42 MFLOP/s/core). Plain scalar FMA should manage 1–3 GFLOP/s *per core*. The gap is the
+  work of this brief.
+
+Dev-box reference points (README): model load 610 s, prefill (7 tok) 412.8 s, decode 50–70 s/tok.
+
+### Goal
+
+Decode on the target box from ~27 s/token to **low single-digit seconds per token or better**,
+with warmup measured in minutes instead of hours, without breaking token-exact correctness for
+any architecture. Once compute is fixed, decode becomes memory-bandwidth-bound: ~26 GB of expert
+weights must cross the memory bus per token no matter what. Phase 0 measures the real bandwidth;
+until then the floor estimate (~0.025–0.06 s/token — Granite Rapids-AP carries 12 DDR5
+channels/socket, ~1 TB/s-class aggregate) is an **estimate — do not present it as a promise**. Reaching it likely needs the follow-on items in §9 (not in scope).
+
+---
+
+## 2. Architecture numbers (verified)
+
+Config constants asserted against the real checkpoint's `config.json` in
+`src/kimi_k3/config.rs:245-307` (`loads_the_real_checkpoints_shape`):
+
+| constant | value |
+|---|---|
+| `hidden_size` | 7168 |
+| layers | 93 total; `first_k_dense_replace: 1` → 1 dense + 92 MoE |
+| attention layout | KDA : full MLA ≈ 3 : 1 (`kda_layers`/`full_attn_layers` arrays) |
+| `num_experts` / `num_experts_per_token` / `num_shared_experts` | 896 / 16 / 2 |
+| `moe_intermediate_size` | 3072 |
+| `routed_expert_hidden_size` (latent width) | 3584 |
+| `vocab_size` | 163,840 |
+| expert quantization | OCP MXFP4 (`quantization_config.format: "mxfp4-pack-quantized"`, group 32) |
+
+Derived (arithmetic, not measured):
+
+- **One routed expert** = gate `[3072,3584]` + up `[3072,3584]` + down `[3584,3072]`
+  = 33,030,144 params. On disk/in cache as MXFP4: 16,515,072 B data + 1,032,192 B scale
+  = **16.73 MiB/expert**.
+- **Per decoded token**: 1472 expert applications, 48.6 G routed params ≈ 97 GFLOP routed,
+  ≈ 170 GFLOP total (latent down/up, shared experts, attention, lm_head). **~25.8 GB** of
+  expert weight bytes traversed.
+- **Cache footprint**: capacity 64 (default, `src/main.rs:58`) ≈ 96 GiB; capacity 896 = full
+  residency ≈ **1.32 TiB** experts + ~40 GB dense (at default `dbits=4`).
+
+K3's routed experts run at the **latent width** (3584), not `hidden_size` — routing scores use
+the full-width hidden states, expert compute happens after a down-projection. This is why K3 has
+its own dispatch function instead of reusing `glm52::moe::moe()` (see K2 below). The two widths
+are carried as two `Cfg` values (`cfg_full` / `cfg_expert`) differing in `.hidden`.
+
+---
+
+## 3. Verified code map — read these before touching anything
+
+**K1 — the scalar kernel (the main cost).** `matmul_mxfp4`, `src/kernels.rs:222-242`. Inner
+loop per element: byte load, `k & 1` parity branch, nibble mask/shift, `e2m1_decode(nibble)`
+(already a table lookup + sign flip, `src/quant.rs:285-288`), and `e8m0_decode(bs[k / 32])`
+**recomputed for every element instead of once per 32-block**. Scalar only — the module doc at
+`src/kernels.rs:216-221` says so and why ("no real MXFP4 checkpoint to benchmark against yet"
+— no longer true). Dispatched from `matmul_qt` at `src/kernels.rs:938`.
+
+**K2 — K3's own dispatch loop (the restructure target).** `latent_moe`,
+`src/kimi_k3/moe.rs:64-117`. Routing at full width → `record_selection` → down-proj →
+chunked `ensure_loaded` → **sequential** `for &eid in chunk { apply_single_expert(...) }` →
+optional RMSNorm → up-proj. Fully synchronous load-then-apply; **no early drain, no io_uring
+here**. Its module doc explicitly says the overlap machinery was deliberately deferred. This
+means: (a) the restructure never touches `glm52::moe::moe()` (GLM/Kimi-Linear's perf-tuned hot
+path — **out of scope, do not modify it**); (b) warm-cache K3 decode is already deterministic,
+so a bit-identical acceptance gate works (§8, Phase 5).
+
+**K3 — the meaningless counter.** `io_wait_nanos` is incremented in exactly one place, the
+io_uring completion path (`src/expert_cache.rs:754-755`). MXFP4 naming **never gets a ring**
+(`ring: None`, `src/expert_cache.rs:610-615`, reason in `ExpertNaming::is_mxfp4`'s doc), so for
+K3 the CLI's `[Xs actual disk wait]` (printed at `src/main.rs:151,156`) is always 0.0.
+
+**K4 — sequential loads.** `sequential_fallback`, `src/expert_cache.rs:855-857`:
+`misses.iter().map(|&eid| load_expert(...)).collect()` — one expert at a time, one thread.
+`load_expert` → `qt_load_mxfp4` (`src/expert_cache.rs:884-900`) does two `read_raw(name,
+false)` calls (packed + scale; `false` = no `POSIX_FADV_DONTNEED` afterward) with byte-count
+shape checks, then moves the buffers into `QTKind::MxFp4` **with no transcode** — a K3 expert
+"load" from page cache is nearly pure memcpy. `Shards::read_raw`/`read_at` are `&self` and
+thread-safe (`pread` via `FileExt::read_at`; **never mmap** — see `src/safetensors.rs` module
+doc for why that rule exists; it is binding).
+
+**K5 — cache mechanics.** `ExpertCache` (`src/expert_cache.rs:522+`): per-layer, LRU at
+`capacity`, plus a `pinned` tier fed lazily from the `.rabbit_usage` histogram
+(`warm_start` → `mark_pin_candidates` → `insert_or_pin` promotes on first real load, `mlock`
+best-effort). At `capacity == n_experts` eviction is impossible and pinning is pointless.
+`get` stamps LRU recency (takes `&mut self`) — relevant borrow puzzle in Phase 5.
+
+**K6 — thread pool.** `configure_thread_pool`, `src/main.rs:286-293`: default =
+`num_cpus::get_physical()`. The SMT-off default was measured on the **12-core laptop**
+(bandwidth-bound there); do not assume it transfers to 192 cores — Phase 0 sweeps it.
+
+**K7 — build profile.** `Cargo.toml` has **no `[profile.release]` section at all**: defaults
+(no LTO, 16 codegen units) and — unless `RUSTFLAGS` says otherwise — a generic x86-64 baseline
+for everything outside the `is_x86_feature_detected!` hand-written kernels.
+
+**K8 — existing MXFP4 test coverage.** `src/kernels.rs:1202`
+(`matmul_mxfp4_matches_manual_e2m1_decode_across_two_blocks`) and `src/kernels.rs:1384`
+(`matmul_qt_mxfp4_matches_manual_dequant_dot_product`); E2M1/E8M0 codec tests in
+`src/quant.rs:777+`. **No criterion bench entry exists for MXFP4** in `benches/kernels.rs`.
+Critically: **the tiny teacher-forcing oracle does NOT exercise `matmul_mxfp4`** — oracle
+fixtures use plain float `.weight` tensors (naming `KimiK3`, not `KimiK3Mxfp4`). The kernel
+unit tests + the real checkpoint are the only MXFP4 correctness coverage. Extend the unit
+tests; do not lean on the oracle for kernel changes.
+
+**K9 — measurement harnesses.**
+- `examples/teacher_forced_decode_bench.rs` — THE decode timing tool. Feeds a fixed token
+  sequence (free-running greedy is not run-to-run reproducible on the GLM path by documented
+  design; use this harness for all decode timing). **Requires tokenizer files in the model dir**
+  (`Tokenizer::load`, line 51).
+- `examples/k3_smoke.rs` — load/prefill/decode structural probe on **raw token ids**
+  (`--prompt-len`), no tokenizer needed.
+- `benches/kernels.rs` — criterion, kernel-level.
+- `tests/teacher_forcing_k3.rs` — token-exact architecture correctness vs the tiny oracle
+  (fixtures gitignored; regenerate via `tests/oracle/make_k3_oracle.py`, Docker-based — read
+  its docstring). **Skips, not fails, when fixtures are absent — a green `cargo test` does not
+  mean it ran. Check for SKIP lines.**
+- `Shards::open` scans `*.safetensors` in the directory, **sorted by filename, no
+  `index.json` needed** (`src/safetensors.rs:211,235`) — the Phase 1 generator relies on this.
+
+---
+
+## 4. Invariants and house rules (binding)
+
+1. **`cargo test` fully green at every phase**, including regenerated oracle fixtures where a
+   phase could affect them (state in the PR/commit message whether oracles actually ran or
+   skipped).
+2. **Exactness contracts are per-path and load-bearing** (`src/kernels.rs` module doc):
+   integer IDOT tiers must agree bit-for-bit; float paths with reassociation (AVX-512
+   `matmul_i4` precedent) are within-tolerance only. New MXFP4 tiers follow the same split:
+   Phase 2's scalar refactor must be **bit-exact**; Phase 3's AVX-512 tier is
+   **within-tolerance** with a documented test tolerance.
+3. **`pread`, never mmap** (`src/safetensors.rs` module doc — RSS accounting is a design
+   pillar).
+4. **Module docs carry the reasoning.** Every touched module's `//!` block gets updated when a
+   decision it documents changes. Provenance discipline: never write "confirmed against X"
+   unless you actually read X; name the artifact.
+5. **No new dependencies** without the kind of justification comment `Cargo.toml` already
+   models (`fancy-regex`, `base64`). NUMA work uses `libc` or external `numactl` — no `hwloc`,
+   no `numa` crates.
+6. **`PERFORMANCE.md` gets a dated section per phase** — the command, the hardware, the
+   before/after numbers, and **techniques that didn't help** (the "Tried and didn't help"
+   section is a deliberate part of the file's value).
+7. **Scope discipline — do not touch:** `glm52::moe::moe()` and its early-drain/io_uring
+   streaming path; MLA/KDA/attention math; the checkpoint converters; `kv_session` formats;
+   versioning (no tags or `release/` branches — the owner cuts releases). One commit per phase,
+   descriptive message.
+8. Comments in English; test names are full sentences; match the existing wide-line rustfmt
+   style of the file you're in.
+
+---
+
+## 5. Measurement protocol
+
+- **Decode timing**: `teacher_forced_decode_bench` only. Before/after via `git worktree` at
+  the pre-phase and post-phase commits, identical command lines — the exact method
+  `PERFORMANCE.md`'s "Reproducing these numbers" documents.
+- **Canonical target-box command** (after Phase 1, substitute the synthetic dir for iteration):
+
+  ```
+  cargo run --release --example teacher_forced_decode_bench -- \
+      --model /data/hf/hub/kimi-k3 --steps 30 --expert-cache 896
+  ```
+
+- **Warm/cold discipline**: run everything twice; report both, label them. First run warms the
+  page cache and the expert cache; steady-state claims come from run 2.
+- **Kernel timing**: `cargo bench --bench kernels -- mxfp4` (entries added in Phase 2).
+- Record `RUSTFLAGS`, thread count, `numactl` invocation, and THP state with every number.
+
+---
+
+## 6. Phase 0 — Baseline + build configuration (no logic changes)
+
+**0a. Record the target box.** Topology was captured 2026-07-31 (owner-provided `lscpu` +
+`numactl --hardware`); paste the raw output into the new `PERFORMANCE.md` section verbatim.
+Summary of what it shows:
+
+- 2× Intel Xeon 6975P-C (Granite Rapids-AP, custom cloud SKU), 96 cores/socket, SMT on →
+  192 physical / 384 logical CPUs. KVM guest (cloud instance).
+- **6 NUMA nodes** — SNC3, 3 sub-NUMA domains per socket; 64 logical CPUs + ~507 GB RAM per
+  node. Distances: 10 local, 15–17 intra-socket, 21–28 cross-socket; socket 0 = nodes 0–2,
+  socket 1 = nodes 3–5.
+- Full AVX-512 (F/DQ/BW/VL/VNNI/VBMI2/BF16/FP16) and **AMX** (`amx_tile`/`amx_int8`/
+  `amx_bf16`) — Phase 3 and §9's AMX follow-on are both hardware-supported.
+- A live free-memory imbalance was captured while a run was resident (node 2: 19 GB free vs
+  node 0: 467 GB) — first-touch concentration observed in the wild, exactly what Phase 6
+  targets. Note that 1.32 TiB of expert slots cannot fit in one 507 GB node: without
+  deliberate placement, allocations spill node-to-node in load order.
+
+Still to record: a memory-bandwidth number (any STREAM-triad binary; record which — Granite
+Rapids-AP has 12 DDR5 channels/socket, so ~1 TB/s-class aggregate is plausible), THP state
+(`cat /sys/kernel/mm/transparent_hugepage/enabled`), and the guest kernel (`uname -r`).
+
+**0b. Baseline runs** (real checkpoint, current `main`): the canonical command at
+`--expert-cache 64` and `896`, plus `k3_smoke` for load/prefill numbers.
+
+**0c. Experiments, zero code** — each one canonical-command before/after:
+- `--threads` sweep: 48 / 96 / 192 / 384 (SMT is on; 192 = the physical-core default).
+- `numactl --interleave=all` vs default.
+- Single-socket control run: `numactl --cpunodebind=0-2 --membind=0-2` with `--threads 96` —
+  halves peak bandwidth but removes all cross-socket traffic; tells you what the 21–28
+  cross-socket distances actually cost.
+- THP `always` vs `madvise` (system toggle).
+
+**0d. Build profile.** Add to `Cargo.toml`:
+
+```toml
+[profile.release]
+lto = "thin"
+codegen-units = 1
+```
+
+Do **not** add `panic = "abort"` (the `--serve` accept loop's failure mode changes) and do
+**not** commit `target-cpu=native` anywhere (open decision D1) — instead measure
+`RUSTFLAGS="-C target-cpu=native"` as part of 0c and document the result.
+
+**Gate:** numbers recorded in `PERFORMANCE.md`; `cargo test` green; no behavior change beyond
+the profile.
+
+---
+
+## 7. Phase 1 — Synthetic K3-at-scale fixture (test infrastructure)
+
+Real-checkpoint iteration costs a 1.56 TB load per experiment. Decode cost is per-layer
+homogeneous, so a random-weight checkpoint at **real per-layer width** but few layers
+reproduces kernel/parallelism/NUMA behavior at ~5–10% footprint.
+
+**Build** `src/bin/gen_k3_synth.rs` (or an example — implementer's call): writes a loadable
+K3-shaped checkpoint:
+
+- `config.json`: adapt the template in `src/kimi_k3/config.rs:249-276` (it is condensed from
+  the real checkpoint) — parameterized `num_hidden_layers` (default 6: layer 0 dense, then a
+  3:1 KDA:full-MLA pattern), all K3 fields on, `quantization_config` present so
+  `mxfp4_experts=true` selects `ExpertNaming::KimiK3Mxfp4`.
+- All tensor names under the `language_model.` prefix (see `ExpertNaming::KimiK3Mxfp4`'s doc
+  and the fixture in `src/model.rs`'s tests for the dense-name pattern).
+- Routed experts: real shapes (`[3072,3584]`×2 + `[3584,3072]`) as U8
+  `{...}.w?.weight_packed` + `.weight_scale` pairs with the exact byte counts
+  `qt_load_mxfp4` checks (`rows*ceil(cols/2)`, `rows*ceil(cols/32)`). Keep E8M0 scale bytes in
+  ~[120,130] so dequantized magnitudes stay sane — logits are garbage by design, but NaN/inf
+  poisoning is avoidable noise.
+- Dense/KDA/attention tensors at real dims, F32 or BF16 (D6). One shard file per layer;
+  `Shards::open` needs no index (K9). At 6 layers expect ~90–100 GB.
+- Reuse `glm52::convert::writer::write_safetensors` if its API fits raw-U8 output; if not,
+  hand-roll the header+bytes (the format is trivial — see `write_safetensors` test helpers in
+  `src/model.rs`). Don't force the reuse.
+- For `teacher_forced_decode_bench` (needs a tokenizer, K9): fetch the real tokenizer files
+  once via `python3 tools/fetch_k3_tokenizer_fixture.py` and copy
+  `tiktoken.model`+`tokenizer_config.json` into the synthetic dir. `k3_smoke` needs nothing.
+
+**Gate:** `rabbit --model <synth>` loads and decodes; `k3_smoke` and the bench run; per-token
+decode time ≈ (6/92) × the real checkpoint's per-token time within reasonable noise — if the
+extrapolation is badly off, that's a finding to record, not to hide.
+
+---
+
+## 8. The main work
+
+### Phase 2 — Scalar MXFP4 kernel hygiene (bit-exact)
+
+In `matmul_mxfp4` (K1): restructure the inner loop to iterate 32-element scale blocks —
+decode `e8m0_decode(bs[b])` **once per block**; within a block iterate 16 packed bytes
+decoding both nibbles (kills the `k & 1` branch). Keep the per-element multiply order
+`x * e2m1 * scale` exactly as today: same inputs to the same decode functions in the same
+per-element order → **bit-identical output**. Do not introduce per-block partial sums in this
+phase (that reassociates — it belongs behind the tolerance-tested AVX-512 tier).
+
+Also add criterion entries to `benches/kernels.rs` at real dims (3072×3584 and 3584×3072,
+s=1 and s=8) — they're the before/after instrument for this phase and the next (K8: none
+exist today).
+
+**Gate:** K8's two kernel tests green unchanged; a new test pinning bit-exactness vs a
+straight-port reference copy of the old loop (kept inside `#[cfg(test)]`); bench delta
+recorded. Expected impact: 1.5–4× on the kernel — unknown until measured; record whatever it
+is, including "less than hoped".
+
+### Phase 3 — AVX-512 tier for `matmul_mxfp4`
+
+The format is nearly designed for AVX-512: **one 32-element scale block = 16 packed bytes = two
+16-lane f32 vectors**, and E2M1 has exactly 16 code points → the whole decode table fits one
+`zmm` for `_mm512_permutexvar_ps`.
+
+Per output row: iterate blocks; load 16 bytes; unpack low/high nibbles to two i32 vectors;
+`permutexvar_ps` against the E2M1 table (sign handled by the table itself — 16 entries cover
+sign×magnitude); FMA against `x`; multiply the block's E8M0 scale (scalar-decoded per block,
+broadcast) into the block's contribution; two independent accumulator chains + tree reduction
+(the exact pattern of `dot_i4_f32_avx512`, which the module doc documents as the precedent for
+"more accurate but not bit-identical"). Handle the `cols % 32` tail scalar.
+
+Runtime selection: same ladder as `matmul_i4` (`is_x86_feature_detected!` at dispatch,
+`#[target_feature(enable = ...)]` unsafe inner fn, `pub use` the tier for benches). No AVX2
+tier unless it falls out for free (D4).
+
+**Gate:** within-tolerance parity tests vs scalar across random matrices including edge dims
+(cols not divisible by 32/64, odd rows), tolerance chosen and documented the way the
+`matmul_i4` tiers' tests do; kernel bench delta; **real-checkpoint sanity on the target box**
+(same prompt, coherent answer — note in `PERFORMANCE.md` that reassociation may legitimately
+flip a near-tie argmax; that is expected, not a bug); end-to-end decode delta via the canonical
+command. Expected: this is the big one — kernel should go from ~8 GFLOP/s-class to
+memory-bound; decode into low single digits of seconds. Measure, don't assume.
+
+### Phase 4 — Expert loading: parallel misses, preload, honest logging
+
+**4a. Parallelize `sequential_fallback`** (K4): `misses.par_iter().map(load_expert).collect()`
+— rayon's ordered collect preserves input order, so cache insertion order (and pin promotion,
+K5) is unchanged and results stay deterministic. `Shards` reads are `&self`/`pread`,
+thread-safe (K4). This collapses K3's 4–6 s/token warmup cost and — as a side effect —
+distributes NUMA first-touch of expert buffers across nodes.
+
+**4b. `--preload-experts` flag.** New CLI flag (`src/main.rs` parse + `USAGE`), threaded
+through `chat::LoadArgs` (`src/chat.rs`) into session setup: after `ExpertCaches::new` +
+`warm_start`, load every MoE layer's experts up front — chunked per layer, parallel within
+chunk (4a's machinery), one progress line per layer. Dispatch through a new method on
+`crate::model::ExpertCaches` with per-family arms (all three families get it — it's generic).
+Semantics at `capacity < n_experts`: fill to capacity in usage-histogram order when
+`.rabbit_usage` exists, else expert-id order, and say so in the log (D2). Applies to `--serve`
+too (same `Session` path). Expected: full 1.32 TiB preload in minutes (parallel reads) versus
+hundreds of slow-warmup tokens; token 1 runs at steady-state speed.
+
+**4c. Honest logging** (K3): when the expert path has no io_uring ring (MXFP4), stop printing
+`[0.0s actual disk wait]` — print nothing or `[disk wait n/a]`. The counter misled the owner
+within the first minutes of the first real run; that's a bug in communication if not in code.
+
+**4d. (Flagged, default off — D3) `--drop-os-cache`:** pass `drop_cache: true` through the
+expert-load `read_raw` calls when set, using the existing `fadvise_dontneed` machinery, to
+avoid double-residency (1.32 TiB of cache slots + the page cache's copy of the same 1.56 TB
+checkpoint on a 3 TB box). Tradeoff to document: with it, restarts re-read from disk; without
+it, the kernel reclaims page cache under pressure anyway. Off by default.
+
+**Gate:** oracle tests green (loads produce identical bytes; only concurrency changed); warmup
+wall-time before/after on the synthetic fixture and, once, on the real checkpoint; preload
+timing recorded; no `io_wait` regression lies in the log.
+
+### Phase 5 — Across-expert parallelism in `latent_moe` (K2)
+
+Today (post-Phase-3) each of the 16 experts' three matmuls runs one after another, each matmul
+internally fanning out across all 192 cores (~16–19 rows/thread) and joining — ~48
+fork/join regions per MoE layer, ~4,400 per token, each hammering one expert's
+single-NUMA-node weights with every core.
+
+**v1 (mandatory):** in `latent_moe`'s apply loop — after `ensure_loaded`, collect the chunk's
+resident `&ExpertSlot`s, then `par_iter` over experts, each task computing its expert's full
+gate→activation→down chain into a **per-expert output buffer** (`s × 3584` f32 — 14 KB at
+decode; reuse `apply_single_expert`'s body refactored to write into a caller-supplied buffer
+instead of `+=` into shared `routed`), then reduce the buffers into `routed` **sequentially, in
+the exact order today's loop applies experts** (chunk iteration order). Per-expert math is
+unchanged and reduction order is unchanged → **bit-identical to current output**. Nested rayon
+(outer per-expert tasks, inner `par_chunks_mut` in the matmuls) shares one pool — no
+oversubscription; work-stealing balances it.
+
+Borrow note (K5): `cache.get(eid)` takes `&mut self` (LRU stamp). Two passes: stamp/collect
+indices mutably, then take immutable slot refs via a `peek`-style accessor (add one if
+needed) before the parallel region. Keep `record_selection`/usage semantics untouched.
+
+**v2 (conditional — D5):** only if profiling on the target box shows v1 leaves cores idle:
+flatten to (expert × row-block) tasks with row-range kernel entry points. Do not build this
+speculatively.
+
+**Gate:** `teacher_forcing_k3` (this path IS exercised by the tiny oracle — the fixtures'
+plain-float naming still flows through `latent_moe`) plus a new determinism test on any
+fixture: two identical warm runs produce bit-identical logits. Decode delta on the synthetic
+fixture at 192 cores, then the real checkpoint. GLM-5.2/Kimi-Linear timings must be untouched
+(their path wasn't modified — verify with one bench run anyway).
+
+### Phase 6 — NUMA (conditional)
+
+Only if Phase 0c's interleave experiment moved the number materially. The box is 6-node SNC3
+(Phase 0a), and placement granularity is naturally per-expert: one expert (16.73 MiB) sits
+entirely on one node, so a preload (4b) that round-robins experts across nodes yields an even
+spread with no allocator tricks. Prefer, in order: documented `numactl` invocation (zero
+code); first-touch distribution already gained from Phases 4a/4b (parallel loads spread
+first-touch across the threads that load each expert); per-buffer `libc::madvise`/`mbind` as
+a last resort with a measured justification. No new crates (rule 5).
+
+---
+
+## 9. Follow-on (documented for the owner — NOT in scope for this brief)
+
+- **Prefill/batch integer path + AMX.** `apply_single_expert` already batches all tokens
+  routed to an expert into one `nr`-row matmul (verified) — prefill's structure is right; it
+  lacks a fast batched MXFP4 kernel. The codebase's own threshold (`I4_IDOT_MIN_S = 2`,
+  `src/kernels.rs`) says int8-activation IDOT pays at S≥2; VNNI tiers exist for int4/int8. An
+  MXFP4→int8 IDOT path, then an AMX tile kernel, is the natural sequel — it's the workload a
+  192-core AMX box is shaped for. (AMX does not help batch-1 decode: one activation row leaves
+  ~15/16 of the tile idle.)
+- **Speculative decoding / MTP** — the only lever that beats the per-token weight-traffic
+  floor. First step is a 5-minute check: grep the real checkpoint's
+  `model.safetensors.index.json` for MTP-head tensors.
+- **io_uring-batched MXFP4 loads** (teach the ring the packed+scale pair) — matters for
+  disk-bound hosts, not the 3 TB box.
+- **Converter MXFP4-input support** (`convert_shard` currently has none — its `read_f32`
+  path doesn't know `.weight_packed`/`.weight_scale` pairs), enabling a one-time int4
+  conversion as an alternative decode path, and a `.qs`-style pre-quantized dense sidecar to
+  cut the 610 s model load.
+- **Carving + determinism for `glm52::moe::moe()`** (the early-drain path) — same idea as
+  Phase 5 but entangled with io_uring streaming; separate design needed.
+
+---
+
+## 10. Open decisions (resolve explicitly — with the owner if it changes behavior)
+
+- **D1** Commit `target-cpu=native` (e.g. `.cargo/config.toml`)? Recommendation: no — document
+  in `PERFORMANCE.md`; binaries stop being portable across the owner's machines otherwise.
+- **D2** Preload semantics when `capacity < n_experts`. Recommendation: fill to capacity,
+  usage-order when available, warn loudly.
+- **D3** `--drop-os-cache` default. Recommendation: off.
+- **D4** AVX2 MXFP4 tier. Recommendation: skip — both known machines have AVX-512; the module
+  doc's own precedent is tiers-on-measured-need.
+- **D5** Phase 5 v2 trigger. Recommendation: only on target-box profiling evidence of idle
+  cores after v1.
+- **D6** Synthetic fixture dense dtype (F32 simplicity vs BF16 size) and default layer count
+  (recommendation: 6).
+
+---
+
+## 11. Definition of done
+
+Per phase: code + tests + module-doc updates + a dated `PERFORMANCE.md` section (command,
+hardware block, before/after, surprises, failures) + one commit. Overall: baseline vs final
+canonical-command numbers on the target box at `--expert-cache 896` (warm), `cargo test` green
+with oracle fixtures present and actually run (state it), GLM-5.2/Kimi-Linear timings
+demonstrably unregressed, and §9 untouched.
+
+## 12. Known traps (read twice)
+
+1. Fixture-dependent tests **skip silently** when fixtures are absent (K9). Green ≠ ran.
+2. The tiny K3 oracle does **not** exercise the MXFP4 kernel (K8) — kernel changes live and
+   die by the `kernels.rs` unit tests and the real checkpoint.
+3. Free-running greedy decode is **not** a valid timing comparison on this codebase — use the
+   teacher-forced harness (K9; the harness's module doc explains why).
+4. `[0.0s actual disk wait]` on K3 stdout is structurally zero, not evidence of a warm disk
+   path (K3) — Phase 4c removes the trap.
+5. Laptop-derived defaults (SMT-off thread count, `I4_IDOT_MIN_S`, io_uring-slower-than-pread
+   bench notes) encode **12-core measurements**. Re-measure before trusting any of them on 192
+   cores; don't silently change them either — measure, then change with the number in hand.
+6. Line numbers in this brief were valid at `5853baa`; symbols are the durable anchors.
diff --git a/NUMA_AMX_BRIEF.md b/NUMA_AMX_BRIEF.md
new file mode 100644
index 0000000..c3c6cbb
--- /dev/null
+++ b/NUMA_AMX_BRIEF.md
@@ -0,0 +1,456 @@
+# Brief: NUMA-affine execution + AMX follow-on for Kimi K3 on the target box
+
+This is an implementation brief for an AI (or engineer) with **no prior context** on this
+project, in the same genre as `K3_OPTIMIZE_BRIEF.md` (read that first — it is this document's
+prerequisite, and its §1–§4 context is not repeated here). It was written 2026-08-01 against
+commit `5853baa` (v0.23.0), from a design session with the owner on the dev Mac. Companion
+documents: `CLAUDE.md` (repo conventions — binding), `PERFORMANCE.md` (measurement history),
+`K3_OPTIMIZE_BRIEF.md` (the preceding perf brief; its rules §4 and measurement protocol §5 are
+binding here too).
+
+Every `file:line` anchor below was re-verified at `5853baa` while writing this document. If an
+anchor doesn't match what you find, stop and re-read the surrounding module doc — do not guess.
+
+**REV 2 (2026-08-01, later the same day):** `K3_OPTIMIZE_BRIEF.md`'s Phases 0–6 have since
+been EXECUTED on the instance (commits `a26e5bf`..`62238a1`; results in `PERFORMANCE.md`'s K3
+section — read it before this document). All prerequisites are met and **this brief is now the
+active plan.** Its predecessor's Phase 6 ran as `numactl --interleave=all` only (measured 1.4×
+on the synthetic fixture); the compute-affinity half (N1–N3 here) is unbuilt. One structural
+change in rev 2: `K3_OPTIMIZE_BRIEF.md` Phase 5's deferred **v2 is fused into Phase N3** (§7)
+— the two are one restructure, do not build them separately. A second: attention work is
+promoted to its own **Phase N5** (§9), on the strength of the live serving profile; AMX moves
+to N6.
+
+---
+
+## 1. The design principle (owner's directive)
+
+> All tensors involved in one "computation group" live on one NUMA node, and the compute for
+> that group runs on that node's cores.
+
+Two clarifications agreed with the owner, binding on the implementation:
+
+1. **This is bandwidth locality, not cache locality.** In decode every weight byte is read
+   exactly once per token — there is no temporal reuse for caches to exploit, and adjacency
+   between different tensors is irrelevant at these sizes (individual matrices are tens–hundreds
+   of MB; L2 is 2 MB/core). What placement buys is *concurrent use of all 6 nodes' DRAM
+   channels*: an expert whose 16.73 MiB lives on one node serves the whole 192-core machine at
+   that one node's ~1/6 share of aggregate bandwidth, with 5/6 of the traffic additionally
+   crossing SNC/UPI boundaries. Affinity (compute where the data is) is what converts placement
+   into bandwidth.
+2. **The schedulable granularity is 6 nodes** (or 2 sockets), not 96 heads. 96-head layers map
+   to 16 heads per node; the 16 selected experts map to ~2.7 experts per node on average.
+
+### What execution measured (instance, 2026-08-01) — binding context for every phase below
+
+1. **Decode is scheduling-overhead-bound before it is bandwidth-bound.** The AVX-512 MXFP4
+   kernel came out 3–5× faster at the kernel and ~0% faster end-to-end: an s=1 matmul forks
+   ~3072 one-row tasks and hits a ~4.8 ms rayon fork/join floor that both tiers share
+   (`PERFORMANCE.md` Phase 3). Removing that floor (v2's row-block tasks) and adding affinity
+   (N3) are the same restructure — hence their fusion in §7.
+2. **Bandwidth still matters underneath:** `numactl --interleave=all` alone was 1.4× with
+   variance collapse (synthetic fixture, `PERFORMANCE.md` Phase 6) — which also proves the
+   guest's NUMA topology has real teeth: a decorative topology cannot produce a placement
+   effect.
+3. **Live serving baseline** (real checkpoint, interleave + `--expert-cache 896
+   --preload-experts --threads 48`; `/profile` misses frozen at 82,432 ⇒ the expert bucket is
+   pure compute): **1.42 s/token wall** on a 128-token turn — expert matmul 0.83 s/token
+   (59%), attention 0.53 s/token (37%), lm_head ~0.01 s. Two conclusions: the expert bucket
+   moves 25.8 GB/token at only ~31 GB/s effective against a machine-class ~600 GB/s — **an
+   order of magnitude for N3 to claim** — and attention costs 0.53 s/token at near-zero
+   context (≤180 tokens; the MLA scan is trivial there), i.e. it is KDA-recurrence/scheduling
+   cost, not KV traffic. **Once N3 lands, attention is ~80% of every token** — that is Phase
+   N5's justification.
+
+### The box (from `K3_OPTIMIZE_BRIEF.md` Phase 0a — owner-provided `lscpu`/`numactl`, not re-verified here)
+
+2× Intel Xeon 6975P-C (Granite Rapids-AP), 192 physical cores / 384 threads, **6 NUMA nodes**
+(SNC3: 3 sub-NUMA domains per socket, 64 logical CPUs + ~507 GB each, 3 TB total). Distances:
+10 local, 15–17 intra-socket, 21–28 cross-socket. Full AVX-512 + AMX (`amx_tile`/`amx_int8`/
+`amx_bf16`). **KVM guest** — see the go/no-go probe in Phase N0; if the guest's NUMA topology
+is not backed by real host pinning, most of this brief is void.
+
+**Provenance caveat (rev 2):** the owner is not certain every `PERFORMANCE.md` number came
+from one and the same machine. Do not inherit this topology on trust: run `lscpu` and
+`numactl --hardware` on the box you are actually on, record them with your numbers, and derive
+every node/core count from `numa::topology()` at runtime — nothing in the code may hardcode 6
+nodes or 192 cores.
+
+### Where the traffic is (derived in `K3_OPTIMIZE_BRIEF.md` §2)
+
+~26 GB of weight bytes cross the memory bus per decoded token; **~25.8 GB of that is routed
+experts**. Dense weights (attention, shared experts, latent down/up, lm_head) are ~40 GB
+resident but only a fraction is read per token; MLA KV caches and KDA states add hundreds of
+MB/token at long context. Priority order follows the traffic: experts first (Phase N3), dense
+row-sharding second (N4), state/cache placement last (N4c).
+
+---
+
+## 2. Verified code map
+
+All line numbers checked at `5853baa` on 2026-08-01.
+
+- **Thread pool.** One global rayon pool, no affinity anywhere: `configure_thread_pool`,
+  `src/main.rs:286-293` (`num_threads(n).build_global()`, default = physical cores).
+- **Matmul parallelism.** Every `matmul_*` fans output rows across the whole global pool:
+  `yt.par_chunks_mut(s)` sites throughout `src/kernels.rs` (e.g. `:103`, `:653`, `:885`);
+  layout rationale in the module doc at `src/kernels.rs:32-38`. Dispatch: `matmul_qt`,
+  `src/kernels.rs:921`.
+- **QT storage.** `src/quant.rs:52-83`: each weight is ONE contiguous allocation per tensor
+  (`QTKind::{F32,I8,I4,I4Grouped,MxFp4}`, `Vec`-backed). MXFP4: `data.len() =
+  rows*ceil(cols/2)`, `block_scale.len() = rows*ceil(cols/32)`, allocated in `alloc_mxfp4`
+  (`src/quant.rs:106-113`). Row-major, so a contiguous **row range** of any QT is a contiguous
+  byte range in each of its buffers — this is what makes per-node sharding clean (N4).
+- **Experts.** `ExpertSlot { eid, gate_up, down, used }`, `src/expert_cache.rs:473-478`;
+  `ExpertCache`, `src/expert_cache.rs:526-578` (per-layer LRU + pinned tier);
+  `ensure_loaded(&mut self, ...)`, `src/expert_cache.rs:671`. **`get` is `&self` and
+  read-only** (`src/expert_cache.rs:779`) — note this contradicts `K3_OPTIMIZE_BRIEF.md`'s K5
+  ("`get` stamps LRU recency, takes `&mut self`"); the code is what I verified, and it means
+  collecting `&ExpertSlot` refs for a parallel region needs no borrow gymnastics. MXFP4 loads:
+  `qt_load_mxfp4`, `src/expert_cache.rs:884` — allocation and first write (= NUMA first touch)
+  happen on the **calling thread**, inside `Shards::read_raw`. Whichever thread runs an
+  expert's load decides where its pages land. This is the placement mechanism for N3.
+- **K3 expert dispatch.** `latent_moe`, `src/kimi_k3/moe.rs:64-116`: route → down-proj →
+  chunked `ensure_loaded` → sequential `for &eid in chunk { apply_single_expert(...) }` →
+  optional RMSNorm → up-proj. (Post-`K3_OPTIMIZE_BRIEF.md`-Phase-5 this loop is a `par_iter`
+  with per-expert output buffers and a fixed-order sequential reduction — N3 builds on that
+  shape.) Expert math: `apply_single_expert`, `src/glm52/moe.rs:460`.
+- **KDA.** Already 96-way head-parallel on the global pool:
+  `state.heads.par_iter_mut()...`, `src/kimi_k3/generate.rs:258` (Kimi-Linear twin at
+  `src/kimi_linear/generate.rs:281`). Per-head state = `KdaState` 128×128 f32 = 64 KB
+  (`src/kimi_linear/kda.rs:92-96`). Heads are fully independent — the cleanest fit for
+  16-heads-per-node partitioning.
+- **MLA.** Absorbed decode is already 96-way head-parallel:
+  `ctx_row.par_chunks_mut(vh)...`, `src/glm52/attention.rs:481`. But the compressed KV cache
+  is **shared across all heads by construction** (`KvCache`, `src/glm52/attention.rs:86-91`:
+  one `l`/`r` stream per layer, `kv_lora=512`; every head's score loop reads the same rows) —
+  it cannot be partitioned by head. Weights partition per head; the KV stream must be
+  interleaved or replicated instead (N4c).
+- **Scope guard.** `glm52::moe::moe()` (GLM/Kimi-Linear's early-drain io_uring path) stays
+  untouched, same as in the previous brief (its rule §4.7). All NUMA dispatch work happens in
+  `latent_moe` and below `matmul_qt`.
+
+### Per-block mapping of the principle (agreed with the owner)
+
+| block | group unit | fit | scheme |
+|---|---|---|---|
+| routed experts (16 of 896/layer) | one expert (16.73 MiB, self-contained) | **clean — highest value** | home node per expert; compute on home node's pool (N3) |
+| KDA (69 layers × 96 heads) | head (state + weight row-blocks) | clean | 16 heads/node static; weights row-sharded, states node-allocated (N4b) |
+| MLA (24 layers × 96 heads) | head — **weights only** | partial by design | weight rows per head (16/node); shared KV stream interleaved (N4c) |
+| dense matmuls (shared experts, latent down/up, dense MLP L0, lm_head) | row-block of each tensor | clean | shard each QT's rows 6 ways, node i computes rows on node i (N4a) |
+
+Expected per-token routing skew (N3): which 16 experts a token picks is random, so per-node
+load is 2.7 ± a few; some tokens will put 5–6 experts on one node. Do **not** rebalance by
+stealing across nodes (that reintroduces the remote traffic this design removes) — it averages
+out over 92 MoE layers per token. Log the skew; don't fight it.
+
+---
+
+## 3. Invariants (binding, additive to `K3_OPTIMIZE_BRIEF.md` §4)
+
+1. **NUMA placement must never change math.** Placement and affinity are pure scheduling: same
+   per-expert compute, same reduction order → decode output stays **bit-identical** with
+   `--numa` on vs off. This is the acceptance gate for every phase except N6 (AMX, which is a
+   tolerance-tier kernel like every reassociating SIMD tier before it).
+2. **Everything is `#[cfg(target_os = "linux")]` + a `--numa` CLI flag, default off.** The dev
+   laptop (12-core AMD, single node) and the owner's Mac must build and behave exactly as
+   today. Non-Linux gets a no-op stub, not a compile error.
+3. **No new runtime dependencies.** `libc` is already in `Cargo.toml`. `sched_setaffinity` is
+   declared in the libc crate directly; `mbind`/`set_mempolicy` have **no glibc wrapper** (they
+   live in libnuma, which is banned) — call them as raw syscalls:
+   `libc::syscall(libc::SYS_mbind, ...)`. No `hwloc`, no `numa`/`numactl` crates. The one open
+   exception is the AMX C shim's `cc` build-dependency (D4, owner sign-off required).
+4. **Record the `numactl` invocation, `--numa` state, `--threads`, THP state, and `RUSTFLAGS`
+   with every number** in `PERFORMANCE.md`. A NUMA measurement without its placement recorded
+   is unreproducible noise.
+5. Fixture-dependent tests skip silently when fixtures are absent — check for `SKIP` lines
+   before claiming green (previous brief's trap #1; it still applies).
+
+---
+
+## 4. Phase N0 — Go/no-go probes (zero code, run on the instance first)
+
+**N0a. Is the guest's NUMA topology real?** KVM guests only benefit from guest-side affinity
+if the host pins vCPUs and backs each virtual node with matching host memory. Probe with any
+STREAM-triad binary (record which):
+
+```
+numactl --cpunodebind=0 --membind=0 <stream>     # local
+numactl --cpunodebind=0 --membind=2 <stream>     # intra-socket remote
+numactl --cpunodebind=0 --membind=3 <stream>     # cross-socket remote
+```
+
+**Gate:** local vs cross-socket bandwidth should split ≥2×. If the three numbers are flat, the
+guest topology is decorative — **stop here**, report to the owner, and fall back to
+`K3_OPTIMIZE_BRIEF.md`'s Phase 6 minimum (documented `numactl --interleave=all` as the launch
+default). Everything from N1 on assumes this gate passed. (Partially de-risked already: the
+measured 1.4× interleave win implies real placement effects. Run the probe anyway — its
+local:remote ratio sizes how much N3 can win and feeds decision D1.)
+
+**N0b. Baseline sensitivity.** With the AVX-512 kernel + Phase 4/5 landed, canonical command
+(`teacher_forced_decode_bench --expert-cache 896`, warm) under: default policy,
+`--interleave=all`, and `--cpunodebind=0-2 --membind=0-2 --threads 96` (single socket). If
+interleave barely moves the number, decode is not yet bandwidth-bound — investigate what still
+dominates before building N2–N4. Record everything in `PERFORMANCE.md`.
+
+**Recorded prior (owner, real checkpoint, scalar-kernel baseline, 2026-08-01):** `--threads
+48` decodes much faster than the 192-physical-core default; all cores is *slower*. This is
+consistent with the diagnosis in §1 — first-touch concentration means high thread counts add
+contention on 1–2 nodes' channels + UPI, not bandwidth, and 192-way fork/joins thin the
+per-thread grain to ~16 rows. Consequences: (a) re-run the thread sweep at every phase gate —
+the optimum will move; (b) baseline numbers should be taken at both 48 and the default, both
+recorded; (c) the NUMA phases' headline success criterion can be stated as: **after N3/N4,
+high pinned thread counts finally beat 48 unpinned** — if the sweep still says 48 after N3/N4,
+placement isn't working; stop and investigate rather than shipping the flag.
+
+**Recorded baseline (rev 2 — live serve, post-Phases-2–6, 2026-08-01):** canonical serve launch:
+
+```
+numactl --interleave=all ./target/release/rabbit --model /data/hf/hub/kimi-k3 --serve \
+    --port 8000 --expert-cache 896 --no-usage-cache --preload-experts --threads 48
+```
+
+Startup shape: ~15 min dense load (RSS → ~30 GiB, ~10 s/layer) then ~30 min expert preload at
+~0.8 GB/s (RSS → ~1.44 TiB), then the port opens and `/profile`'s `misses` freezes at 82,432
+(= 92 × 896, the preload itself). Baseline on a 128-token turn: **1.42 s/token** (expert 0.83 /
+attention 0.53 / lm_head 0.01). N3's decode gate is measured against THIS number and this
+launch configuration.
+
+**N0c. Record** `uname -r`, THP state, and re-paste `numactl --hardware` from the live box.
+
+---
+
+## 5. Phase N1 — `src/numa.rs`: topology + placement primitives
+
+New module, `libc`-only, fully `cfg`-gated, with a no-op twin for non-Linux. Public surface
+(keep it this small):
+
+- `topology() -> Option<Topology>` — parse `/sys/devices/system/node/node*/cpulist`. `None`
+  (or 1 node) ⇒ everything downstream degrades to current behavior.
+- `pin_current_thread(node)` — `sched_setaffinity` to the node's CPU list.
+- `bind_region(ptr, len, node)` / `interleave_region(ptr, len)` — raw `SYS_mbind` on an
+  address range. Note `mbind` sets a **VMA policy**: pages faulted *later* in the range follow
+  it, which is exactly what makes it work for reserve-then-grow buffers (N4c) — but it does
+  not migrate pages already touched (use first-touch discipline instead of `MPOL_MF_MOVE`;
+  moving pages is a rescue, not a design).
+- First-touch is the preferred placement mechanism everywhere a load/init already runs on a
+  pinned thread (N3 preload); `bind_region` is for buffers whose faulting thread can't be
+  controlled.
+
+Unit tests: topology parser against fixture strings; a `cfg(target_os = "linux")` smoke test
+that pins, allocates, touches, and reads back `move_pages`/`get_mempolicy` for the placement
+(skip-not-fail when run on a 1-node machine, matching the repo's fixture-test convention).
+
+**Gate:** `cargo test` green on Linux and macOS; `--numa` flag parsed (`src/main.rs` + USAGE)
+but changing nothing yet.
+
+---
+
+## 6. Phase N2 — Pinned per-node pools
+
+A `NodePools` singleton (built once in `configure_thread_pool`'s successor when `--numa` is
+active): one `rayon::ThreadPool` per node, each worker pinned via
+`ThreadPoolBuilder::start_handler(|_| numa::pin_current_thread(node))`. **Pool size derives
+from the effective `--threads` total, not from the core count**: `total_threads / n_nodes`
+(e.g. `--threads 48` → 8/node, default 192 → 32/node). The owner's observed 48-beats-192
+result (N0b) means the thread sweep must stay meaningful with `--numa` on — hardcoding
+32/node would silently override it. The global pool remains for non-NUMA paths and other
+architectures — GLM/Kimi-Linear behavior must not change.
+
+Two hazards to design around (they are the real content of this phase):
+
+1. **Never call `pool_b.install(...)` from inside a `pool_a` worker** — it blocks a worker
+   thread waiting on another pool. Cross-pool fan-out uses `spawn` + a latch/channel join from
+   the *orchestrating* (non-pool) thread, or `rayon::scope` per pool from outside.
+2. **Decide 6 pools vs 2 (per-socket) empirically.** SNC3 nodes share a socket's cache/UPI;
+   per-socket pools (2×96) capture the expensive cross-socket hops with a third of the
+   orchestration overhead. Build the pool count from `topology()`, benchmark both (D1).
+
+**Gate:** a criterion bench of a sharded matvec (N4a's kernel) at both pool configs; no
+user-visible behavior change with `--numa` off.
+
+---
+
+## 7. Phase N3 — Expert home nodes, FUSED with Phase 5 v2 (the main event)
+
+Prerequisites (all landed): `K3_OPTIMIZE_BRIEF.md` Phase 4b (`--preload-experts`) and Phase 5
+v1 (per-expert parallel apply with fixed-order reduction).
+
+**Why fused:** v2 (flatten to expert × row-block tasks, one fork per MoE layer, each task
+running the serial AVX-512 kernel over a row range) and home-node dispatch want the exact same
+task shape. Building v2 on the global pool first and re-plumbing it onto node pools afterward
+is the same surgery twice — do it once, here. This phase owns both the fork/join floor (~48
+all-core matmul forks per MoE layer today) and locality.
+
+**N3a. Assignment.** `home_node(layer, eid) = (layer * 896 + eid) % n_nodes` — static,
+deterministic, no state to persist. ~2.2 GB/layer/node, ~220 GB/node total at full residency:
+comfortably under 507 GB/node alongside interleaved dense weights (N4) and the OS page cache.
+
+**N3b. Placement.** In the preload path, run each expert's `load_expert`/`qt_load_mxfp4` **on
+its home node's pool** — allocation + `pread` fill happen on a pinned thread, so first touch
+lands the pages correctly with zero `mbind` calls (see the code-map note on
+`src/expert_cache.rs:884`). Misses loaded later during generation (capacity < 896, or cold
+start without preload) go through the same routing. Store nothing new in `ExpertSlot` —
+`home_node` is a pure function.
+
+**N3c. Dispatch (the v2 task shape).** Add row-range entry points to the MXFP4 kernel:
+compute rows `[r0, r1)` of one matmul with the serial AVX-512 tier inside — **no inner rayon**.
+In `latent_moe`'s apply stage: group the chunk's selected experts by home node → submit each
+expert's row-block tasks to its home node's pool (per-expert output buffer, `s × 3584`, ~14 KB
+at decode, transient) → **one fork/join per MoE layer across all pools** instead of ~48
+all-core forks → reduce the per-expert buffers into `routed` **sequentially in chunk iteration
+order**. Keep each expert's gate→activation→down chain order and per-row math identical to v1
+so the output stays bit-identical to v1 (which is itself bit-identical to the original
+sequential loop). Grain check: ~48 matmuls × ~3–3.5k rows split across a node's threads =
+hundreds of rows of real serial AVX-512 work per task — nothing like today's one-row
+micro-tasks.
+
+**N3d. Observability.** Extend `StepProfile`/`GET /profile`'s phase timings with per-node
+busy time for the MoE phase, plus a per-token expert→node skew counter. This is how D1 (6 vs 2
+pools) and the "don't rebalance" decision get validated with data instead of vibes.
+
+**Gate:** `teacher_forcing_k3` green (fixtures present and RUN — say so); the Phase-5
+determinism test (two warm runs → bit-identical logits) still green with `--numa` on; decode
+delta on the canonical command, `--numa` on vs off, recorded. This phase is the one expected
+to move the number materially — if it doesn't, N0b's sensitivity data was misread; stop and
+re-profile before building N4. **Numeric success criteria against the rev-2 serve baseline:
+expert bucket well under 0.3 s/token (from 0.83), and the thread sweep finally favoring high
+pinned counts over 48 unpinned.**
+
+---
+
+## 8. Phase N4 — Dense weights, KDA/MLA weights, caches
+
+**N4a. Row-sharded QT.** New `QTSharded` (or a `Vec<QT>` + row-offset table — implementer's
+call): a weight's output rows split into `n_nodes` contiguous blocks, block *i* allocated on
+node *i* (first-touch at load time by a pinned thread, same trick as N3b). A
+`matmul_qt_sharded` issues one row-range task per node pool and joins. Per-row math and
+per-row output slots are unchanged → bit-identical. Apply in descending traffic order:
+lm_head (163,840×7,168 — the poster child), latent `down_proj`/`up_proj` (92 layers), shared
+experts, dense MLP layer 0. **Do not convert every last tensor** — each conversion adds a
+6-pool barrier per matmul (~4,400 fork/join regions/token already exist; measure the barrier
+cost at decode dims in a criterion bench before and after).
+
+**N4b. KDA/MLA per-head weights.** Head-aligned row blocks: 96 heads / 6 nodes = 16 heads ⇒
+e.g. KDA q/k/v projections `[12288, 7168]` shard into 6 × 2048-row blocks on head boundaries;
+MLA `q_b`/`kv_b` same treatment. (Rev 2: the recurrence loop's task grain and the `KdaState`
+allocations moved to Phase N5 — the serve profile showed attention at 37% of the token, so
+that work was promoted to its own phase. N4b is weights only.)
+
+**N4c. The shared streams that can't be partitioned.** MLA's KV cache is one stream read by
+all heads (code map): `reserve` it to a max-context capacity at session start and
+`interleave_region` the reservation (VMA policy covers future faults — this is why N1's
+`mbind` wrapper exists), so all nodes draw balanced bandwidth during the score scan. The
+hidden-state activation vector (7168 f32 = 28 KB/token) is read by everyone but fits in L2 —
+leave it alone. Replicating KV per node (6 × ~7 GB at 128k ctx across 24 MLA layers) is
+affordable on this box but fans every append out 6× — only revisit if profiling shows the
+interleaved scan bound on remote latency (D3).
+
+**Gate:** bit-identity vs `--numa` off (N4a/b change nothing about order); decode + prefill
+delta recorded; GLM-5.2/Kimi-Linear timings demonstrably unregressed (they share `matmul_qt` —
+run one bench).
+
+---
+
+## 9. Phase N5 — Attention: KDA scheduling + state placement (new in rev 2)
+
+Justification (measured, §1): attention costs **0.53 s/token at near-zero context** — that is
+per-head scheduling overhead plus the scalar KDA recurrence, not KV traffic, and it becomes
+~80% of every token once N3 lands. Scope guard unchanged: the *math* of MLA/KDA is
+untouchable; task grain, scheduling, and memory placement around it are exactly this phase.
+
+- **N5a. Measure first.** Split the profile's `attention_s` into KDA recurrence vs projections
+  vs MLA, per layer kind (extend `StepProfile`). 69 KDA layers × (six projections + a
+  96-head recurrence + short-conv) has several candidate costs — don't guess the split.
+- **N5b. Task grain.** The per-head `par_iter` (96 tasks, each a ~64 KB state update) is
+  plausibly fork/join-floor-bound exactly like the expert matmuls were. Batch heads: one task
+  per node covering its 16 heads serially, states allocated on that node (absorbed from old
+  N4b), one fork per KDA layer. Same bit-identity argument as the existing head loop —
+  disjoint outputs, no cross-head reduction, only execution order changes.
+- **N5c. Projections** are ordinary matmuls — they ride N4a's row-sharding (at head-aligned
+  boundaries) if N4a's barrier-cost measurements came out favorable (D5).
+- Floor check: KDA state traffic is 69 × 96 × 64 KB ≈ 425 MB/token read+write ⇒ ~1–2 ms/token
+  at machine bandwidth. Attention's ceiling is nowhere near the current 0.53 s — there is real
+  room here.
+
+**Gate:** teacher-forced bit-identity (scheduling only); attention-bucket delta on the rev-2
+serve baseline; GLM-5.2/Kimi-Linear unregressed with their oracle tests **actually running**
+(KDA code is shared with Kimi Linear — this gate is not optional).
+
+---
+
+## 10. Phase N6 — AMX (follow-on; design-level here, own brief when reached)
+
+AMX multiplies 16×64 tiles — it needs ≥16 activation rows to fill a tile. **Batch-1 decode
+gets nothing from it** (one row ⇒ 15/16 idle, and post-N3/N4 decode is bandwidth-bound
+anyway). The AMX targets are prefill and batched expert application — `apply_single_expert`
+already batches all tokens routed to an expert into one multi-row matmul, so the structure is
+ready today.
+
+- **N6a. Transcode at preload, not on disk.** On this box experts are RAM-resident, so the
+  cache-slot format need not match the disk format. MXFP4→int8-per-block is **exact**: E2M1's
+  16 code points are {0, ±0.5, ±1, ±1.5, ±2, ±3, ±4, ±6} — multiply by 2 and they're all
+  integers in [−12, 12], so store `value×2` as int8 with the block scale halved (E8M0 is a
+  pure power of two; fold the ×½ into the per-32-block f32 scale). Same 1.32 TiB footprint.
+  This unlocks the *existing* VNNI IDOT tiers (`dot_i8i8_avx512vnni`, `src/kernels.rs:294`)
+  for s≥2 before any AMX code exists — likely the best effort/payoff step of the whole phase.
+- **N6b. AMX kernel.** Stable Rust has no AMX intrinsics (nightly-only as of this writing);
+  the clean options are a small C file (`immintrin.h`, `-mamx-int8`) via `build.rs` + `cc`
+  (needs D4 sign-off — first new build-dep) or inline asm. Runtime requirements: XFD
+  permission via `arch_prctl(ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA)` once per process before
+  first tile op, `ldtilecfg` per kernel, `is_x86_feature_detected!("amx-int8")` in the dispatch
+  ladder. Tolerance-tier parity tests (it reassociates), gated by an `s >= threshold` check
+  measured the way `I4_IDOT_MIN_S` was.
+- Scope note: MTP/speculative decoding (which would make AMX matter for *decode* by batching
+  candidate tokens) remains out of scope, per `rabbit-plan.md`.
+
+---
+
+## 11. Measurement protocol
+
+Identical to `K3_OPTIMIZE_BRIEF.md` §5 (teacher-forced bench only, worktree before/after,
+warm/cold labeled, canonical command at `--expert-cache 896`) plus: every number carries its
+`--numa` state and `numactl` prefix; every phase records per-node busy/skew from N3d once it
+exists. Free-running greedy decode is still not a timing instrument.
+
+## 12. Open decisions
+
+- **D1** 6 pools (per-node) vs 2 (per-socket). Recommendation: build from `topology()` so
+  both are one flag apart; decide on N2's bench + N3d's skew data.
+- **D2** `--numa` default once proven. Recommendation: off until it has survived one full
+  real-checkpoint session (`--chat`) without regression, then flip with the owner.
+- **D3** KV cache interleave vs per-node replication. Recommendation: interleave; revisit
+  only on profiling evidence.
+- **D4** AMX shim route: C-via-`cc` vs inline asm vs nightly toolchain. Owner sign-off
+  required either way (`cc` is the repo's first build-dep; inline asm avoids it at the cost of
+  hand-written tile config). No recommendation until N6 is actually scheduled.
+- **D5** Whether N4a extends to MLA/KDA projections at all if N4a's first conversions show
+  the barrier overhead eating the locality win at decode dims (s=1).
+
+## 13. Known traps
+
+1. All of `K3_OPTIMIZE_BRIEF.md` §12 still applies (silent fixture skips, teacher-forced-only
+   timing, stale laptop defaults).
+2. **`pool.install` from another pool's worker deadlocks under load** — cross-pool fan-out
+   only from an orchestrating thread (N2).
+3. **`Vec` growth breaks placement**: `extend_from_slice` reallocs land wherever the growing
+   thread runs. Any buffer with a placement policy must be exact-sized (expert QTs already
+   are) or reserved-then-`mbind`ed (KV cache, N4c).
+4. **THP vs sharding**: separate per-node allocations are THP-safe; do NOT try to `mbind`
+   row-ranges *inside* one contiguous QT allocation — 2 MB huge pages straddle the boundaries
+   and the policy silently loses. This is why N4a shards the allocation itself.
+5. `ExpertCache::get` is `&self` (`src/expert_cache.rs:779`) — if you read
+   `K3_OPTIMIZE_BRIEF.md`'s K5 first, its "`&mut self` borrow puzzle" note does not match the
+   code at `5853baa`. Trust the code.
+6. The KVM guest can lie (N0a). Also re-check after any instance resize/migration — host
+   repinning silently invalidates every measurement after it.
+7. `home_node` skew is per-token noise — do not add cross-node work stealing to "fix" it
+   (§2); over 92 MoE layers it averages out, and stealing reintroduces remote traffic.
+
+## 14. Definition of done
+
+Per phase: code + tests + module-doc updates + dated `PERFORMANCE.md` section (command,
+placement recorded per §11, before/after, failures included) + one commit. Overall: canonical
+command on the target box, warm, `--numa` on vs off, bit-identical logits (N3/N4/N5) and the
+speedup number; `cargo test` green on Linux **and** macOS with fixture status stated;
+GLM-5.2/Kimi-Linear unregressed; N6 untouched unless separately scheduled with the owner.
diff --git a/PERFORMANCE.md b/PERFORMANCE.md
index 4bd7f05..1b3006b 100644
--- a/PERFORMANCE.md
+++ b/PERFORMANCE.md
@@ -384,3 +384,752 @@ Hardware throughout: AMD Ryzen AI 9 HX 370 (12 cores / 24 threads, AVX2 + AVX-51
 [`jlnsrk/GLM-5.2-colibri-int4`](https://huggingface.co/jlnsrk/GLM-5.2-colibri-int4) checkpoint
 (378 GB, 744B params, community int4 conversion via colibrì's own tooling). See `rabbit-plan.md`
 for the full phase-by-phase development history behind each version.
+
+---
+
+# Kimi K3 on the target box (Granite Rapids-AP) — performance work
+
+Everything below this line is a **separate effort from the dev-laptop GLM numbers above**: the
+real 2.8T-param Kimi K3 checkpoint (`moonshotai/Kimi-K3`, MXFP4 experts) on a 2-socket, 6-NUMA
+Intel Xeon 6975P-C cloud instance. K3 shipped correctness-first (v0.23.0) with **zero** decode
+performance work; this is that work, executed against `K3_OPTIMIZE_BRIEF.md`. Different hardware,
+different checkpoint, different unit (seconds/token, not words/sec) — do not compare these numbers
+to the table above.
+
+## Phase 0 — baseline + build configuration (2026-07-31)
+
+### 0a. The target box (owner-provided `lscpu` + `numactl --hardware`, captured 2026-07-31)
+
+- **CPU:** 2× Intel Xeon 6975P-C (Granite Rapids-AP, custom cloud SKU), 96 cores/socket, SMT on →
+  **192 physical / 384 logical CPUs**. `Model name: Intel(R) Xeon(R) 6975P-C`, family 6 model 173
+  stepping 1. KVM guest (`Hypervisor vendor: KVM`, full virtualization).
+- **ISA:** full AVX-512 (`avx512f/dq/bw/vl/vnni/vbmi/vbmi2/bf16/fp16/ifma/bitalg/vpopcntdq`) plus
+  **AMX** (`amx_tile`, `amx_int8`, `amx_bf16`) and `avx_vnni`. Phase 3's AVX-512 tier and §9's AMX
+  follow-on are both hardware-supported here.
+- **NUMA:** 6 nodes (SNC3 — 3 sub-NUMA domains per socket). 64 logical CPUs + ~507 GB RAM per
+  node; socket 0 = nodes 0–2, socket 1 = nodes 3–5. `node distances`: 10 local, 15–17
+  intra-socket, 21–28 cross-socket.
+
+  ```
+  NUMA node0 CPU(s):   0-31,192-223      node 0 size: 507306 MB
+  NUMA node1 CPU(s):   32-63,224-255     node 1 size: 507905 MB
+  NUMA node2 CPU(s):   64-95,256-287     node 2 size: 507951 MB
+  NUMA node3 CPU(s):   96-127,288-319    node 3 size: 507951 MB
+  NUMA node4 CPU(s):   128-159,320-351   node 4 size: 507951 MB
+  NUMA node5 CPU(s):   160-191,352-383   node 5 size: 507926 MB
+  node distances:
+  node     0    1    2    3    4    5
+     0:   10   15   17   21   28   26
+     1:   15   10   15   23   26   23
+     2:   17   15   10   26   23   21
+     3:   21   28   26   10   15   17
+     4:   23   26   23   15   10   15
+     5:   26   23   21   17   15   10
+  ```
+
+- **RAM:** 2.9 TiB total. A live free-memory imbalance was visible while a run was resident
+  (node 2: 19 GB free vs node 0: 466 GB) — first-touch concentration observed in the wild, exactly
+  what Phase 6 targets. 1.32 TiB of expert slots cannot fit in one 507 GB node, so without
+  deliberate placement allocations spill node-to-node in load order.
+- **Kernel:** `5.14.0-687.15.1.el9_8.x86_64` (Rocky Linux 9.8 guest). **THP:** `[always] madvise
+  never` (system default is `always`).
+- **Memory bandwidth** (hand-rolled OpenMP triad `a=b+scale*c` and a read-only sum, f64, 8 GiB
+  arrays, `gcc -O3 -march=native -fopenmp`, run under `numactl`; not canonical McCalpin STREAM —
+  a ballpark instrument, recorded as such):
+  - read-only, `--interleave=all`, 384 threads: **~620 GB/s**
+  - read-only, single socket (`--cpunodebind=0-2 --membind=0-2`), 192 threads: ~545 GB/s
+  - triad (write-allocating), `--interleave=all`, 384 threads: ~342 GB/s (RFO write traffic caps it)
+
+  Decode streams read-only expert weights, so the ~620 GB/s read figure sets the floor:
+  25.8 GB/token ÷ 620 GB/s ≈ **0.042 s/token** — inside the brief's 0.025–0.06 s/token estimate.
+  **This is a floor, not a promise** (reaching it needs the §9 follow-ons).
+
+### 0b. Baseline (real checkpoint, `main` @ `5853baa`, before the build-profile change)
+
+Command: `teacher_forced_decode_bench --model /data/hf/hub/kimi-k3 --steps 12 --expert-cache 896`
+(the canonical command at a reduced 12 steps — one real-checkpoint run costs ~13 min to load +
+~7 min to decode, so the exploratory sweeps below are deferred to the synthetic fixture; see 0c).
+`RUSTFLAGS` unset, generic x86-64 baseline, default thread count (192 physical), THP `always`,
+no `numactl`. Single process, page cache warm from prior owner runs.
+
+| stage | number |
+|---|---|
+| model load | **773.0 s** (~13 min; dense-weight transcode-dominated) |
+| prefill (6-token prompt) | 192.2 s |
+| decode, 12 steps | **441.2 s = 36.8 s/token average** |
+| decode, steady state (steps 7–12) | **≈ 34 s/token** |
+
+Expert cache at cache-896 (no eviction): hits+misses grow by exactly 1472/step (16 experts × 92
+MoE layers). By step 12: 13595 hits / 10250 misses — 80% hit rate, still ~289 single-threaded
+expert loads/step, so 34 s/token is *compute + residual loads*, not pure compute. This ~34 s/token
+is the "before" anchor Phases 2/3/5 must crush; per the brief, decode is dominated by the scalar
+MXFP4 matmul (§3 K1), which at ~8 GFLOP/s aggregate across 192 cores is leaving ~40× on the table.
+
+### 0c. Zero-code experiments — status
+
+The brief's 0c sweep (threads 48/96/192/384, `numactl --interleave=all` vs default, single-socket
+control, THP `always` vs `madvise`, `RUSTFLAGS=-C target-cpu=native`) is **deferred to run on the
+Phase 1 synthetic fixture**, which the brief explicitly sanctions ("substitute the synthetic dir
+for iteration"). Rationale recorded honestly: each real-checkpoint run is a ~20-minute round trip
+dominated by a 13-minute load that tells us nothing about decode scaling, so running 8+ of them
+back-to-back is ~2.5 hours of mostly-load. The synthetic fixture (real per-layer widths, 6 layers)
+loads in a fraction of that and reproduces the thread-scaling and kernel behaviour these sweeps
+probe. **The one experiment that genuinely needs real 1.32 TiB scale — `interleave=all` vs default,
+which gates Phase 6 — is run once on the real checkpoint at the final milestone**, where its result
+matters most. `target-cpu=native`'s effect on the kernels is measured cheaply via `cargo bench`
+(Phases 2–3) rather than a full decode run. All results land in the phase sections that produce them.
+
+### 0d. Build profile
+
+Added `[profile.release] lto = "thin", codegen-units = 1` to `Cargo.toml` (was: no
+`[profile.release]` at all → no LTO, 16 codegen units). Purpose: let the optimizer inline across
+the `is_x86_feature_detected!` dispatchers into their `#[target_feature]` inner kernels. **Not**
+added: `panic = "abort"` (the `--serve` accept loop unwinds panicked request threads) and
+committed `target-cpu=native` (decision D1: no — keeps binaries portable; measured via `RUSTFLAGS`
+instead). No behaviour change beyond codegen.
+
+**Gate:** topology + bandwidth + baseline recorded above; `cargo test` green (oracle fixtures:
+state whether they ran or skipped in the commit); build-profile is the only code change.
+
+## Phase 1 — synthetic K3-at-scale fixture (2026-07-31)
+
+`src/bin/gen_k3_synth.rs` writes a **loadable, random-weight K3 checkpoint at the real per-layer
+widths** but few layers, so kernel/parallelism/NUMA iteration costs ~90 GB and a ~90 s load
+instead of 1.56 TB and ~13 min. Weights are garbage (this is a performance proxy, never a
+correctness one — that stays `tests/teacher_forcing_k3.rs`'s job); routed experts are emitted as
+raw MXFP4 `.weight_packed`/`.weight_scale` byte pairs with the exact counts `qt_load_mxfp4` checks,
+scale bytes pinned to [120,130] so magnitudes stay finite. One `*.safetensors` shard per layer,
+K3's real `language_model.` tensor names, config with the real dims and a 3:1 KDA:full-MLA pattern.
+
+Generated: `gen_k3_synth --out /data/k3-synth6 --layers 6 --experts 896` → **88.6 GiB in 185 s**
+(layer 0 KDA/dense, layer 2 MLA/MoE, rest KDA/MoE).
+
+**Gate (k3_smoke, `--expert-cache 896`, Phase-0 binary, real per-layer widths):**
+
+| | synthetic (6 layers) | real (93 layers) | ratio |
+|---|---|---|---|
+| model load | **93.2 s** | 773 s | ~8× faster |
+| decode, steady state | **~2.0 s/token** | ~34 s/token | 0.059 |
+
+The brief's extrapolation `(6/92) × real ≈ 2.2 s/token` predicts ~2.2 s; measured ~2.0 s — within
+noise, so the fixture faithfully reproduces per-token decode behaviour and is a valid stand-in for
+the later phases' iteration (and the deferred 0c sweeps). RSS after decode: ~7 GiB (dense resident
++ warmed experts), confirming `pread` streaming, not mmap.
+
+## Phase 2 — scalar MXFP4 kernel hygiene, bit-exact (2026-07-31)
+
+`matmul_mxfp4`'s inner loop restructured to iterate the row's 32-element scale blocks:
+`e8m0_decode` (a `2^(byte-127)` `powi`) is now evaluated **once per block** instead of once per
+element (the old loop recomputed it for all 32 elements sharing a scale byte), and each packed
+byte's two nibbles are decoded together, dropping the per-element `k & 1` branch. Per-element
+arithmetic `x*e2m1*scale` and accumulation order are unchanged → **bit-identical** output, pinned
+by `matmul_mxfp4_matches_the_pre_block_reference` (compares raw f32 bits against a straight port of
+the old loop, across dims incl. `i` not a multiple of 32/64 and odd `o`).
+
+Bench: `cargo bench --bench kernels -- matmul_mxfp4`, before via `git worktree` at the Phase-1
+commit with the new bench file copied in, after at the Phase-2 tip. Real per-expert dims, `RUSTFLAGS`
+unset, 384 threads (rayon default), on the target box.
+
+| case (i×o) | before (old scalar) | after (new scalar) | speedup |
+|---|---|---|---|
+| gate/up 3584×3072, **s=1 (decode)** | 4.45 ms | 4.19 ms | 1.06× |
+| gate/up 3584×3072, s=8 | 8.41 ms | 4.15 ms | **2.03×** |
+| down 3072×3584, **s=1 (decode)** | 4.74 ms | 4.04 ms | 1.17× |
+| down 3072×3584, s=8 | 8.36 ms | 4.18 ms | **2.00×** |
+
+**Honest read (the brief asked for it, "including less than hoped"):** removing the redundant
+`e8m0` `powi` is a clean **2× at s=8**, but only **~6–17% at s=1**, which is the batch-1 decode
+case (each expert sees one token's row). At s=1 a single [3072,3584] matmul is only ~5.5 MB of
+weight bytes touched in ~4 ms (~1.4 GB/s, far below the 620 GB/s floor) — so it is dominated by
+rayon fork/join dispatch and memory latency across 384 threads on ~8 rows each, not by the arithmetic
+Phase 2 trimmed. The decode kernel's real win is Phase 3's AVX-512 tier (and Phase 5 attacks the
+fork/join count). Sanity check that the bench is representative: 92 MoE layers × 16 experts × 3
+matmuls × ~4 ms ≈ 17.7 s/token, matching the ~21 s scalar-compute component of the Phase 0 baseline.
+
+*Tried and didn't help / notably:* nothing reverted this phase. Note s=1 vs s=8 barely differing in
+the NEW kernel (4.19 vs 4.15 ms) is the fork/join-dominated regime showing through — flagged for
+Phase 5.
+
+## Phase 3 — AVX-512 tier for matmul_mxfp4 (2026-07-31)
+
+`matmul_mxfp4` is now a dispatcher (AVX-512F/BW > scalar, same ladder as `matmul_i4`).
+`matmul_mxfp4_avx512` / `dot_mxfp4_f32_avx512`: one 32-element E8M0 block = 16 packed bytes = two
+16-lane f32 vectors; the 16 E2M1 code points fit one `zmm` addressed by `_mm512_permutexvar_ps`
+(low 4 bits of each index = the nibble, sign included). Per block: unpack low/high nibbles (same
+interleave as `dot_i4_f32_avx512`), gather E2M1 via permute, fold the block's scalar-decoded E8M0
+scale in, FMA into two accumulator chains reduced by one tree-sum; `cols % 32` tail scalar.
+**Within-tolerance, not bit-exact** (scale-fold + reassociated reduction) — new parity test
+`matmul_mxfp4_avx512_matches_scalar_within_tolerance` (relative-with-floor 2e-3 across edge dims).
+The K8 `matmul_qt_mxfp4_matches_manual_dequant_dot_product` test was relaxed from `assert_eq!` to
+the same within-tolerance form its **int4** sibling has always used (matmul_qt now dispatches the
+MxFp4 path to the reassociating AVX-512 tier).
+
+**The kernel is 3–5× faster — but end-to-end decode barely moves. This is the phase's key finding.**
+
+Kernel, single-thread (`RAYON_NUM_THREADS=1`, isolates arithmetic), real per-expert dims:
+
+| case | scalar | avx512 | speedup |
+|---|---|---|---|
+| gate/up 3584×3072, s=1 | 49.6 ms | 9.84 ms | **5.0×** |
+| down 3072×3584, s=8 | 144.5 ms | 48.3 ms | **3.0×** |
+
+Same matmul at the **default 384 threads** (rayon), the real decode configuration:
+
+| case | scalar | avx512 | speedup |
+|---|---|---|---|
+| gate/up s=1 | 4.82 ms | 4.97 ms | **~1.0× (none)** |
+| down s=8 | 5.03 ms | 4.88 ms | ~1.0× |
+
+End-to-end synth decode (k3_smoke, `--expert-cache 896`): **~2.0 → ~1.9 s/token (~5%)**.
+
+**Why (measured, not assumed):** at s=1 the matmul forks `par_chunks_mut(1)` = ~3072 one-row
+micro-tasks across 384 threads. Single-thread scalar is 49.6 ms; at 384 threads it is 4.82 ms —
+only ~10× from 384 cores (~2.6% parallel efficiency). The ~4.8 ms is a **rayon fork/join +
+task-scheduling floor**, hit by both tiers: the AVX-512 kernel is so much faster that there is
+almost no compute left for the 384 threads to divide, so its win is entirely masked. Decode's
+MXFP4 matmul is **scheduling-overhead-bound, not compute-bound** — the 8 GFLOP/s-aggregate gap the
+Phase 0 baseline noted is fork/join overhead from per-matmul 384-way parallelism over tiny work,
+not the kernel's scalar-ness. **The AVX-512 tier is the necessary compute foundation; Phase 5
+(across-expert parallelism, which collapses ~4400 fork/join regions/token to ~92) is the actual
+decode unlock — where this 3–5× kernel finally shows.** Real-checkpoint coherence + decode delta
+are folded into the post-Phase-5 milestone (synth weights are garbage, so coherence can't be
+checked here, and Phase 3 alone doesn't move real decode either — same scheduling floor).
+
+*Tried and didn't help (this phase, in isolation):* the AVX-512 matmul tier for end-to-end decode
+— correct and 3–5× at the kernel, but ~0% end-to-end at 384-way per-matmul parallelism. Kept
+because it is the foundation Phase 5 needs, not reverted. Recorded so nobody concludes "AVX-512
+didn't help MXFP4" without the Phase 5 context.
+
+## Phase 4 — expert loading: parallel misses, preload, honest logging (2026-08-01)
+
+**4a — parallel `sequential_fallback`.** `misses.iter().map(load_expert)` → `misses.par_iter()...`.
+rayon's *ordered* collect preserves insertion (miss) order, so cache insertion/pin order and
+determinism are byte-for-byte unchanged; `Shards` reads are `&self` `pread` (thread-safe, never
+mmap). This is the only load path K3's MXFP4 experts ever take (no io_uring ring).
+
+**4b — `--preload-experts`.** New CLI flag threaded through `LoadArgs` into `load_session`; after
+warm-start it loads every MoE layer's experts up front via a new `model::ExpertCaches::preload`
+(per-family arms — all three families) over the shared `expert_cache::preload_layers`. At
+`capacity >= n_experts` loads all (full residency); at `capacity < n_experts` fills to capacity in
+usage-histogram order when `.rabbit_usage` exists else expert-id order (D2), logged. One progress
+line per layer. Applies to `--serve` too (same `Session` path).
+
+**4c — honest logging.** `ExpertCache::has_ring()` / `ExpertCaches::any_has_ring()`: when no ring
+(MXFP4/K3), the CLI prints `[disk wait n/a]` instead of the structurally-always-`[0.0s actual disk
+wait]` bracket that misled the owner in the first minutes of the first real K3 run.
+
+**4d — `--drop-os-cache`: deferred.** Default-off, the least impactful sub-item (the OS reclaims
+page cache under pressure anyway per the brief), and its `drop_cache` bool would have to thread
+through the shared `begin_loading`/`sequential_fallback`/`load_expert` path used by all families
+and the io_uring branch. Deliberately left out of this pass rather than churn that hot path for a
+default-off flag; noted here so it's a known gap, not an oversight.
+
+**Measured (synth fixture, `rabbit --model /data/k3-synth6 --preload-experts --expert-cache 896`):**
+
+| | parallel (192 threads) | sequential-equiv (`RAYON_NUM_THREADS=1`) |
+|---|---|---|
+| preload 4480 experts (~73 GB) | **26.3 s** | 23.9 s |
+
+**Honest 4a finding:** parallel ≈ sequential here — the preload is **disk-bandwidth-bound** (~73 GB
+over NVMe at ~3 GB/s ≈ 24 s), not per-expert-CPU-bound, so 384-way parallelism has nothing to
+divide. 4a is expected to pay off in the **warm** regime the target box actually runs in — the real
+1.56 TB checkpoint is largely page-cache-resident (node 2 was full at baseline), so mid-decode
+expert loads are memory-bandwidth memcpy where parallelism helps, not cold disk reads. Recorded as
+"measured, no win in the disk-bound case." **4b clearly works regardless:** with `--preload-experts`
+the expert cache reaches full residency (4480/4480, 0 further misses) and **token 1 runs at
+steady-state 1.1 s with no warmup ramp** (vs ~2.3 s ramping down without it). **4c confirmed:** the
+log now reads `... in disk I/O [disk wait n/a]`, not the misleading `[0.0s actual disk wait]`.
+
+*Tried and didn't help:* parallelizing the expert loads (4a) on a cold, disk-bound fixture — no
+speedup because the storage bandwidth, not the loader, is the ceiling there. Kept (correct, and a
+win in the warm/page-cache regime), not reverted.
+
+## Phase 5 — across-expert parallelism in latent_moe (2026-08-01)
+
+`latent_moe`'s apply loop no longer runs the chunk's experts one after another (each expert's
+three matmuls fanning out over all cores and joining — ~48 fork/join regions per MoE layer). It now
+collects the chunk's resident `&ExpertSlot`s, runs each expert's full gate→activation→down chain
+into its **own zeroed `s×moe_hidden` buffer in parallel** (`slots.par_iter()`), then reduces the
+buffers into `routed` **sequentially in chunk order**. Per-expert math and the across-expert
+accumulation order are byte-for-byte the old loop's (each buffer is nonzero only at its expert's
+token rows, so adding a whole buffer only adds `+0.0` elsewhere), so the output is **bit-identical**
+— pinned by the existing naive-reference tests plus a new `latent_moe_is_bit_identical_across_two_warm_runs`
+determinism test (the reduction is fixed-order, not completion-order, so there is no run-to-run
+variance).
+
+**Borrow note:** the brief's K5 said `cache.get` takes `&mut self` (LRU stamp) and a `peek`
+accessor might be needed. At this commit `get` is already `&self` — LRU stamping happens in
+`begin_loading` (inside `ensure_loaded`), which runs before the parallel region — so the slots are
+collected with plain immutable `get` and no new accessor.
+
+**Required infra fix (`kernels.rs::with_yt_scratch`):** the matmuls' thread-local `yt` scratch
+assumed single-level parallelism ("rayon workers never call back into `matmul_*`"). Phase 5 nests
+an outer per-expert `par_iter` around the matmuls' inner `par_chunks_mut`, so a worker thread that
+work-steals a second expert's matmul while holding its own borrow hit `RefCell already borrowed`.
+Fixed additively: the reentrant (inner) call falls back to a one-shot heap buffer; the non-nested
+GLM/Kimi path still hits the zero-alloc thread-local fast path unchanged (verified: `matmul_i4`
+scalar 5.03 ms / avx512 4.23 ms, normal — GLM's hot path is structurally untouched, only
+`kimi_k3::moe` changed).
+
+**Measured (synth, k3_smoke `--expert-cache 896`, warm):**
+
+| | Phase 4 (sequential experts) | Phase 5 (parallel experts) | speedup |
+|---|---|---|---|
+| decode, steady state | ~1.9 s/token | **~1.0 s/token** | **1.9×** |
+| prefill (8 tokens) | 9.6 s | **4.77 s** | 2.0× |
+
+**v2 not built (brief gates it on profiling evidence):** v1 still runs each expert's matmuls with
+the inner per-matmul `par_chunks_mut`, so the fork/join floor Phase 3 identified is only *overlapped*
+across experts, not removed — the ~1.9× comes from that overlap, not from the AVX-512 kernel's full
+3–5× finally showing. The evidence that cores are still underused (Phase 3's fork/join finding +
+this residual inner fork) points at v2 — flatten to (expert × row-block) tasks with row-range kernel
+entry points, one fork per layer, each task doing real serial AVX-512 compute — as the next lever
+toward the memory-bandwidth floor. Deliberately left for a measured follow-up rather than shipped
+speculatively; this phase delivers the bit-identical overlap win first.
+
+## Real-checkpoint milestone — Phases 2–5 combined (2026-08-01)
+
+The exact Phase 0 baseline command on the real 2.8T checkpoint, all of Phases 2–5 in place:
+`teacher_forced_decode_bench --model /data/hf/hub/kimi-k3 --steps 12 --expert-cache 896`,
+`RUSTFLAGS` unset, 192 threads, no `numactl`, no preload (loads experts during decode, exactly
+like the baseline).
+
+| metric | Phase 0 baseline | Phases 2–5 | speedup |
+|---|---|---|---|
+| model load | 773.0 s | 774.6 s | unchanged (the kernel/MoE work doesn't touch load) |
+| **prefill (6-token prompt)** | 192.2 s | **84.8 s** | **2.27×** |
+| **decode, 12 steps** | 441.2 s = 36.8 s/token | **202.4 s = 16.9 s/token** | **2.18×** |
+| decode, steady state | ~34 s/token | **~16.5 s/token** | ~2.1× |
+
+**Apples-to-apples, pure compute win:** step-12 cache totals are 13608 hits / 10239 misses vs the
+baseline's 13595 / 10250 — essentially identical routing and expert-loading (same misses ⇒ same
+disk/memory traffic), so the ~2× is entirely the compute the kernel (Phase 3) and across-expert
+parallelism (Phase 5) sped up, not a caching artifact. Correctness holds (Phase 2 bit-identical,
+Phase 3 within-tolerance, Phase 5 bit-identical); the run produced its teacher-forced sequence with
+no NaN/inf and the same routing as baseline.
+
+**Where the remaining headroom is:** 16.5 s/token is still far above the ~0.042 s/token
+memory-bandwidth floor. Two levers remain, both measured below: (1) **NUMA placement** — this
+milestone ran with default first-touch, which Phase 6 shows leaves ~1.4× on the table; and (2)
+**Phase 5 v2** (expert × row-block tasks, one fork/layer, serial AVX-512 per task) to remove the
+inner per-matmul fork/join Phase 3 identified, so the kernel's full 3–5× shows end-to-end. This run
+used neither `numactl` nor v2, so ~16.5 s/token is a conservative floor for what Phases 2–5 already
+enable.
+
+## Phase 6 — NUMA (conditional): TRIGGERED — `numactl --interleave=all` (2026-08-01)
+
+The brief gates Phase 6 on the interleave experiment moving the number materially. **It does** — and
+the intuition that "compute-bound ⇒ NUMA doesn't matter" was wrong, which is exactly why the brief
+says measure. The compute *is* streaming 25.8 GB of expert weights from memory per token; default
+first-touch concentrates those weights on whichever nodes the loading threads ran on (the live
+imbalance captured at baseline — node 2 at 19 GB free vs node 0 at 466 GB), so 384 threads spread
+across all 6 nodes contend on 1–2 memory controllers. Interleaving the pages across all nodes
+removes that hotspot.
+
+Measured on the synthetic fixture (k3_smoke, `--expert-cache 896 --prompt-len 8 --max-tokens 6`, warm):
+
+| | decode per token | notes |
+|---|---|---|
+| default (first-touch) | ~1.5 s (1.22–2.00, high variance) | weights clustered on a few nodes |
+| **`numactl --interleave=all`** | **~1.07 s (1.03–1.13, tight)** | **~1.4× faster, variance collapses** |
+
+**Recommendation (zero code — the brief's preferred Phase 6 outcome): run with `numactl
+--interleave=all`.** Real-scale (1.32 TiB spanning all 6 nodes) can only make the default hotspotting
+worse, so the real checkpoint likely benefits at least as much — the milestone above (default
+placement) would land lower under interleave. No `madvise`/`mbind` code was needed: interleaving via
+`numactl` gets the win with none of the allocator surgery, exactly the "documented `numactl` first"
+ordering the brief asks for. Per-expert round-robin placement during preload (Phase 4b) is the
+natural in-process equivalent if a code-level default is ever wanted, but it isn't needed to capture
+the win today.
+
+## Phase 5 v2 — row-blocked expert dispatch, one fan-out per stage (2026-08-01/02)
+
+Provenance, stated plainly: this restructure was found complete-but-uncommitted in the working
+tree at the start of the session that executed `NUMA_AMX_BRIEF.md` (built in a prior working
+session; its comments referenced measurements that had never been recorded here). It was
+re-verified from scratch — full test suite including the oracle teacher-forcing suites, which now
+RUN on this box (fixtures present under `tests/oracle/`), plus the controlled numbers below — and
+is committed as part of Phase N3, which the brief fuses it into.
+
+What it is:
+
+- `kernels.rs::par_rows` — every `matmul_*` now hands rayon ~4 row-block tasks per pool thread
+  instead of ONE TASK PER OUTPUT ROW (the fork/join pathology Phase 3 diagnosed: thousands of
+  one-dot-product tasks). Task granularity never changes a row's own accumulation order —
+  bit-identity unaffected, all existing kernel tests unchanged.
+- `kernels.rs::matmul_qt_rows` + `RowActs` — row-range entry points that compute `[r0, r1)` of a
+  matmul serially through the SAME per-row kernel bodies the whole-matrix entry points use
+  (factored `row_dot_*` helpers), so reassembling row blocks reproduces `matmul_qt` bit for bit —
+  pinned across all six `QTKind`s by `matmul_qt_rows_reassembled_is_bit_identical_to_matmul_qt`.
+- `kimi_k3/moe.rs` — `latent_moe`'s apply stage flattened to (expert × row-block) tasks: TWO
+  fan-outs per chunk (gate/up + activation, then down + accumulate), no nested rayon. Stage B
+  parallelizes over OUTPUT ROWS, walking experts in chunk order per row — the float addition
+  order is exactly the sequential loop's, so output is bit-identical (test:
+  `latent_moe_row_blocked_dispatch_is_bit_identical_to_applying_experts_one_at_a_time`).
+- `glm52/moe.rs` — visibility-only (`expert_rows` extracted; `Activation::{combine,apply}`
+  `pub(crate)`). `glm52::moe::moe()` itself untouched (scope rule 4.7).
+
+**Measured, worktree v1 (`ef8a40d`) vs v2, synth fixture, canonical command at `--steps 30
+--expert-cache 896`, warm, RUSTFLAGS unset, THP `[always]`, default placement:**
+
+| threads | v1 | v2 | change |
+|---|---|---|---|
+| 48 | 3.03 s (0.101 s/token) | 2.88–3.16 s (~0.10 s/token) | **~none** |
+| 96 | 5.57 s | **2.48 s (0.083 s/token)** | **2.2×** |
+| 384 | 27.50 s | 12.87 s | 2.1× |
+
+**Honest read:** v2 does NOT speed up the 48-thread configuration the serve baseline runs at —
+it removes the *negative* thread scaling. Under v1 every added thread past ~48 made decode
+slower; under v2 the optimum moves to ~96 threads (0.083 s/token, 22% faster than any v1
+configuration) and the 384-thread cliff drops from 9× to 4×. The remaining high-thread-count
+degradation is NOT the expert path (see N3's finding below — it is attention-side, Phase N5's
+target). GLM/Kimi-Linear shared kernels: `matmul_i4` bench after the coarsening runs 3.52–3.71 ms
+vs the 4.23–5.03 ms recorded at Phase 5 — improved, not regressed.
+
+## Phase N0 (NUMA_AMX_BRIEF) — go/no-go probes (2026-08-01)
+
+Everything from here down executes `NUMA_AMX_BRIEF.md` rev 2 (which fuses `K3_OPTIMIZE_BRIEF.md`
+Phase 5's deferred v2 into its Phase N3). Box re-verified per the brief's provenance caveat:
+`lscpu` + `numactl --hardware` re-captured on the live instance and **identical to Phase 0a's
+record** (6 nodes SNC3, 64 logical CPUs + ~507 GB each, distances 10 / 15–17 / 21–28, AMX
+present, KVM). Kernel `5.14.0-687.15.1.el9_8.x86_64`, THP `[always]`, box idle, ~1.4 TiB of the
+checkpoint warm in page cache.
+
+**N0a — is the guest NUMA topology real? YES (gate passed).** Hand-rolled OpenMP triad + read
+probe (`gcc -O3 -march=native -fopenmp`, 2 GiB f64 arrays — same ballpark-instrument genre as
+Phase 0a, not canonical STREAM), `numactl --cpunodebind=0 --membind={0,2,3}`:
+
+| placement | triad | read-only |
+|---|---|---|
+| local (mem 0) | 153.7 GB/s | 183.0 GB/s |
+| intra-socket remote (mem 2) | 141.0 GB/s | 175.8 GB/s |
+| cross-socket remote (mem 3) | 79.7 GB/s | 87.1 GB/s |
+
+Local : cross-socket = **2.10× read** (1.93× triad) — ≥2×, the guest topology has real teeth; GO.
+The surprise worth keeping: **intra-socket-remote is nearly free (−4%)** — SNC3 sub-node
+boundaries barely cost anything; the socket boundary is the wall. (Feeds D1: per-node pools
+already capture almost everything per-socket pools would.)
+
+**N0b** is folded into the Phase 5 v2 / N3 sections below — the sensitivity matrix was run on
+the v2 tree since that is what N3 ships on.
+
+## Phase N1 — `src/numa.rs`: topology + placement primitives (2026-08-01)
+
+`topology()` (parses `/sys/devices/system/node/*/cpulist`; pure parser, unit-tested),
+`pin_current_thread` (`sched_setaffinity`), `bind_region`/`interleave_region` (raw `SYS_mbind` —
+no glibc wrapper exists), `node_of_page` (`SYS_get_mempolicy`, the test instrument). libc-only,
+no new dependencies, no-op stub twin for non-Linux. `--numa` parsed and deliberately inert (N1
+gate). The placement smoke tests run REAL on this box and caught two genuine hazards:
+
+- **`mbind` demands a page-aligned start** and heap pointers aren't (glibc's mmap'd chunks carry
+  a 16-byte header) — the wrappers page-align internally, leaving sliver pages on the default
+  policy (noise at multi-MB buffer sizes).
+- **Memory policy is inherited per-thread from the creator** — under the canonical
+  `numactl --interleave=all` launch every pool worker would start `MPOL_INTERLEAVE`, and first
+  touch would spray each expert's pages across all six nodes, silently defeating the entire
+  home-node scheme. `pin_current_thread` therefore also resets the calling thread's policy to
+  default (first-touch-local); a dedicated test reproduces the inherited-interleave condition
+  and asserts pages still land on the pinned node. Without this, `--numa` under the standard
+  serve launch would have been a placement no-op that *looked* like "NUMA doesn't help".
+
+## Phase N2 — pinned per-node pools (2026-08-01)
+
+`numa::NodePools`: one pinned rayon pool per node, each sized `total/n_nodes` from the effective
+`--threads` total (hardcoding cores/node would silently override the owner's thread sweep), built
+once when `--numa` is active on a real multi-node box. `run_all(f)` is the single cross-pool
+fan-out primitive — orchestrated from short-lived non-pool threads, which structurally rules out
+the install-from-another-pool deadlock (brief trap #2).
+
+Gate bench (`examples/numa_pool_bench.rs`: `[32768 × 7168]` int4 matvec at s=1 — N4a's
+`matmul_qt_sharded` prototype; min / median of 30, RUSTFLAGS unset):
+
+| total threads | global pool | per-node sharded (6) | per-socket sharded (2) |
+|---|---|---|---|
+| 48 | 0.739 / 0.941 ms | 0.904 / 0.998 ms | 0.931 / 1.197 ms |
+| 192 | 1.391 / 1.955 ms | 0.847 / 0.973 ms | 0.604 / 1.136 ms |
+
+At 48 threads the three configurations tie; at 192 the global pool **degrades ~2×** while the
+pinned pools hold steady — pinning is what makes high thread counts usable at all. Per-socket ≈
+per-node with more variance → **D1: per-node**, revisit only if N3d's skew data argues otherwise.
+
+## Phase N3 — expert home nodes, fused with Phase 5 v2 (2026-08-01/02)
+
+- **N3a** `numa::home_node(layer, eid) = (layer × n_experts + eid) mod n_nodes` — pure function,
+  the single agreement point between placement and dispatch.
+- **N3b placement**: `expert_cache::sequential_fallback` → `numa_homed_load` when `--numa` is
+  active: each missing expert loads INSIDE its home node's pinned pool, so allocation + `pread`
+  fill (= first touch of every page) happen on the node that will compute it. Zero `mbind`
+  calls. Miss-order results preserved → cache insertion order, pin promotion, determinism all
+  unchanged. Covers preload and mid-decode misses alike.
+- **N3c dispatch**: `latent_moe` → `dispatch_numa`: ONE `run_all` fan-out per chunk; each node
+  runs its own experts' gate→activation→down (same shared `gate_up_block` task body as the
+  global path, row-block-parallel within the pool), and the routing-weighted accumulation moves
+  to a sequential chunk-order scatter afterward — same additions per output element in the same
+  order as the `--numa`-off path, so **bit-identical by construction** (unit test against real
+  6-node pools: `numa_dispatch_is_bit_identical_to_the_global_pool_dispatch`).
+- **N3d**: cumulative per-node busy/expert counters, served as `numa_moe` in `GET /profile`.
+- `--numa-threads N` decouples the pinned-pool total from `--threads` — measured below, the two
+  pools want very different widths.
+
+**Bit-identity gate: PASS everywhere.** The bench now folds every decode step's full logits into
+an FNV-1a fingerprint. Synth: `af9ee5068abb90b0` across all 13 runs (on/off × 48/96/192/384,
+coupled and decoupled). Real checkpoint: `60c0f42c265f66a5` across all 4 runs (on/off × 48/96).
+Same routing (identical hit/miss totals per step), same logits, only scheduling differs.
+
+**Synth timings** (canonical command `--steps 30 --expert-cache 896`, warm, default policy):
+coupled `--numa` LOSES at every width (48: 4.25 vs 3.03 off; 192: 5.78; 384: 13.35) — but
+**numa@384 ≈ off@384 (13.35 vs 12.87)**, which acquits the expert dispatch: the high-thread
+collapse is attention + dense matmuls degrading with GLOBAL pool width in both configurations
+(measured ~10 s of the 13 s at 384). That reframes the brief's thread-sweep criterion — the
+expert bucket and the attention bucket want different thread counts, hence `--numa-threads`.
+Decoupled (global 48 for attention, pinned N for experts): p96 2.95 s, **p192 2.54 s**, p384
+2.64 s — vs off@48 3.03 s and off@96 2.48 s. On synth, NUMA reaches parity with the best global
+configuration, no more: synth decode moves ~1.3 GB/token at ~0.08 s/token ≈ 16 GB/s, nowhere
+near bandwidth-bound, so home-node locality has nothing to win there and its costs (routing skew
+— a node draws up to ~5 of 16 experts while others idle; cross-pool orchestration) show. Recorded
+as expected behavior, not failure: the bandwidth case is the real checkpoint at 1.32 TiB.
+
+**Real checkpoint** (`--steps 12 --expert-cache 896`, no preload, warm page cache, no `numactl`
+prefix, steady state = steps 7–12):
+
+| config | 12-step total | steady s/token |
+|---|---|---|
+| off, 48 threads | 31.06 s | 2.43 |
+| off, 96 threads | 32.52 s | 2.55 |
+| **--numa, global 48 / pinned 384** | **28.31 s** | **2.30** |
+| --numa, global 96 / pinned 384 | 29.64 s | 2.35 |
+
+NUMA-decoupled wins every pairing, but modestly (~6–9%) — this command mixes ~300 mid-decode
+expert loads into every step. The full-residency serve measurement below is the clean gate.
+
+**Serve gate** (canonical launch + `--numa --numa-threads 384`, i.e. `numactl --interleave=all
+... --serve --expert-cache 896 --no-usage-cache --preload-experts --threads 48 --numa
+--numa-threads 384`; 128-completion-token turn, 41-token prompt, `/profile`, misses frozen at
+82,432 = pure compute):
+
+| | rev-2 baseline (v1, 48 thr) | v2 + `--numa` g48/p384 |
+|---|---|---|
+| startup: preload | ~30 min | **687.6 s (~11.5 min, 2.6×)** |
+| wall | 1.42 s/token | 1.51 s/token |
+| expert bucket | 0.83 s/token | **0.90 s/token — GATE FAILED** (target < 0.3) |
+| attention bucket | 0.53 s/token | 0.56 s/token |
+
+**The failure is precisely diagnosed by N3d's own counters, and it is NOT the placement or the
+compute.** Per-node busy time for the turn: 31.8–36.9 s over 128 tokens — cumulative max-node
+busy is **0.288 s/token, already at the gate target**, and the expert→node spread is balanced
+(±8%). The 0.6 s/token gap between that and the 0.90 wall decomposes as roughly:
+
+- **~0.2 s/token routing skew** — per-LAYER max-node load (~4.5–5 of 16 experts) vs the 2.7
+  mean. The brief's "skew averages out over 92 layers" (§2) is true of cumulative totals and
+  false of wall time, which sums per-layer *maxima*. Measured, this argues for flipping **D1 to
+  per-socket pools**: 16 experts over 2 sockets has a max/mean of ~1.19 vs ~1.7 over 6 nodes,
+  and N0a measured intra-socket-remote reads at only −4% vs local — most of the skew cost
+  bought back for almost no locality cost.
+- **~0.18 s/token cross-pool fan-out latency** — measured directly (`numa_pool_bench`'s
+  `run_all` micro-bench): a no-op `run_all` costs ~0.4 ms, but with real work at 64 threads/node
+  the median is ~2.0 ms — a sleeping pool's wake cascade, paid once per MoE layer (92/token).
+  Fix candidates: persistent per-pool feeder threads (kills the 6 OS-thread spawns/layer, ~0.4
+  ms) and keeping pools from sleeping between layers (harder; rayon has no public spin config).
+- remainder: latent down/up matmuls + routing on the (deliberately narrow) 48-thread global
+  pool, sequential scatter, gather.
+
+**Control run — plain v2, no `--numa`, `--threads 96` (the synth optimum), same launch
+otherwise, same prompt:** wall 1.49 s/token, expert **0.73** (better than the 0.83 baseline),
+attention **0.71** (much worse than 0.53 at 48 threads). Preload: 425.5 s (page cache warmer
+than the numa run's 687.6 s — both crush the ~30 min baseline; the parallel-loading fix works
+under either scheduler).
+
+The three serve configurations side by side (per token, same 128-token turn shape):
+
+| | v1 @48 (rev-2 baseline) | v2+numa g48/p384 | v2 @96 |
+|---|---|---|---|
+| wall | **1.42** | 1.51 | 1.49 |
+| expert | 0.83 | 0.90 | **0.73** |
+| attention | **0.53** | 0.56 | 0.71 |
+| preload | ~30 min | 11.5 min | 7 min |
+
+Bottom line for this phase, stated honestly: **placement works (balanced nodes, preload 3–4×
+faster, bit-identical logits everywhere), the per-node compute hits the gate number (0.29
+s/token max-node busy), but the dispatch orchestration re-introduces a fork/join floor at the
+cross-pool level and the serve gate FAILS (expert 0.90 vs < 0.3 target).** No configuration
+moved the serve WALL, because the expert and attention buckets want opposite thread counts:
+experts scale past 48 (0.83→0.73@96, 0.29 achievable pinned), attention degrades past 48
+(0.53→0.71@96) — per-token wall is now genuinely attention-gated as much as expert-gated. Per
+the brief's own rule (stop-for-owner-review after N3's gate; "if it doesn't move the number,
+stop and re-profile before building N4"), work STOPS here for owner review. Decisions on the
+table: (a) flip D1 to per-socket pools (skew 1.7→1.19 at −4% locality) + persistent feeders
+(−0.4 ms/layer) and re-gate N3; (b) run serving as plain v2 `--threads 48` (baseline wall,
+none of the NUMA machinery) until N5 lands; (c) prioritize N5 — attention at 0.53–0.71 s/token
+at near-zero context is scheduling cost, the same disease in the other bucket, and is now the
+larger half of the token.
+
+## D1 flip — per-socket pools, flattened domain dispatch (2026-08-02, owner-approved)
+
+Owner decision after the N3 review: flip D1 to per-socket pools; expected expert ~0.6 s/token
+(<0.3 needs the pool-wake residual, spun off as its own timeboxed investigation below); **gate
+the flip on synth + the cheap real bench only — the serve re-gate is deliberately deferred into
+one combined boot after N5** (deviation from re-gating each phase at the server, recorded here).
+
+What changed:
+- `Topology::socket_domains` groups nodes by the kernel distance matrix (<20 = same socket;
+  identity fallback), and `NodePools::init` builds per-socket domains. `home_node`, placement
+  and dispatch all follow automatically (they key off `pools.n()`).
+- `run_all` rebuilt as nested `in_place_scope`s (no OS threads, no `'static`): a no-op fan-out
+  fell from ~0.4 ms to **0.012–0.029 ms**.
+- **A trap worth its own line:** the first per-socket build ran each domain's experts
+  sequentially, splitting ONE expert's rows across the whole 96–192-thread pool —
+  `block_rows(3072, 192×4)` is a 4-row task, v2's micro-task pathology reborn, and it measured
+  as a straight regression (synth g48/p192: 3.16 s vs per-node's 2.54). Fixed by flattening
+  each domain's fan-out across ALL its experts (the same `gate_up_blocked` shape the global
+  path uses, scoped to the domain) — g48/p192 back to 2.60 s. Lesson: task grain must be sized
+  against the pool that runs it, wherever the dispatch happens.
+
+Gate results (fingerprints unchanged everywhere — synth `af9ee5068abb90b0`, real
+`60c0f42c265f66a5`):
+
+| config | synth (30 steps) | real steady (steps 7–12) |
+|---|---|---|
+| off, 48 threads | 3.04 s | 2.57 s/token |
+| **--numa g48 / p192** | 2.60 s | **2.03 s/token (−21% vs off, −12% vs per-node p384's 2.30)** |
+| --numa g48 / p384 | 3.79 s | 3.28 s/token |
+
+**Operating point: `--numa-threads 192` (96/socket).** Wider pinned pools regress until the
+wake/distribution cost is fixed.
+
+**"Pools stay hot" (timeboxed investigation — findings only, no fix built):** the
+`numa_pool_bench` `run_all` micro-bench with a token's-worth of work, back-to-back vs after a
+5 ms idle gap (decode's real shape — attention runs between MoE layers, pools sleep):
+24/96/192 threads-per-pool cost 0.10/0.37/1.84 ms back-to-back and 0.40/0.76/2.94 ms after the
+gap. Both components — sleep-wake (~0.3–1.1 ms) and task distribution (the rest) — scale with
+pool width. At p192 that is ~0.07 s/token of overhead (acceptable); at p384 ~0.27 s/token
+(prohibitive) — which is exactly why p384 regresses. Fix candidates for the follow-up: a
+keep-warm spinner per pool (kills the wake share), coarser minimum task grain at wide pools,
+two-level fan-out (socket → node subgroups). Not built in this timebox.
+
+## Phase N5a — attention split, measured (2026-08-02)
+
+`attention_s` now splits into `attn_kda_proj_s` / `attn_kda_recur_s` / `attn_mla_s`
+(instrumentation only — timing calls around unchanged code in `kimi_k3::generate`; served in
+`/profile` and printed per-token by the bench, which now runs `step_profiled`).
+
+Real checkpoint, g48/p192 run above: **attention 0.380 s/token = KDA projections 0.274 + KDA
+recurrence 0.050 + MLA 0.055.** The brief's N5b hypothesis — that the 96-head recurrence
+`par_iter` is fork/join-floor-bound like the expert matmuls were — is REFUTED by the
+measurement: the recurrence is 13% of the bucket and its floor-check estimate (~1–2 ms/token)
+was closer to right than the fear was. **72% of attention is the eight per-token projection
+matmuls per KDA layer** (q/k/v/o are ~[12288×7168]-class int4 matvecs — 552 small matmuls/token
+on the global pool). Consequence, owner-visible: **N5b (head-batching the recurrence, ceiling
+0.05 s/token) is descoped on this evidence**; the attention lever is the projections, which is
+N4b/N5c's row-sharding — the next phase in the approved order anyway. Also worth recording:
+in the p384 run the same buckets bloat (kda_proj 0.637, attention 0.849) — oversubscribed
+pinned pools actively hurt the global pool's attention, another reason p192 is the operating
+point.
+
+## Phase N4 — dense-weight row-sharding: lm_head kept, KDA projections built-measured-reverted (2026-08-02)
+
+`QTSharded` (kernels.rs): a dense weight split into one contiguous row-block `QT` per NUMA
+domain, each COPIED inside its domain's pinned pool at load (first touch = placement — N3b's
+trick applied to dense weights; `QT::copy_rows` is the mechanism). `matvec`/`matvec_sharded_batch`
+run s=1 matvecs as ONE cross-domain fan-out with every domain writing its disjoint output rows
+directly (at s=1 the transposed layout IS the output layout, so assembly is free). `DenseQT`
+(`Plain`/`Sharded`) is the storage type for converted weights; everything loads `Plain` and a
+post-load `Model::distribute_dense` shards iff `--numa` pools exist. **Bit-identity pinned at
+the unit level across all six `QTKind`s** (single + batched, real pools, uneven row split), and
+the end-to-end fingerprints never moved (synth `af9ee5068abb90b0`, real `60c0f42c265f66a5`).
+
+What the fan-out-frequency arithmetic ruled out up front (recorded so nobody re-tries it
+blind): the latent down/up (184 matmuls/token, whole cost ~48 ms/token) and the small KDA
+projections can never repay a ~0.7 ms fan-out each. Candidates were lm_head (1/token) and the
+big KDA q/k/v/o (N5a's 72%-of-attention target, 2 fan-outs/layer with q/k/v batched into one).
+
+**Measured — lm_head: WIN, kept.** Synth per-token lm_head bucket 0.008 → 0.003 s (real
+0.009 → 0.008; one fan-out per token is easily repaid by 6-domain-local reads of the
+[163840×7168] weight).
+
+**Measured — KDA q/k/v/o: REGRESSION on synth AND real, reverted to plain.** Synth kda_proj
+0.018 → 0.027 s/token; real 0.346 → 0.443 s/token (12-step averages, same-fingerprint runs).
+At 69 KDA layers × 2 fan-outs = 138 fan-outs/token, the measured ~0.7 ms wake/distribution
+cost per fan-out (~0.10 s/token) plus per-fan-out latency jitter exceeds what domain-local
+reads win back. This is the same verdict the "pools stay hot" timebox predicted: **the
+fan-out cost, not placement, is the binding constraint on every high-frequency NUMA dispatch.**
+The `DenseQT` fields, the batched q/k/v fan-out in `kda_step`, and `distribute_dense` all stay
+in place as the ready mechanism — re-enabling is four `shard_in_place` lines + a re-gate once
+pools-stay-hot lands. Post-revert sanity: synth `--numa` g48/p192 = **2.40 s / 30 steps —
+the best synth configuration measured to date** (off@48 2.90–3.16, off@96 2.48).
+
+**N4c (KV-cache interleave): deferred, with justification.** The MLA bucket is 0.055 s/token
+at short context (N5a) — KV traffic is not yet a cost; `KvCache` is shared with GLM-5.2; and
+its buffers grow by `extend` (growth reallocs silently drop an mbind VMA policy — trap #3), so
+doing this properly means a reserve-at-capacity change to shared code. Worth doing when
+long-context serving is the workload; not worth the shared-code risk for 0 measured win today.
+
+**N5b (recurrence head-batching): descoped** on N5a's measurement (ceiling 0.05 s/token) —
+recorded in the N5a section above. N5c (projections riding N4a's sharding) is subsumed by the
+KDA measurement above: same verdict, same re-enable path.
+
+## Combined boot — one serve re-gate for the flip + N5a + N4 together (2026-08-02)
+
+The owner-approved deviation resolved: instead of re-gating each phase at the server, ONE boot
+with everything in. Launch = the canonical serve command plus the measured operating point:
+
+```
+numactl --interleave=all ./target/release/rabbit --model /data/hf/hub/kimi-k3 --serve \
+    --port 8000 --expert-cache 896 --no-usage-cache --preload-experts \
+    --threads 48 --numa --numa-threads 192
+```
+
+(48 global threads for attention, 2 pinned per-socket pools × 96 for experts + the sharded
+lm_head.) Same 128-completion-token turn shape as every serve measurement on this page,
+`/profile` buckets, misses frozen at 82,432. Turn 1 is first-touch-cold, turn 2 is the steady
+number:
+
+| per token | v1 @48 (rev-2 baseline) | combined boot, turn 1 | **combined boot, turn 2 (warm)** |
+|---|---|---|---|
+| wall | 1.42 s | 1.385 s | **1.264 s (−11%)** |
+| expert | 0.83 s | 0.755 s | **0.656 s (−21%)** |
+| attention | 0.53 s | 0.578 s | 0.556 s (+5%) |
+| lm_head | 0.0084 s | 0.0034 s | **0.0031 s (2.7×)** |
+| preload | ~30 min | **542 s (~9 min)** | — |
+
+Per-socket MoE busy is balanced (37.0 / 36.2 s over the first turn — the D1 skew argument
+holding at 2 domains). An accidental extra data point worth keeping: a stale-binary boot (old
+per-node dispatch, pre-N4) at the same g48/p192 measured wall 1.362 / expert 0.784 — i.e.
+per-node vs per-socket at the server is within run-to-run noise on wall; the flip's clear wins
+were on the teacher-forced bench and in the skew counters.
+
+**Where this leaves K3 serving.** Wall is now 1.26 s/token, attention-gated as much as
+expert-gated (0.66 + 0.56 + 0.01 ≈ the wall). The two remaining levers, both already
+diagnosed with data on this page: (1) **pools stay hot** — the ~0.4–0.8 ms per-fan-out
+wake/distribution cost is the binding constraint on the expert bucket's remaining ~0.35
+s/token of orchestration gap AND the thing that made the KDA-projection sharding (attention's
+0.4 s/token kda_proj chunk) a measured regression; fixing it unlocks both at once, and the
+re-enable path is four lines in `distribute_dense`. (2) The expert compute floor at current
+pool widths is ~0.29 s/token (measured max-domain busy) — reaching it is (1) again. N6 (AMX)
+remains unscheduled.
+
+**Full-effort scoreboard** (real checkpoint): decode 36.8 s/token at the Phase 0 baseline →
+16.9 (Phases 2–5) → **1.26 s/token in live serving** at this boot — with logits bit-identical
+across every scheduling configuration tested, pinned by fingerprints at every gate.
+
+## Tried and didn't help (target-box round)
+
+- **`RUSTFLAGS="-C target-cpu=native"`** (2026-08-02, closing the measurement the original
+  brief's 0c deferred): synth canonical command, default flags vs native — off@48 2.88–3.16 s
+  (session range) vs 3.43 s native; numa g48/p192 2.40 s vs 2.32 s native. Within run-to-run
+  noise both directions, logits fingerprint unchanged. Expected in hindsight: every hot matmul
+  is hand-written `#[target_feature]` intrinsics behind runtime dispatch, so `native` only
+  recompiles the glue around them, which the phase buckets show is not where the time goes.
+  Confirms D1's "don't commit it" from the measurement side; binaries stay portable.
+
+### Reproducing the K3 numbers
+
+Decode timing via `teacher_forced_decode_bench` only (free-running greedy is not run-to-run
+reproducible on this codebase — see the harness's module doc). Canonical command:
+`cargo run --release --example teacher_forced_decode_bench -- --model /data/hf/hub/kimi-k3
+--steps 30 --expert-cache 896` (Phase 0 baseline used `--steps 12` for cost; note the step count
+with every number). Kernel timing via `cargo bench --bench kernels -- mxfp4`. Before/after per
+phase via `git worktree` at the pre/post-phase commits, identical command lines. Record
+`RUSTFLAGS`, thread count, `numactl` invocation, and THP state with every number — and since the
+NUMA phases, the `--numa`/`--numa-threads` state too. The bench prints a per-run **logits
+fingerprint** (FNV-1a over every step's logits bits): identical fingerprints = bit-identical
+logits, the acceptance instrument for scheduling-only changes.
diff --git a/benches/kernels.rs b/benches/kernels.rs
index 0413b9d..afa09a9 100644
--- a/benches/kernels.rs
+++ b/benches/kernels.rs
@@ -12,11 +12,11 @@
 //! benchmark toward an unrealistically huge single dot product).
 
 use criterion::{Criterion, criterion_group, criterion_main};
-use rabbit::kernels::{axpy_i4_f32_scalar, dot_i4_f32_scalar, dot_i4i8_scalar, dot_i8i8_scalar, matmul_i4_scalar};
+use rabbit::kernels::{axpy_i4_f32_scalar, dot_i4_f32_scalar, dot_i4i8_scalar, dot_i8i8_scalar, matmul_i4_scalar, matmul_mxfp4_scalar};
 #[cfg(target_arch = "x86_64")]
 use rabbit::kernels::{
     axpy_f32_avx2, axpy_i4_f32_avx512, dot_f32_avx2, dot_i4_f32_avx512, dot_i4i8_avx2, dot_i4i8_avx512vnni, dot_i8i8_avx2,
-    dot_i8i8_avx512vnni, matmul_i4_avx2, matmul_i4_avx512,
+    dot_i8i8_avx512vnni, matmul_i4_avx2, matmul_i4_avx512, matmul_mxfp4_avx512,
 };
 use std::hint::black_box;
 
@@ -118,6 +118,33 @@ fn bench_matmul_i4(c: &mut Criterion) {
     group.finish();
 }
 
+/// K3 routed-expert MXFP4 matmul (`matmul_mxfp4`) at the real per-expert dims: gate/up projections
+/// are `[moe_inter=3072, moe_hidden=3584]` (i=3584, o=3072), down is `[3584, 3072]` (i=3072,
+/// o=3584). Benched at s=1 (decode) and s=8 (a small batch), the before/after instrument for
+/// Phase 2's scalar block restructure and Phase 3's AVX-512 tier. In Phase 2 this measures the
+/// single scalar tier (`matmul_mxfp4` is scalar); Phase 3 splits it into scalar/avx512 entries.
+fn bench_matmul_mxfp4(c: &mut Criterion) {
+    let mut group = c.benchmark_group("matmul_mxfp4");
+    for &(name, i, o) in &[("gate_up_i3584_o3072", 3584usize, 3072usize), ("down_i3072_o3584", 3072usize, 3584usize)] {
+        let data = random_bytes(o * i.div_ceil(2), 21);
+        let scale = random_bytes(o * i.div_ceil(32), 22);
+        for &s in &[1usize, 8] {
+            let x: Vec<f32> = random_i8(s * i, 23).iter().map(|&v| v as f32).collect();
+            let mut y = vec![0f32; s * o];
+            group.bench_function(format!("scalar/{name}/s{s}"), |b| {
+                b.iter(|| matmul_mxfp4_scalar(black_box(&mut y), black_box(&x), black_box(&data), black_box(&scale), s, i, o))
+            });
+            #[cfg(target_arch = "x86_64")]
+            if is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512bw") {
+                group.bench_function(format!("avx512/{name}/s{s}"), |b| {
+                    b.iter(|| unsafe { matmul_mxfp4_avx512(black_box(&mut y), black_box(&x), black_box(&data), black_box(&scale), s, i, o) })
+                });
+            }
+        }
+    }
+    group.finish();
+}
+
 /// `qt_addrow`'s int4 branch (MLA-absorb `q_nope` absorption) at the real `kv_lora=512` this
 /// runs at in `attention.rs`'s absorbed decode path — see that module's doc for why this and
 /// `qt_matvec_rows` are fixed O(kv_lora) per call, independent of context length.
@@ -204,6 +231,7 @@ criterion_group!(
     bench_dot_i8i8,
     bench_dot_i4i8,
     bench_matmul_i4,
+    bench_matmul_mxfp4,
     bench_qt_addrow_i4,
     bench_qt_matvec_rows_i4,
     bench_mla_score_and_vmix
diff --git a/examples/numa_pool_bench.rs b/examples/numa_pool_bench.rs
new file mode 100644
index 0000000..68b57fb
--- /dev/null
+++ b/examples/numa_pool_bench.rs
@@ -0,0 +1,177 @@
+//! Phase N2 gate bench (`NUMA_AMX_BRIEF.md` §6): a NUMA-sharded matvec — the prototype of
+//! N4a's `matmul_qt_sharded` and the empirical input to decision D1 (per-node vs per-socket
+//! pools) — timed against the exact same matvec on today's single global pool.
+//!
+//! Three configurations over one `[rows, 7168]` int4 weight at s=1 (batch-1 decode, the shape
+//! every dense matmul in the token loop has):
+//!
+//!  - `global`: one `matmul_qt` on the global rayon pool, weight allocated wherever first touch
+//!    put it — exactly today's behavior.
+//!  - `per-node`: weight rows split into one contiguous block per NUMA node, each block
+//!    allocated AND filled inside that node's pinned pool (first touch = placement), then each
+//!    step computed as one `run_all` fan-out with every pool running `matmul_qt_rows` over its
+//!    own block's rows in parallel within the pool.
+//!  - `per-socket`: same, but with nodes merged into per-socket domains (built from node
+//!    distances: nodes closer than 20 share a socket) — D1's alternative shape.
+//!
+//! Not criterion: the pinned pools and placement are process-global state that criterion's
+//! multi-process harness would rebuild per benchmark; a plain loop with a discarded warmup and
+//! a reported min/median over 30 reps is honest enough for a scheduling-floor comparison.
+//!
+//! Run on the target box (RAYON_NUM_THREADS sets the global pool AND the per-config total):
+//! `RAYON_NUM_THREADS=48 cargo run --release --example numa_pool_bench`
+
+use rabbit::kernels::{matmul_qt, matmul_qt_rows, RowActs};
+use rabbit::numa::{self, NumaNode};
+use rabbit::quant::QT;
+use rayon::prelude::*;
+use std::time::Instant;
+
+const COLS: usize = 7168;
+const ROWS: usize = 32768; // lm_head-class row count, scaled down 5x to keep fill time sane
+const REPS: usize = 30;
+
+fn random_vec(n: usize, seed: &mut u32) -> Vec<f32> {
+    (0..n)
+        .map(|_| {
+            *seed ^= *seed << 13;
+            *seed ^= *seed >> 17;
+            *seed ^= *seed << 5;
+            (*seed as f32 / u32::MAX as f32) - 0.5
+        })
+        .collect()
+}
+
+fn bench<F: FnMut()>(name: &str, mut f: F) {
+    f(); // warmup, discarded
+    let mut times: Vec<f64> = (0..REPS)
+        .map(|_| {
+            let t = Instant::now();
+            f();
+            t.elapsed().as_secs_f64() * 1e3
+        })
+        .collect();
+    times.sort_by(f64::total_cmp);
+    println!("{name}: min {:.3} ms, median {:.3} ms", times[0], times[REPS / 2]);
+}
+
+/// Row-blocks a weight across `domains`, allocating+filling each block inside its domain's
+/// pinned pool (first touch), then benches one fan-out per step.
+fn bench_sharded(name: &str, domains: &[NumaNode], threads_per: usize, w_src: &[f32], x: &[f32]) {
+    let pools: Vec<rayon::ThreadPool> = domains
+        .iter()
+        .map(|d| {
+            let pin = d.clone();
+            rayon::ThreadPoolBuilder::new().num_threads(threads_per).start_handler(move |_| numa::pin_current_thread(&pin)).build().unwrap()
+        })
+        .collect();
+    let n = domains.len();
+    let per = ROWS.div_ceil(n);
+    // Placement: each shard filled inside its own pinned pool -> its pages live on that domain.
+    let shards: Vec<QT> = pools
+        .iter()
+        .enumerate()
+        .map(|(i, pool)| {
+            pool.install(|| {
+                let r0 = i * per;
+                let nr = per.min(ROWS - r0);
+                let mut t = QT::alloc(nr, COLS, 4, false);
+                t.fill(&w_src[r0 * COLS..(r0 + nr) * COLS]);
+                t
+            })
+        })
+        .collect();
+    let mut outs: Vec<Vec<f32>> = shards.iter().map(|t| vec![0f32; t.rows]).collect();
+    bench(name, || {
+        let out_refs: Vec<std::sync::Mutex<&mut Vec<f32>>> = outs.iter_mut().map(std::sync::Mutex::new).collect();
+        std::thread::scope(|s| {
+            for (i, pool) in pools.iter().enumerate() {
+                let (shard, out, x) = (&shards[i], &out_refs[i], &x);
+                s.spawn(move || {
+                    pool.install(|| {
+                        let mut out = out.lock().unwrap();
+                        let acts = RowActs::prepare(x, shard, 1);
+                        let blk = shard.rows.div_ceil(pool.current_num_threads() * 4).max(1);
+                        out.par_chunks_mut(blk).enumerate().for_each(|(b, seg)| {
+                            matmul_qt_rows(seg, &acts, shard, b * blk, seg.len());
+                        });
+                    });
+                });
+            }
+        });
+    });
+}
+
+fn main() {
+    let threads: usize = std::env::var("RAYON_NUM_THREADS").ok().and_then(|s| s.parse().ok()).unwrap_or_else(|| num_cpus::get_physical());
+    rayon::ThreadPoolBuilder::new().num_threads(threads).build_global().unwrap();
+    let Some(topo) = numa::topology() else {
+        eprintln!("no NUMA topology — nothing to compare");
+        return;
+    };
+    println!("{} nodes, {} total threads, weight [{}x{}] int4", topo.n_nodes(), threads, ROWS, COLS);
+
+    let mut seed = 0x2545F491u32;
+    let w_src = random_vec(ROWS * COLS, &mut seed);
+    let x = random_vec(COLS, &mut seed);
+
+    // Today's behavior: one global-pool matmul_qt, weight placed by whatever thread fills it.
+    let mut w = QT::alloc(ROWS, COLS, 4, false);
+    w.fill(&w_src);
+    let mut y = vec![0f32; ROWS];
+    bench("global-pool matmul_qt   ", || matmul_qt(&mut y, &x, &w, 1));
+
+    bench_sharded("per-node sharded matvec ", &topo.nodes, (threads / topo.n_nodes()).max(1), &w_src, &x);
+
+    // The cross-pool fan-out floor: what does ONE NodePools::run_all cost when the pools have
+    // (a) nothing and (b) a token's worth of trivial work to do? K3 decode pays this once per
+    // MoE layer (92/token on the real checkpoint) — measured after the serve gate showed ~0.6
+    // s/token of expert-phase wall NOT accounted for by node busy time.
+    {
+        use rabbit::numa::NodePools;
+        for per_pool_total in [48usize, 192, 384] {
+            // Per-socket domains — the shape NodePools::init builds since the D1 flip; the
+            // run_all costs measured here are the ones decode actually pays.
+            let topo = rabbit::numa::topology().unwrap();
+            let Some(pools) = NodePools::build(topo.socket_domains(rabbit::numa::sys_distance_row), per_pool_total) else { break };
+            // Wake the pools once so the first bench iteration isn't a cold outlier beyond
+            // what steady decode sees (pools sleep between layers there too).
+            pools.run_all(|_| {});
+            bench(&format!("run_all noop        (total {per_pool_total})"), || pools.run_all(|_| {}));
+            bench(&format!("run_all 1ms-ish work (total {per_pool_total})"), || {
+                pools.run_all(|_| {
+                    (0..pools.threads_per_pool() * 8).into_par_iter().for_each(|_| {
+                        std::hint::black_box((0..20_000u64).fold(0u64, |a, x| a.wrapping_add(x * x)));
+                    });
+                })
+            });
+            // Same work, but after a 5 ms idle gap — the shape decode actually has (attention
+            // runs between MoE layers, so the pinned pools go to sleep every layer). The delta
+            // vs the back-to-back row above separates SLEEP-WAKE cost from task-distribution
+            // cost: that split is what the timeboxed "pools stay hot" investigation keys on.
+            // (Hand-rolled timing loop rather than `bench` so the sleep itself stays untimed.)
+            {
+                let mut times: Vec<f64> = (0..REPS)
+                    .map(|_| {
+                        std::thread::sleep(std::time::Duration::from_millis(5));
+                        let t = Instant::now();
+                        pools.run_all(|_| {
+                            (0..pools.threads_per_pool() * 8).into_par_iter().for_each(|_| {
+                                std::hint::black_box((0..20_000u64).fold(0u64, |a, x| a.wrapping_add(x * x)));
+                            });
+                        });
+                        t.elapsed().as_secs_f64() * 1e3
+                    })
+                    .collect();
+                times.sort_by(f64::total_cmp);
+                println!("run_all same, 5ms gap (total {per_pool_total}): min {:.3} ms, median {:.3} ms", times[0], times[REPS / 2]);
+            }
+        }
+    }
+
+    // Per-socket domains (the D1 production shape since the flip — numa::socket_domains).
+    let sockets = topo.socket_domains(rabbit::numa::sys_distance_row);
+    if sockets.len() < topo.n_nodes() {
+        bench_sharded("per-socket sharded matvec", &sockets, (threads / sockets.len()).max(1), &w_src, &x);
+    }
+}
diff --git a/examples/teacher_forced_decode_bench.rs b/examples/teacher_forced_decode_bench.rs
index 44ca19c..851dfc0 100644
--- a/examples/teacher_forced_decode_bench.rs
+++ b/examples/teacher_forced_decode_bench.rs
@@ -22,7 +22,12 @@
 //! Usage:
 //!   cargo run --release --example teacher_forced_decode_bench -- --model <dir> \
 //!       [--prompt <text>] [--steps N] [--dbits N] [--ebits N] [--expert-cache N] \
-//!       [--shard-dirs <dir1,dir2,...>]
+//!       [--shard-dirs <dir1,dir2,...>] [--numa]
+//!
+//! The final line includes a **logits fingerprint**: an FNV-1a hash folded over every decode
+//! step's full logits vector (raw f32 bits). Two runs that print the same fingerprint computed
+//! bit-identical logits at every step — this is the acceptance instrument for scheduling-only
+//! changes (`NUMA_AMX_BRIEF.md` invariant 1: same fingerprint with `--numa` on and off, warm).
 
 use rabbit::model::{self, ExpertCaches, KvState, Model, Tokenizer};
 use rabbit::safetensors::Shards;
@@ -43,6 +48,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let cache_capacity: usize = arg_value(&args, "--expert-cache").and_then(|s| s.parse().ok()).unwrap_or(64);
     let mut shard_dirs = vec![model_dir.clone()];
     shard_dirs.extend(arg_value(&args, "--shard-dirs").into_iter().flat_map(|s| s.split(',').map(PathBuf::from).collect::<Vec<_>>()));
+    if args.iter().any(|a| a == "--numa") {
+        // Node pools sized from the global pool's total (which honors RAYON_NUM_THREADS) unless
+        // `--numa-threads N` decouples them — the diagnostic that separates "how many unpinned
+        // threads should attention get" from "how many pinned threads should the experts get",
+        // which the N3 sweep showed are different questions.
+        let total = arg_value(&args, "--numa-threads").and_then(|s| s.parse().ok()).unwrap_or_else(rayon::current_num_threads);
+        match rabbit::numa::NodePools::init(total) {
+            Ok(p) => eprintln!("--numa: {} pinned node pools x {} threads", p.n(), p.threads_per_pool()),
+            Err(why) => eprintln!("--numa: {why}"),
+        }
+    }
 
     eprintln!("loading model (dbits={dbits}, ebits={ebits})...");
     let t0 = Instant::now();
@@ -66,9 +82,24 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // construction -- no argmax/sample call anywhere in this loop -- so the exact same sequence
     // of `step` calls (same ids, same positions) happens on every run, on every binary.
     let t_decode = Instant::now();
+    let mut fingerprint: u64 = 0xcbf29ce484222325; // FNV-1a offset basis
+    let mut phases = rabbit::generate::Phases::default();
+    let mut lm_head_s = 0f32;
     for (i, pos) in (0..steps).zip(prompt_ids.len()..) {
         let id = prompt_ids[i % prompt_ids.len()];
-        model::step(&model, &shards, &mut caches, &mut kv, &[id], pos)?;
+        let (logits, prof) = model::step_profiled(&model, &shards, &mut caches, &mut kv, &[id], pos)?;
+        phases.attention_s += prof.phases.attention_s;
+        phases.attn_kda_proj_s += prof.phases.attn_kda_proj_s;
+        phases.attn_kda_recur_s += prof.phases.attn_kda_recur_s;
+        phases.attn_mla_s += prof.phases.attn_mla_s;
+        phases.expert_wait_s += prof.phases.expert_wait_s;
+        phases.expert_matmul_s += prof.phases.expert_matmul_s;
+        lm_head_s += prof.lm_head_s;
+        for v in logits {
+            for b in v.to_bits().to_le_bytes() {
+                fingerprint = (fingerprint ^ b as u64).wrapping_mul(0x100000001b3);
+            }
+        }
         let (hits, misses, _) = caches.hit_miss_totals();
         eprintln!(
             "  step {}/{} at {:.1}s elapsed (expert cache totals: {hits} hits, {misses} misses)",
@@ -78,6 +109,19 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         );
     }
     let decode_elapsed = t_decode.elapsed().as_secs_f32();
-    println!("{steps} teacher-forced decode steps in {decode_elapsed:.2}s ({:.4} tok/s)", steps as f32 / decode_elapsed);
+    println!("{steps} teacher-forced decode steps in {decode_elapsed:.2}s ({:.4} tok/s), logits fingerprint {fingerprint:016x}", steps as f32 / decode_elapsed);
+    // Per-token phase buckets over the decode steps (N5a instrumentation — the same numbers
+    // `/profile` serves in --serve, without needing a server boot to read them).
+    let per = |v: f32| v / steps as f32;
+    println!(
+        "per-token buckets: expert {:.3}s (wait {:.3}s), attention {:.3}s (kda_proj {:.3}s, kda_recur {:.3}s, mla {:.3}s), lm_head {:.3}s",
+        per(phases.expert_matmul_s),
+        per(phases.expert_wait_s),
+        per(phases.attention_s),
+        per(phases.attn_kda_proj_s),
+        per(phases.attn_kda_recur_s),
+        per(phases.attn_mla_s),
+        per(lm_head_s)
+    );
     Ok(())
 }
diff --git a/src/bin/gen_k3_synth.rs b/src/bin/gen_k3_synth.rs
new file mode 100644
index 0000000..7fcd117
--- /dev/null
+++ b/src/bin/gen_k3_synth.rs
@@ -0,0 +1,323 @@
+//! Generates a **synthetic, loadable Kimi K3 checkpoint** at the real per-layer widths but only
+//! a handful of layers — the Phase 1 test fixture from `K3_OPTIMIZE_BRIEF.md`. Real-checkpoint
+//! iteration costs a 1.56 TB / ~10-minute load per experiment; K3 decode is per-layer
+//! homogeneous, so a random-weight checkpoint with the SAME per-layer shapes but few layers
+//! reproduces the kernel / parallelism / NUMA behaviour this brief tunes at ~5-10% of the
+//! footprint (≈90 GB at the default 6 layers).
+//!
+//! The weights are random garbage: this fixture is a PERFORMANCE proxy, never a correctness one
+//! (token-exact correctness is `tests/teacher_forcing_k3.rs`'s job, against the tiny oracle). The
+//! routed experts are emitted as raw MXFP4 `.weight_packed`/`.weight_scale` byte pairs with the
+//! exact byte counts `qt_load_mxfp4` checks (`rows*ceil(cols/2)` and `rows*ceil(cols/32)`); their
+//! contents are random bytes, with E8M0 scale bytes pinned to [120,130] so dequantised magnitudes
+//! stay finite (garbage logits are fine, NaN/inf poisoning is avoidable noise — see the brief).
+//!
+//! Layout: one `*.safetensors` shard per layer plus one model-level shard, all under the output
+//! dir. `Shards::open` scans `*.safetensors` sorted by filename with no index needed
+//! (`src/safetensors.rs`), so the `aa_model` / `layer_NN` names below load as one checkpoint.
+//! Tensor names use K3's real `language_model.` prefix and `ExpertNaming::KimiK3Mxfp4`'s
+//! `block_sparse_moe.experts.{eid}.w{1,3,2}` expert names (verified against `tensor_specs`).
+//!
+//! Usage:
+//!   cargo run --release --bin gen_k3_synth -- --out <dir> [--layers 6] [--experts 896] \
+//!       [--tokenizer-src /data/hf/hub/kimi-k3]
+//!
+//! `--tokenizer-src DIR` copies `tiktoken.model` + `tokenizer_config.json` from DIR so
+//! `teacher_forced_decode_bench` (which needs a tokenizer) can run against the fixture; `k3_smoke`
+//! needs no tokenizer.
+
+use serde_json::{Map, Value, json};
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use std::path::{Path, PathBuf};
+
+// ---- Real Kimi K3 per-layer dimensions (from config.rs / model.rs, verified against the real
+// checkpoint's config.json). Everything a decoded token actually touches is at these widths. ----
+const HIDDEN: usize = 7168; // hidden_size (d)
+const MLA_HEADS: usize = 96; // num_attention_heads (h)
+const QK_NOPE: usize = 128;
+const QK_ROPE: usize = 64;
+const V_HEAD: usize = 128;
+const KV_LORA: usize = 512;
+const Q_LORA: usize = 1536;
+const VOCAB: usize = 163840;
+const DENSE_INTER: usize = 33792; // intermediate_size (dense layer 0)
+const MOE_INTER: usize = 3072; // moe_intermediate_size
+const MOE_HIDDEN: usize = 3584; // routed_expert_hidden_size (latent width)
+const N_SHARED: usize = 2;
+const KDA_HEAD_DIM: usize = 128; // linear_attn_config.head_dim
+const KDA_N_HEADS: usize = 96; // linear_attn_config.num_heads
+const KDA_KERNEL: usize = 4; // short_conv_kernel_size
+const FIRST_DENSE: usize = 1; // first_k_dense_replace
+
+fn d_inner() -> usize {
+    KDA_HEAD_DIM * KDA_N_HEADS
+}
+
+/// One tensor to emit into a shard: logical name, safetensors dtype string, shape, and the
+/// generator that fills its bytes (fed a per-tensor deterministic seed).
+struct Tensor {
+    name: String,
+    dtype: &'static str,
+    shape: Vec<usize>,
+    nbytes: usize,
+    fill: Fill,
+}
+
+#[derive(Clone, Copy)]
+enum Fill {
+    /// Random f32 in ~[-1,1], little-endian.
+    F32,
+    /// Random MXFP4 packed nibble bytes (any value is a valid pair of E2M1 codes).
+    PackedU8,
+    /// E8M0 scale bytes constrained to [120,130] so 2^(b-127) stays a small finite factor.
+    ScaleU8,
+}
+
+fn xorshift(seed: &mut u32) -> u32 {
+    *seed ^= *seed << 13;
+    *seed ^= *seed >> 17;
+    *seed ^= *seed << 5;
+    *seed
+}
+
+fn f32t(name: String, shape: Vec<usize>) -> Tensor {
+    let n: usize = shape.iter().product::<usize>().max(1);
+    Tensor { name, dtype: "F32", shape, nbytes: n * 4, fill: Fill::F32 }
+}
+
+fn packed(name: String, rows: usize, cols: usize) -> Tensor {
+    let cols_b = cols.div_ceil(2);
+    Tensor { name, dtype: "U8", shape: vec![rows, cols_b], nbytes: rows * cols_b, fill: Fill::PackedU8 }
+}
+
+fn scale(name: String, rows: usize, cols: usize) -> Tensor {
+    let cols_b = cols.div_ceil(32);
+    Tensor { name, dtype: "U8", shape: vec![rows, cols_b], nbytes: rows * cols_b, fill: Fill::ScaleU8 }
+}
+
+/// Writes one shard file: 8-byte little-endian header length, JSON header, then each tensor's
+/// bytes generated on the fly (never materialising the whole >10 GB blob in memory).
+fn write_shard(path: &Path, tensors: &[Tensor]) -> std::io::Result<()> {
+    let mut header = Map::new();
+    header.insert("__metadata__".to_string(), json!({"format": "rabbit-k3-synth"}));
+    let mut offset = 0u64;
+    for t in tensors {
+        let start = offset;
+        offset += t.nbytes as u64;
+        header.insert(
+            t.name.clone(),
+            json!({"dtype": t.dtype, "shape": t.shape, "data_offsets": [start, offset]}),
+        );
+    }
+    let header_bytes = serde_json::to_vec(&Value::Object(header)).unwrap();
+    let f = File::create(path)?;
+    let mut w = BufWriter::with_capacity(1 << 22, f);
+    w.write_all(&(header_bytes.len() as u64).to_le_bytes())?;
+    w.write_all(&header_bytes)?;
+
+    // 4 MiB scratch, regenerated per tensor from a name-derived seed (deterministic output).
+    let mut buf = vec![0u8; 1 << 22];
+    for t in tensors {
+        let mut seed = t.name.bytes().fold(0x9E3779B9u32, |a, b| a.wrapping_mul(31).wrapping_add(b as u32)) | 1;
+        let mut remaining = t.nbytes;
+        while remaining > 0 {
+            let chunk = remaining.min(buf.len());
+            match t.fill {
+                Fill::F32 => {
+                    // Fill whole f32 words; chunk is a multiple of 4 except possibly the tail,
+                    // but nbytes is always a multiple of 4 for F32 tensors so alignment holds.
+                    for word in buf[..chunk].chunks_mut(4) {
+                        let r = xorshift(&mut seed);
+                        let v = ((r as f32 / u32::MAX as f32) - 0.5) * 2.0;
+                        word.copy_from_slice(&v.to_le_bytes()[..word.len()]);
+                    }
+                }
+                Fill::PackedU8 => {
+                    for b in &mut buf[..chunk] {
+                        *b = (xorshift(&mut seed) & 0xFF) as u8;
+                    }
+                }
+                Fill::ScaleU8 => {
+                    for b in &mut buf[..chunk] {
+                        *b = 120 + (xorshift(&mut seed) % 11) as u8; // [120,130]
+                    }
+                }
+            }
+            w.write_all(&buf[..chunk])?;
+            remaining -= chunk;
+        }
+    }
+    w.flush()
+}
+
+/// Builds the tensor list for one transformer layer (attention + FFN + residual-block norms).
+fn layer_tensors(i: usize, is_kda: bool, is_moe: bool, n_experts: usize) -> Vec<Tensor> {
+    let d = HIDDEN;
+    let p = |s: &str| format!("language_model.model.layers.{i}.{s}");
+    let ap = |s: &str| format!("language_model.model.layers.{i}.self_attn.{s}");
+    let mut ts = Vec::new();
+    ts.push(f32t(p("input_layernorm.weight"), vec![d]));
+    ts.push(f32t(p("post_attention_layernorm.weight"), vec![d]));
+    // attn_res_block_size = 2 -> per-layer residual norms/projs on every layer.
+    ts.push(f32t(p("self_attention_res_norm.weight"), vec![d]));
+    ts.push(f32t(p("self_attention_res_proj.weight"), vec![1, d]));
+    ts.push(f32t(p("mlp_res_norm.weight"), vec![d]));
+    ts.push(f32t(p("mlp_res_proj.weight"), vec![1, d]));
+
+    if is_kda {
+        let di = d_inner();
+        ts.push(f32t(ap("q_proj.weight"), vec![di, d]));
+        ts.push(f32t(ap("k_proj.weight"), vec![di, d]));
+        ts.push(f32t(ap("v_proj.weight"), vec![di, d]));
+        ts.push(f32t(ap("q_conv1d.weight"), vec![di, 1, KDA_KERNEL]));
+        ts.push(f32t(ap("k_conv1d.weight"), vec![di, 1, KDA_KERNEL]));
+        ts.push(f32t(ap("v_conv1d.weight"), vec![di, 1, KDA_KERNEL]));
+        ts.push(f32t(ap("f_a_proj.weight"), vec![KDA_HEAD_DIM, d]));
+        ts.push(f32t(ap("f_b_proj.weight"), vec![di, KDA_HEAD_DIM]));
+        ts.push(f32t(ap("dt_bias"), vec![di]));
+        ts.push(f32t(ap("A_log"), vec![1, 1, KDA_N_HEADS, 1]));
+        ts.push(f32t(ap("b_proj.weight"), vec![KDA_N_HEADS, d]));
+        // use_full_rank_gate = true -> single g_proj (see config.rs).
+        ts.push(f32t(ap("g_proj.weight"), vec![di, d]));
+        ts.push(f32t(ap("o_norm.weight"), vec![KDA_HEAD_DIM]));
+        ts.push(f32t(ap("o_proj.weight"), vec![d, di]));
+    } else {
+        // MLA with q_lora>0 and mla_use_output_gate.
+        ts.push(f32t(ap("q_a_proj.weight"), vec![Q_LORA, d]));
+        ts.push(f32t(ap("q_a_layernorm.weight"), vec![Q_LORA]));
+        ts.push(f32t(ap("q_b_proj.weight"), vec![MLA_HEADS * (QK_NOPE + QK_ROPE), Q_LORA]));
+        ts.push(f32t(ap("kv_a_proj_with_mqa.weight"), vec![KV_LORA + QK_ROPE, d]));
+        ts.push(f32t(ap("kv_a_layernorm.weight"), vec![KV_LORA]));
+        ts.push(f32t(ap("kv_b_proj.weight"), vec![MLA_HEADS * (QK_NOPE + V_HEAD), KV_LORA]));
+        ts.push(f32t(ap("o_proj.weight"), vec![d, MLA_HEADS * V_HEAD]));
+        ts.push(f32t(ap("g_proj.weight"), vec![MLA_HEADS * V_HEAD, d]));
+    }
+
+    if !is_moe {
+        ts.push(f32t(p("mlp.gate_proj.weight"), vec![DENSE_INTER, d]));
+        ts.push(f32t(p("mlp.up_proj.weight"), vec![DENSE_INTER, d]));
+        ts.push(f32t(p("mlp.down_proj.weight"), vec![d, DENSE_INTER]));
+    } else {
+        let mp = |s: &str| format!("language_model.model.layers.{i}.block_sparse_moe.{s}");
+        let s_i = MOE_INTER * N_SHARED;
+        ts.push(f32t(mp("gate.weight"), vec![n_experts, d]));
+        ts.push(f32t(mp("gate.e_score_correction_bias"), vec![n_experts]));
+        ts.push(f32t(mp("shared_experts.gate_proj.weight"), vec![s_i, d]));
+        ts.push(f32t(mp("shared_experts.up_proj.weight"), vec![s_i, d]));
+        ts.push(f32t(mp("shared_experts.down_proj.weight"), vec![d, s_i]));
+        ts.push(f32t(mp("routed_expert_down_proj.weight"), vec![MOE_HIDDEN, d]));
+        ts.push(f32t(mp("routed_expert_up_proj.weight"), vec![d, MOE_HIDDEN]));
+        ts.push(f32t(mp("routed_expert_norm.weight"), vec![MOE_HIDDEN]));
+        // Routed experts as MXFP4 pairs (KimiK3Mxfp4 naming: w1/w3 = [moe_inter, moe_hidden],
+        // w2 = [moe_hidden, moe_inter]; qt_load_mxfp4 appends .weight_packed/.weight_scale).
+        for eid in 0..n_experts {
+            let ep = |suf: &str| format!("language_model.model.layers.{i}.block_sparse_moe.experts.{eid}.{suf}");
+            for (w, rows, cols) in [("w1", MOE_INTER, MOE_HIDDEN), ("w3", MOE_INTER, MOE_HIDDEN), ("w2", MOE_HIDDEN, MOE_INTER)] {
+                ts.push(packed(format!("{}.weight_packed", ep(w)), rows, cols));
+                ts.push(scale(format!("{}.weight_scale", ep(w)), rows, cols));
+            }
+        }
+    }
+    ts
+}
+
+fn model_level_tensors() -> Vec<Tensor> {
+    let d = HIDDEN;
+    vec![
+        f32t("language_model.model.embed_tokens.weight".into(), vec![VOCAB, d]),
+        f32t("language_model.lm_head.weight".into(), vec![VOCAB, d]),
+        f32t("language_model.model.norm.weight".into(), vec![d]),
+        f32t("language_model.model.output_attn_res_norm.weight".into(), vec![d]),
+        f32t("language_model.model.output_attn_res_proj.weight".into(), vec![1, d]),
+    ]
+}
+
+/// Writes config.json for `n_layers`, with a 3:1 KDA:full-MLA attention pattern. `kda_layers`
+/// and `full_attn_layers` are **1-indexed** and must together cover every layer exactly once
+/// (config.rs errors otherwise); full-MLA sits at 1-based positions p where p%4==3 (matching the
+/// real 4-layer template's `full_attn_layers:[3]`), KDA everywhere else.
+fn write_config(dir: &Path, n_layers: usize) {
+    let full: Vec<usize> = (1..=n_layers).filter(|p| p % 4 == 3).collect();
+    let kda: Vec<usize> = (1..=n_layers).filter(|p| p % 4 != 3).collect();
+    let linear_attn_config = json!({
+        "head_dim": KDA_HEAD_DIM, "num_heads": KDA_N_HEADS, "short_conv_kernel_size": KDA_KERNEL,
+        "gate_lower_bound": -5.0, "use_full_rank_gate": true,
+        "kda_layers": kda, "full_attn_layers": full
+    });
+    let text_config = json!({
+        "model_type": "kimi_linear",
+        "hidden_size": HIDDEN, "num_hidden_layers": n_layers, "num_attention_heads": MLA_HEADS,
+        "first_k_dense_replace": FIRST_DENSE, "q_lora_rank": Q_LORA, "kv_lora_rank": KV_LORA,
+        "qk_nope_head_dim": QK_NOPE, "qk_rope_head_dim": QK_ROPE, "v_head_dim": V_HEAD,
+        "num_experts": 896, "num_experts_per_token": 16, "num_shared_experts": N_SHARED,
+        "num_expert_group": 1, "topk_group": 1, "moe_intermediate_size": MOE_INTER,
+        "intermediate_size": DENSE_INTER, "vocab_size": VOCAB, "moe_renormalize": true,
+        "rms_norm_eps": 1e-5, "routed_scaling_factor": 1.0, "mla_use_nope": true,
+        "moe_router_activation_func": "sigmoid", "rope_theta": 10000.0, "eos_token_id": 163586,
+        "hidden_act": "situ", "activation_situ_beta": 4.0, "activation_situ_linear_beta": 25.0,
+        "routed_expert_hidden_size": MOE_HIDDEN, "latent_moe_use_norm": true,
+        "attn_res_block_size": 2, "mla_use_output_gate": true,
+        "linear_attn_config": linear_attn_config,
+        // Read from INSIDE text_config (config.rs:211) -> sets mxfp4_experts -> KimiK3Mxfp4 naming.
+        "quantization_config": { "quant_method": "compressed-tensors", "format": "mxfp4-pack-quantized" }
+    });
+    let cfg = json!({ "model_type": "kimi_k3", "text_config": text_config });
+    std::fs::write(dir.join("config.json"), serde_json::to_vec_pretty(&cfg).unwrap()).unwrap();
+}
+
+fn arg(args: &[String], flag: &str) -> Option<String> {
+    args.iter().position(|a| a == flag).and_then(|i| args.get(i + 1)).cloned()
+}
+
+fn main() -> std::io::Result<()> {
+    let args: Vec<String> = std::env::args().collect();
+    let out = PathBuf::from(arg(&args, "--out").expect("--out <dir> is required"));
+    let n_layers: usize = arg(&args, "--layers").and_then(|s| s.parse().ok()).unwrap_or(6);
+    let n_experts: usize = arg(&args, "--experts").and_then(|s| s.parse().ok()).unwrap_or(896);
+    assert!(n_layers >= 2, "need at least 2 layers (1 dense + >=1 MoE) to exercise the MoE path");
+    std::fs::create_dir_all(&out)?;
+
+    write_config(&out, n_layers);
+    eprintln!("wrote config.json ({n_layers} layers, {n_experts} experts/MoE layer)");
+
+    let t0 = std::time::Instant::now();
+    write_shard(&out.join("aa_model.safetensors"), &model_level_tensors())?;
+    eprintln!("wrote model-level shard");
+
+    let full: Vec<usize> = (1..=n_layers).filter(|p| p % 4 == 3).collect();
+    let mut total_bytes = 0u64;
+    for i in 0..n_layers {
+        let is_kda = !full.contains(&(i + 1)); // 1-indexed membership
+        let is_moe = i >= FIRST_DENSE;
+        let ts = layer_tensors(i, is_kda, is_moe, n_experts);
+        let bytes: u64 = ts.iter().map(|t| t.nbytes as u64).sum();
+        total_bytes += bytes;
+        let path = out.join(format!("layer_{i:02}.safetensors"));
+        write_shard(&path, &ts)?;
+        eprintln!(
+            "  layer {i}: {} attn, {} FFN, {:.1} GiB",
+            if is_kda { "KDA" } else { "MLA" },
+            if is_moe { "MoE" } else { "dense" },
+            bytes as f64 / (1024.0 * 1024.0 * 1024.0)
+        );
+    }
+
+    if let Some(src) = arg(&args, "--tokenizer-src") {
+        let src = PathBuf::from(src);
+        for f in ["tiktoken.model", "tokenizer_config.json"] {
+            if src.join(f).exists() {
+                std::fs::copy(src.join(f), out.join(f))?;
+                eprintln!("copied {f}");
+            }
+        }
+    }
+
+    eprintln!(
+        "done in {:.1}s — {:.1} GiB total under {}",
+        t0.elapsed().as_secs_f32(),
+        total_bytes as f64 / (1024.0 * 1024.0 * 1024.0),
+        out.display()
+    );
+    Ok(())
+}
diff --git a/src/chat.rs b/src/chat.rs
index c0d52c7..87400dc 100644
--- a/src/chat.rs
+++ b/src/chat.rs
@@ -26,6 +26,9 @@ pub struct LoadArgs {
     /// opt-in CACHE_ROUTE (see `moe::RouteConfig`) — off by default, matching colibrì's own
     /// stance, since it's still unmeasured on rabbit's own architecture.
     pub cache_route: bool,
+    /// `--preload-experts` (Phase 4b): after warm-start, load every MoE layer's experts up front
+    /// so token 1 runs at steady-state speed instead of paying a long streaming warmup.
+    pub preload_experts: bool,
     /// Extra directories to scan for `.safetensors` shards, alongside `model_dir` — lets a
     /// checkpoint's shards be split across separate drives (e.g. a second NVMe added for
     /// capacity/bandwidth, without an OS-level RAID array — see `Shards::open_multi`'s doc).
@@ -81,6 +84,13 @@ pub fn load_session(args: &LoadArgs) -> Result<Session, Box<dyn std::error::Erro
             eprintln!("usage cache: {} historical selections (below auto-pin threshold)", stats.hist);
         }
     }
+    if args.preload_experts {
+        eprintln!("preloading experts (--preload-experts)...");
+        let tp = std::time::Instant::now();
+        caches.preload(&model, &shards)?;
+        let (_, loaded, _) = caches.hit_miss_totals();
+        eprintln!("preload done in {:.1}s ({loaded} experts resident)", tp.elapsed().as_secs_f32());
+    }
     let sampling = SamplingConfig { temperature: args.temperature, nucleus: args.nucleus };
     let rng = Rng::new(args.seed);
     let stop_ids = model.stop_ids();
@@ -178,6 +188,51 @@ fn glm52_render_messages(messages: &[(Role, String)], think: bool) -> String {
     out
 }
 
+/// What the active family's [`OutputFilter`] decided about one freshly generated token.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum Emit {
+    /// Part of the visible reply — show it.
+    Text,
+    /// Structural markup the model emitted to close its own turn — swallow it, but still count
+    /// it as generated.
+    Skip,
+    /// The reply's channel closed: the turn is complete, stop generating.
+    Stop,
+}
+
+/// The output-side counterpart to [`render_turn`]/[`render_messages`]. Those open a turn in the
+/// family's prompt grammar; this closes it, deciding per generated token whether the model is
+/// still producing reply text or has begun emitting the markup that ends the turn.
+///
+/// Only Kimi K3 needs a real one. GLM-5.2 and Kimi Linear 48B end a turn with a single stop id,
+/// which `generate_reply`'s `stop_ids` check already handles without emitting it. K3 is the only
+/// family whose turn ends with a multi-token close sequence (`<|close|>response<|sep|>
+/// <|close|>message<|sep|>`) made of tokens that are not themselves stop ids — see
+/// [`crate::kimi_k3::chat_template::ResponseFilter`].
+pub enum OutputFilter {
+    Passthrough,
+    KimiK3(crate::kimi_k3::chat_template::ResponseFilter),
+}
+
+impl OutputFilter {
+    pub fn step(&mut self, token: &str) -> Emit {
+        match self {
+            OutputFilter::Passthrough => Emit::Text,
+            OutputFilter::KimiK3(f) => f.step(token),
+        }
+    }
+}
+
+/// Builds the [`OutputFilter`] matching how this turn's prompt was rendered — `think` must be the
+/// same flag handed to [`render_turn`]/[`render_messages`], since it decides which channel the
+/// generation prompt left open for the model to close.
+pub fn output_filter(model: &Model, think: bool) -> OutputFilter {
+    match model {
+        Model::KimiK3(_) => OutputFilter::KimiK3(crate::kimi_k3::chat_template::ResponseFilter::new(think)),
+        Model::Glm52(_) | Model::KimiLinear(_) => OutputFilter::Passthrough,
+    }
+}
+
 pub fn render_messages(model: &Model, messages: &[(Role, String)], think: bool) -> String {
     match model {
         Model::Glm52(_) => glm52_render_messages(messages, think),
@@ -202,6 +257,11 @@ pub struct TurnProfile {
     pub hits: u64,
     pub misses: u64,
     pub attention_s: f32,
+    /// N5a sub-buckets of `attention_s` (KDA projections / KDA recurrence / MLA) — zero for
+    /// families that don't split it; see `generate::Phases`.
+    pub attn_kda_proj_s: f32,
+    pub attn_kda_recur_s: f32,
+    pub attn_mla_s: f32,
     pub expert_wait_s: f32,
     pub expert_matmul_s: f32,
     pub lm_head_s: f32,
@@ -211,6 +271,9 @@ pub struct TurnProfile {
 impl TurnProfile {
     fn accumulate(&mut self, step: &StepProfile) {
         self.attention_s += step.phases.attention_s;
+        self.attn_kda_proj_s += step.phases.attn_kda_proj_s;
+        self.attn_kda_recur_s += step.phases.attn_kda_recur_s;
+        self.attn_mla_s += step.phases.attn_mla_s;
         self.expert_wait_s += step.phases.expert_wait_s;
         self.expert_matmul_s += step.phases.expert_matmul_s;
         self.lm_head_s += step.lm_head_s;
@@ -250,11 +313,18 @@ pub enum GenEvent<'a> {
 /// turn's `pos_base`), the count of tokens generated, and this whole turn's [`TurnProfile`]
 /// (every caller gets one — the timing overhead is a handful of `Instant::now()` calls per
 /// layer, negligible next to a real forward pass; only `server.rs` does anything with it).
+///
+/// `filter` must match how `turn_ids` was built: [`output_filter`] with the same `think` flag
+/// when the prompt came from [`render_turn`]/[`render_messages`], or [`OutputFilter::Passthrough`]
+/// for a raw prompt with no chat envelope to close (`--prompt`'s single-shot completion). Note
+/// the returned token count is what the model *generated*, which can exceed the reply text's
+/// token count when the filter swallowed closing markup.
 pub fn generate_reply(
     sess: &mut Session,
     kv: &mut KvState,
     turn_ids: &[usize],
     pos_base: usize,
+    mut filter: OutputFilter,
     mut on_event: impl FnMut(GenEvent),
 ) -> Result<(String, usize, usize, TurnProfile), Box<dyn std::error::Error>> {
     let turn_t = std::time::Instant::now();
@@ -276,17 +346,37 @@ pub fn generate_reply(
     let mut pos = pos_base + turn_ids.len();
 
     let mut out_ids = Vec::with_capacity(sess.max_tokens);
+    // The subset of `out_ids` that survived the filter — what the reply text is built from.
+    // Separate from `out_ids` so `max_tokens`, step indices and `completion_tokens` keep counting
+    // what the model actually generated, not what was shown.
+    let mut kept_ids: Vec<usize> = Vec::with_capacity(sess.max_tokens);
     while out_ids.len() < sess.max_tokens {
         let next = generate::pick_token(&logits, &sess.sampling, &mut sess.rng, None);
         if sess.stop_ids.contains(&next) {
             break;
         }
+        // Decoded here rather than after the forward step below (where it used to live) because
+        // the filter's verdict is needed first: `Stop` ends the turn on a token the family's
+        // grammar says closes the reply, and breaking now skips a forward pass — seconds to tens
+        // of seconds on a real model — whose logits would be thrown away.
+        let decoded = sess.tokenizer.decode(&[next as i32]);
+        let action = filter.step(&String::from_utf8_lossy(&decoded));
+        if action == Emit::Stop {
+            break;
+        }
+        // A `Skip`ped token still generated, so it still counts against `max_tokens` and still
+        // reports a `Token` event (callers' step/timing accounting stays exact) — it just carries
+        // no bytes, which every consumer already treats as nothing to show: `server.rs`'s SSE
+        // writer guards on `!s.is_empty()`, and the CLI prints an empty slice as nothing.
+        let shown: &[u8] = if action == Emit::Text { &decoded } else { &[] };
         out_ids.push(next);
+        if action == Emit::Text {
+            kept_ids.push(next);
+        }
         if out_ids.len() >= sess.max_tokens {
-            let decoded = sess.tokenizer.decode(&[next as i32]);
             on_event(GenEvent::Token {
                 token_id: next,
-                bytes: &decoded,
+                bytes: shown,
                 index: out_ids.len(),
                 max: sess.max_tokens,
                 seconds: 0.0,
@@ -306,10 +396,9 @@ pub fn generate_reply(
         let step_seconds = step_t.elapsed().as_secs_f32();
         (hits, misses, io_ns) = sess.caches.hit_miss_totals();
         io_wait_ns = sess.caches.io_wait_nanos_total();
-        let decoded = sess.tokenizer.decode(&[next as i32]);
         on_event(GenEvent::Token {
             token_id: next,
-            bytes: &decoded,
+            bytes: shown,
             index: out_ids.len(),
             max: sess.max_tokens,
             seconds: step_seconds,
@@ -321,8 +410,8 @@ pub fn generate_reply(
         pos += 1;
     }
 
-    let out_i32: Vec<i32> = out_ids.iter().map(|&id| id as i32).collect();
-    let text = String::from_utf8_lossy(&sess.tokenizer.decode(&out_i32)).into_owned();
+    let kept_i32: Vec<i32> = kept_ids.iter().map(|&id| id as i32).collect();
+    let text = String::from_utf8_lossy(&sess.tokenizer.decode(&kept_i32)).into_owned();
 
     profile.wall_s = turn_t.elapsed().as_secs_f32();
     profile.completion_tokens = out_ids.len();
diff --git a/src/expert_cache.rs b/src/expert_cache.rs
index caaa4ae..d254655 100644
--- a/src/expert_cache.rs
+++ b/src/expert_cache.rs
@@ -55,6 +55,7 @@ use crate::glm52::config::Cfg;
 use crate::glm52::model::{ModelError, qt_load};
 use crate::quant::{QT, QTKind};
 use crate::safetensors::Shards;
+use rayon::prelude::*;
 use std::collections::HashMap;
 
 /// Which on-disk tensor-name convention a checkpoint's routed experts use — the one piece of
@@ -525,6 +526,11 @@ enum LoadKind {
 /// every `ensure_loaded` call over that lifetime; see `uring_load::Ring`'s doc for why.
 pub struct ExpertCache {
     capacity: usize,
+    /// Total routed experts for this layer. When `capacity >= n_experts` the cache holds every
+    /// expert (full residency) and never evicts, so pinning — which only protects against
+    /// eviction — is pointless and is skipped entirely (see `insert_or_pin`). `usize::MAX` (the
+    /// `new` default) means "unbounded / unknown", i.e. treat pinning as potentially useful.
+    n_experts: usize,
     slots: Vec<ExpertSlot>,
     /// Checked before `slots` on every lookup, never touched by `insert`'s LRU eviction —
     /// colibrì's `m->pin[layer]`. Structurally separate from `slots` rather than a "never
@@ -587,17 +593,22 @@ impl ExpertCache {
         self.capacity
     }
 
+    /// Test/simple constructor: `n_experts` unknown, so pinning is treated as potentially useful
+    /// (`usize::MAX`). Production builds go through `for_family` with the real expert count so
+    /// full-residency caches can skip pointless pinning.
     pub fn new(capacity: usize) -> ExpertCache {
-        Self::for_family(capacity, ExpertNaming::Glm52)
+        Self::for_family(capacity, usize::MAX, ExpertNaming::Glm52)
     }
 
     /// Like `new`, but for a checkpoint whose routed experts use a different on-disk
     /// tensor-name convention (see `ExpertNaming`) — e.g. Kimi Linear's
     /// `block_sparse_moe.experts.{eid}.{w1,w2,w3}` instead of GLM-5.2's
-    /// `mlp.experts.{eid}.{gate_proj,up_proj,down_proj}`.
-    pub fn for_family(capacity: usize, naming: ExpertNaming) -> ExpertCache {
+    /// `mlp.experts.{eid}.{gate_proj,up_proj,down_proj}`. `n_experts` is the layer's total routed
+    /// expert count, so `capacity >= n_experts` (full residency) can skip pinning entirely.
+    pub fn for_family(capacity: usize, n_experts: usize, naming: ExpertNaming) -> ExpertCache {
         ExpertCache {
             capacity,
+            n_experts,
             slots: Vec::new(),
             pinned: Vec::new(),
             pin_candidates: std::collections::HashSet::new(),
@@ -625,6 +636,22 @@ impl ExpertCache {
         self.slots.is_empty()
     }
 
+    /// Whether this cache loads through an `io_uring` ring. `false` means every load is the
+    /// synchronous `sequential_fallback` path, whose `io_wait_nanos` counter is **structurally
+    /// always zero** — so a `[0.0s actual disk wait]` readout means "no ring here", NOT "the disk
+    /// was instant" (Phase 4c: the CLI must not print that misleading bracket for MXFP4, whose
+    /// naming never gets a ring — see `ExpertCache::new`).
+    pub fn has_ring(&self) -> bool {
+        #[cfg(target_os = "linux")]
+        {
+            self.ring.is_some()
+        }
+        #[cfg(not(target_os = "linux"))]
+        {
+            false
+        }
+    }
+
     /// Drops every cached slot (next lookup for any id is a miss) without tearing down the
     /// `io_uring` ring — for benchmarking/testing load performance repeatedly against the
     /// same long-lived cache, which is the realistic shape (a fresh `ExpertCache` per
@@ -824,8 +851,14 @@ impl ExpertCache {
     /// straight into `pinned` (mlocked, never evicted) instead of the ordinary LRU `slots` —
     /// the one and only place lazy/sticky promotion actually happens. `used` is irrelevant for
     /// a pinned slot, stamped 0.
+    ///
+    /// Skipped entirely at full residency (`capacity >= n_experts`): the cache then never evicts,
+    /// so pinning-to-prevent-eviction protects against nothing — and would just fail `mlock` for
+    /// every expert under a low `RLIMIT_MEMLOCK` (`--expert-cache 896` on the K3 target box hit
+    /// exactly this). The `&&` short-circuits before `remove`, so a candidate marked earlier is
+    /// simply left unused rather than promoted.
     fn insert_or_pin(&mut self, mut slot: ExpertSlot) {
-        if self.pin_candidates.remove(&slot.eid) {
+        if self.capacity < self.n_experts && self.pin_candidates.remove(&slot.eid) {
             slot.used = 0;
             mlock_best_effort(&slot);
             self.pinned.push(slot);
@@ -851,9 +884,122 @@ impl ExpertCache {
 }
 
 /// Used as `begin_loading`'s non-Linux (or ring-unavailable) synchronous path, and as
-/// `finish_loading`'s fallback when `uring_load::complete_batch` reports an I/O error.
+/// `finish_loading`'s fallback when `uring_load::complete_batch` reports an I/O error. This is the
+/// ONLY load path K3's MXFP4 experts ever take (their naming never gets an io_uring ring — see
+/// `ExpertCache::new`), so it is the whole warmup cost on the target box.
+///
+/// Parallelised over misses with rayon (Phase 4a, `K3_OPTIMIZE_BRIEF.md`): `Shards` reads are
+/// `&self` `pread` (thread-safe, never mmap — `safetensors.rs`), and an MXFP4 `load_expert` from
+/// warm page cache is nearly pure memcpy, so a single-threaded loop left the machine's 384 cores
+/// idle while streaming 16.73 MiB/expert one at a time. rayon's *ordered* `collect` into
+/// `Result<Vec<_>>` preserves input (miss) order, so cache insertion order and pin promotion are
+/// byte-for-byte unchanged and results stay deterministic; a load error short-circuits the collect
+/// exactly as `iter().collect()` did. Side benefit: distributes NUMA first-touch of the expert
+/// buffers across the loading threads instead of concentrating it on one.
 fn sequential_fallback(shards: &Shards, cfg: &Cfg, layer: usize, misses: &[usize], bits: u8, used: u64, naming: ExpertNaming) -> Result<Vec<ExpertSlot>, ModelError> {
-    misses.iter().map(|&eid| load_expert(shards, cfg, layer, eid, bits, used, naming)).collect()
+    if let Some(pools) = crate::numa::NodePools::get() {
+        return numa_homed_load(pools, shards, cfg, layer, misses, bits, used, naming);
+    }
+    misses.par_iter().map(|&eid| load_expert(shards, cfg, layer, eid, bits, used, naming)).collect()
+}
+
+/// Phase N3b (`NUMA_AMX_BRIEF.md`): with `--numa` active, each missing expert is loaded INSIDE
+/// its home node's pinned pool, so the buffer allocation and the `pread` fill — the first touch
+/// of every page — happen on a CPU of the node the expert will later be computed on
+/// (`numa::home_node` is also what `kimi_k3::moe`'s dispatch consults; placement and compute
+/// agree by construction). Zero `mbind` calls: first touch IS the placement mechanism.
+///
+/// The returned Vec is in the exact input (miss) order, same as the `par_iter` collect above, so
+/// cache insertion order, pin promotion, and determinism are unchanged — only WHERE each load
+/// ran differs. Runs during preload and for any mid-decode miss alike. Applies to every family
+/// that takes this path (for GLM/Kimi-Linear the io_uring ring normally bypasses it; K3's MXFP4
+/// always lands here).
+#[allow(clippy::too_many_arguments)]
+fn numa_homed_load(
+    pools: &crate::numa::NodePools,
+    shards: &Shards,
+    cfg: &Cfg,
+    layer: usize,
+    misses: &[usize],
+    bits: u8,
+    used: u64,
+    naming: ExpertNaming,
+) -> Result<Vec<ExpertSlot>, ModelError> {
+    let n_experts = cfg.n_experts as usize;
+    let mut by_node: Vec<Vec<usize>> = vec![Vec::new(); pools.n()];
+    for (k, &eid) in misses.iter().enumerate() {
+        by_node[crate::numa::home_node(layer, n_experts, eid, pools.n())].push(k);
+    }
+    let mut results: Vec<Option<Result<ExpertSlot, ModelError>>> = (0..misses.len()).map(|_| None).collect();
+    {
+        // Each miss index is owned by exactly one node's group, so the mutexes are uncontended —
+        // they exist to hand `&mut` result slots across the `run_all` closure's `Fn` boundary in
+        // safe Rust, not to serialize anything.
+        let slots: Vec<std::sync::Mutex<&mut Option<Result<ExpertSlot, ModelError>>>> = results.iter_mut().map(std::sync::Mutex::new).collect();
+        pools.run_all(|i| {
+            // Parallel WITHIN the node's pinned pool: warm-page-cache MXFP4 loads are memcpy-bound,
+            // and the copying threads are exactly where the pages should land.
+            by_node[i].par_iter().for_each(|&k| {
+                let loaded = load_expert(shards, cfg, layer, misses[k], bits, used, naming);
+                **slots[k].lock().unwrap() = Some(loaded);
+            });
+        });
+    }
+    results.into_iter().map(|r| r.expect("every miss index was assigned to exactly one node group")).collect()
+}
+
+/// Phase 4b preload (`K3_OPTIMIZE_BRIEF.md`): fills every MoE layer's cache up front so token 1
+/// runs at steady-state speed instead of paying hundreds of slow-warmup tokens. Loads are chunked
+/// per layer and parallel within a chunk (4a's `sequential_fallback`). `caches` is the per-layer
+/// `Vec<Option<ExpertCache>>` (None = a dense layer); `cfg` is the *expert-width* config (K3's
+/// latent width), same one the family's dispatch passes to `ensure_loaded`. Generic across all
+/// three families — the caller supplies the right `cfg`/`ebits`/`n_experts`.
+///
+/// Semantics at `capacity < n_experts` (D2): fill exactly to capacity — in usage-histogram order
+/// when a `.rabbit_usage` history was seeded (`warm_start`), else expert-id order — and say which
+/// in the log. At `capacity >= n_experts` every expert is loaded (full residency; the 1.32 TiB
+/// target-box case) and order is irrelevant.
+pub(crate) fn preload_layers(caches: &mut [Option<ExpertCache>], shards: &Shards, cfg: &Cfg, ebits: u8, n_experts: usize) -> Result<(), ModelError> {
+    let total_moe = caches.iter().filter(|c| c.is_some()).count();
+    let mut done = 0;
+    for (layer, cache_opt) in caches.iter_mut().enumerate() {
+        let Some(cache) = cache_opt else { continue };
+        let cap = cache.capacity().max(1);
+        let (eids, order) = preload_order(cache, n_experts, cap);
+        for chunk in eids.chunks(cap) {
+            cache.ensure_loaded(shards, cfg, layer, chunk, ebits)?;
+        }
+        done += 1;
+        eprintln!("  preload: MoE layer {layer} — {} experts resident ({order}) [{done}/{total_moe} layers]", eids.len());
+    }
+    Ok(())
+}
+
+/// The id list `preload_layers` loads for one layer, plus a human label for the log. See
+/// `preload_layers`' doc for the D2 order policy.
+fn preload_order(cache: &ExpertCache, n_experts: usize, cap: usize) -> (Vec<usize>, &'static str) {
+    if cap >= n_experts {
+        return ((0..n_experts).collect(), "full residency");
+    }
+    let mut usage: Vec<(usize, u64)> = cache.usage_counts().filter(|&(_, c)| c > 0).collect();
+    if usage.is_empty() {
+        return ((0..cap).collect(), "expert-id order — no usage history");
+    }
+    usage.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0)));
+    let mut eids: Vec<usize> = usage.into_iter().map(|(e, _)| e).take(cap).collect();
+    if eids.len() < cap {
+        // Usage history covered fewer than `cap` experts — top up in id order so the cache fills.
+        let have: std::collections::HashSet<usize> = eids.iter().copied().collect();
+        for e in 0..n_experts {
+            if eids.len() >= cap {
+                break;
+            }
+            if !have.contains(&e) {
+                eids.push(e);
+            }
+        }
+    }
+    (eids, "usage-histogram order")
 }
 
 fn load_expert(shards: &Shards, cfg: &Cfg, layer: usize, eid: usize, bits: u8, used: u64, naming: ExpertNaming) -> Result<ExpertSlot, ModelError> {
@@ -899,12 +1045,18 @@ fn qt_load_mxfp4(shards: &Shards, base_name: &str, rows: usize, cols: usize) ->
     Ok(QT { rows, cols, bits: 4, kind: QTKind::MxFp4 { data, block_scale } })
 }
 
+/// Set the first time an `mlock` fails this process, so the best-effort warning prints exactly once
+/// instead of once per pinned expert buffer (which, with a low `RLIMIT_MEMLOCK`, is hundreds of
+/// identical lines). See `mlock_best_effort`.
+static MLOCK_WARNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
 /// Best-effort `mlock` on a pinned expert's backing buffers, so the OS can't swap out memory
 /// pinning exists specifically to keep resident — same "unsafe libc call, best-effort, never a
 /// hard error" precedent already established for `posix_fadvise` in `safetensors.rs`. A failure
-/// (commonly `RLIMIT_MEMLOCK` too low for an unprivileged process) is logged once and otherwise
-/// ignored: losing the OS-level guarantee just means a pinned expert *could* get swapped under
-/// real memory pressure, same risk profile as not pinning it at all — not a correctness issue.
+/// (commonly `RLIMIT_MEMLOCK` too low for an unprivileged process) is logged ONCE (see
+/// `MLOCK_WARNED`) and otherwise ignored: losing the OS-level guarantee just means a pinned expert
+/// *could* get swapped under real memory pressure, same risk profile as not pinning it at all —
+/// not a correctness issue.
 fn mlock_best_effort(slot: &ExpertSlot) {
     let gate_up_qts: Vec<&QT> = match &slot.gate_up {
         GateUp::Separate { gate, up } => vec![gate, up],
@@ -925,7 +1077,21 @@ fn mlock_best_effort(slot: &ExpertSlot) {
         };
         for (ptr, len) in bufs {
             if len > 0 && unsafe { libc::mlock(ptr as *const libc::c_void, len) } != 0 {
-                eprintln!("usage cache: mlock failed for a pinned expert (best-effort, continuing): {}", std::io::Error::last_os_error());
+                // Log only the FIRST failure this process. With RLIMIT_MEMLOCK too low every pinned
+                // expert's every buffer fails identically, so the old per-buffer `eprintln!` flooded
+                // stderr with hundreds of copies of the same line (the doc always said "logged
+                // once" -- this makes the code match it). Still best-effort: pinning is silently
+                // skipped, not a hard error.
+                if !MLOCK_WARNED.swap(true, std::sync::atomic::Ordering::Relaxed) {
+                    eprintln!(
+                        "usage cache: mlock failed for a pinned expert ({}) -- expert pinning is \
+                         disabled for this run (best-effort, harmless: it only guards against the OS \
+                         swapping a pinned expert out). Raise RLIMIT_MEMLOCK (`ulimit -l unlimited`) \
+                         or pass `--no-usage-cache` to skip pinning entirely. Further mlock failures \
+                         this run are suppressed.",
+                        std::io::Error::last_os_error()
+                    );
+                }
             }
         }
     }
@@ -1202,7 +1368,7 @@ mod tests {
         let fixture = build_kimi_experts_fixture("rabbit_test_ecache_kimi_naming", 2, 4, 6);
         let shards = Shards::open(&fixture.0).unwrap();
         let cfg = tiny_cfg(2, 4, 6);
-        let mut cache = ExpertCache::for_family(8, ExpertNaming::KimiLinear);
+        let mut cache = ExpertCache::for_family(8, usize::MAX, ExpertNaming::KimiLinear);
 
         let slot = cache.get_or_load(&shards, &cfg, 0, 1, 32).unwrap();
         assert_eq!(slot.gate().rows, 4);
@@ -1224,13 +1390,13 @@ mod tests {
         let shards = Shards::open(&fixture.0).unwrap();
         let cfg = tiny_cfg(3, 4, 6);
 
-        let mut cache_seq = ExpertCache::for_family(8, ExpertNaming::KimiLinear);
+        let mut cache_seq = ExpertCache::for_family(8, usize::MAX, ExpertNaming::KimiLinear);
         let eids = [0usize, 1, 2];
         for &eid in &eids {
             cache_seq.get_or_load(&shards, &cfg, 0, eid, 32).unwrap();
         }
 
-        let mut cache_batch = ExpertCache::for_family(8, ExpertNaming::KimiLinear);
+        let mut cache_batch = ExpertCache::for_family(8, usize::MAX, ExpertNaming::KimiLinear);
         cache_batch.ensure_loaded(&shards, &cfg, 0, &eids, 32).unwrap();
 
         for &eid in &eids {
@@ -1254,7 +1420,7 @@ mod tests {
         let fixture = build_k3_mxfp4_experts_fixture("rabbit_test_ecache_k3_mxfp4", moe_inter, hidden);
         let shards = Shards::open(&fixture.0).unwrap();
         let cfg = tiny_cfg(1, moe_inter as i32, hidden as i32);
-        let mut cache = ExpertCache::for_family(8, ExpertNaming::KimiK3Mxfp4);
+        let mut cache = ExpertCache::for_family(8, usize::MAX, ExpertNaming::KimiK3Mxfp4);
 
         let slot = cache.get_or_load(&shards, &cfg, 0, 0, 4).unwrap();
         assert_eq!(slot.gate().rows, moe_inter);
@@ -1326,7 +1492,7 @@ mod tests {
             sep_cache.get_or_load(&sep_shards, &cfg, 0, eid, 32).unwrap();
         }
 
-        let mut fused_cache = ExpertCache::for_family(8, ExpertNaming::Glm52FusedGateUp);
+        let mut fused_cache = ExpertCache::for_family(8, usize::MAX, ExpertNaming::Glm52FusedGateUp);
         fused_cache.ensure_loaded(&fused_shards, &cfg, 0, &eids, 32).unwrap();
 
         for &eid in &eids {
@@ -1359,7 +1525,7 @@ mod tests {
         let cfg = tiny_cfg(n_experts as i32, moe_inter as i32, hidden as i32);
 
         let mut sep_cache = ExpertCache::new(8);
-        let mut fused_cache = ExpertCache::for_family(8, ExpertNaming::Glm52FusedGateUp);
+        let mut fused_cache = ExpertCache::for_family(8, usize::MAX, ExpertNaming::Glm52FusedGateUp);
 
         for eid in 0..n_experts {
             let sep_slot = sep_cache.get_or_load(&sep_shards, &cfg, 0, eid, 32).unwrap();
@@ -1677,6 +1843,27 @@ mod tests {
         assert_eq!(cache.misses, 1);
     }
 
+    #[test]
+    fn full_residency_cache_never_pins_even_with_marked_candidates() {
+        // capacity == n_experts: the cache holds every expert and can never evict, so pinning is
+        // pointless. A marked candidate must load into the ordinary LRU `slots`, NOT the pinned
+        // tier -- so no `mlock` is ever attempted (the `--expert-cache 896` / low-RLIMIT_MEMLOCK
+        // mlock-spam fix). `for_family` with the real `n_experts` is what production uses; the
+        // `new`-based pin tests above stay unaffected because `new` leaves `n_experts` unbounded.
+        let n_experts = 3;
+        let fixture = build_experts_fixture("rabbit_test_ecache_full_residency_no_pin", n_experts, 4, 6);
+        let shards = Shards::open(&fixture.0).unwrap();
+        let cfg = tiny_cfg(n_experts as i32, 4, 6);
+        let mut cache = ExpertCache::for_family(n_experts, n_experts, ExpertNaming::Glm52);
+
+        cache.mark_pin_candidates(std::iter::once(1usize));
+        cache.get_or_load(&shards, &cfg, 0, 1, 32).unwrap();
+        assert!(!cache.is_pinned(1), "at full residency a candidate must not be pinned");
+        assert_eq!(cache.pinned_len(), 0, "nothing should ever land in the pinned/mlocked tier");
+        assert_eq!(cache.len(), 1, "the expert lives in the ordinary LRU slots instead");
+        assert!(cache.get(1).is_some());
+    }
+
     #[test]
     fn a_promoted_expert_survives_lru_pressure_at_capacity_one() {
         let fixture = build_experts_fixture("rabbit_test_ecache_pin_survives_lru", 4, 4, 6);
diff --git a/src/generate.rs b/src/generate.rs
index 5b492d9..01ff501 100644
--- a/src/generate.rs
+++ b/src/generate.rs
@@ -63,7 +63,7 @@ impl ExpertCaches {
         let v = model
             .layers
             .iter()
-            .map(|l| if matches!(l.ffn, Ffn::Moe(_)) { Some(ExpertCache::for_family(capacity, naming)) } else { None })
+            .map(|l| if matches!(l.ffn, Ffn::Moe(_)) { Some(ExpertCache::for_family(capacity, model.cfg.n_experts as usize, naming)) } else { None })
             .collect();
         ExpertCaches(v)
     }
@@ -84,6 +84,21 @@ impl ExpertCaches {
         self.0.iter().flatten().map(|c| c.io_wait_nanos).sum()
     }
 
+    /// Whether any layer's cache loads through an `io_uring` ring. `false` (every MXFP4/K3 run,
+    /// whose naming never gets a ring) means `io_wait_nanos_total` is structurally zero, so the
+    /// CLI must not present it as "actual disk wait" (Phase 4c) — see `ExpertCache::has_ring`.
+    pub fn any_has_ring(&self) -> bool {
+        self.0.iter().flatten().any(|c| c.has_ring())
+    }
+
+    /// Phase 4b: preload every MoE layer's routed experts up front (see
+    /// `expert_cache::preload_layers`). GLM-5.2 experts are at the model hidden width, so the
+    /// model's own cfg supplies the shapes.
+    pub fn preload(&mut self, model: &Model, shards: &Shards) -> Result<(), ModelError> {
+        let n_experts = model.cfg.n_experts as usize;
+        Ok(crate::expert_cache::preload_layers(&mut self.0, shards, &model.cfg, model.ebits, n_experts)?)
+    }
+
     /// Reads `<model_dir>/.rabbit_usage` (if any), seeds every MoE layer's usage counters from
     /// it, and — if the persisted total selection count (`hist`, summed across ALL layers)
     /// reaches `usage_cache::HIST_THRESHOLD` — marks each layer's top
@@ -170,6 +185,16 @@ pub struct Phases {
     pub attention_s: f32,
     pub expert_wait_s: f32,
     pub expert_matmul_s: f32,
+    /// Phase N5a (`NUMA_AMX_BRIEF.md`) sub-buckets of `attention_s`, filled by architectures
+    /// that split their attention cost (Kimi K3's `layer_forward`/`kda_step` today; GLM-5.2 and
+    /// Kimi Linear leave them 0). `attn_mla_s` is the MLA layers' whole attention time;
+    /// `attn_kda_proj_s` is the KDA layers' eight per-token projection matmuls;
+    /// `attn_kda_recur_s` is their conv/normalize/96-head-recurrence remainder — so
+    /// `attention_s ≈ attn_mla_s + attn_kda_proj_s + attn_kda_recur_s` (small glue excepted).
+    /// Instrumentation only: the math is untouched, per the brief's scope guard.
+    pub attn_kda_proj_s: f32,
+    pub attn_kda_recur_s: f32,
+    pub attn_mla_s: f32,
 }
 
 /// A profiled `step_profiled` call's full timing breakdown: [`Phases`] plus the final lm_head
@@ -1074,26 +1099,30 @@ mod tests {
         // layer 2 (MoE + DSA-shared) gets none, so it must end up with zero candidates.
         crate::usage_cache::save(&usage_path, vec![(1usize, 0usize, 150_000u64), (1, 1, 50_000)].into_iter()).unwrap();
 
-        let cache_capacity = 8;
+        // capacity 2 < n_experts (3): pinning is only meaningful when the cache CAN'T hold every
+        // expert. At full residency (capacity >= n_experts) the cache never evicts, so pinning is
+        // correctly skipped -- covered by
+        // `expert_cache::tests::full_residency_cache_never_pins_even_with_marked_candidates`.
+        let cache_capacity = 2;
         let mut caches = ExpertCaches::new(&model, cache_capacity);
         let stats = caches.warm_start(&dir.0, cache_capacity);
 
         assert_eq!(stats.hist, 200_000);
         assert!((stats.confidence - 1.0).abs() < 1e-6);
-        // budget = floor(8 * 0.5 * 1.0) = 4, but only 2 nonzero entries exist for layer 1 -> 2 candidates.
-        assert_eq!(stats.pin_candidates, 2);
+        // budget = floor(2 * 0.5 * 1.0) = 1 -> only the highest-usage expert (eid 0, 150k) is a candidate.
+        assert_eq!(stats.pin_candidates, 1);
 
         // lazy: marking candidates must not load anything.
         assert!(!caches.0[1].as_ref().unwrap().is_pinned(0));
         assert_eq!(caches.0[1].as_ref().unwrap().pinned_len(), 0);
 
-        // the first real load of each marked candidate promotes it.
+        // the first real load of the marked candidate promotes it; the non-candidate does not.
         caches.0[1].as_mut().unwrap().get_or_load(&shards, &model.cfg, 1, 0, model.ebits).unwrap();
         caches.0[1].as_mut().unwrap().get_or_load(&shards, &model.cfg, 1, 1, model.ebits).unwrap();
         let layer1 = caches.0[1].as_ref().unwrap();
         assert!(layer1.is_pinned(0));
-        assert!(layer1.is_pinned(1));
-        assert_eq!(layer1.pinned_len(), 2);
+        assert!(!layer1.is_pinned(1), "eid 1 was below the pin budget, so it stays in ordinary LRU slots");
+        assert_eq!(layer1.pinned_len(), 1);
 
         let layer2 = caches.0[2].as_ref().unwrap();
         assert_eq!(layer2.pinned_len(), 0, "a layer with no usage history must not get anything pinned");
diff --git a/src/glm52/model.rs b/src/glm52/model.rs
index 7332f4b..45ca91c 100644
--- a/src/glm52/model.rs
+++ b/src/glm52/model.rs
@@ -220,6 +220,8 @@ impl Model {
         let fused_gate_up = detect_fused_gate_up(&cfg, &shards);
 
         let mut layers = Vec::with_capacity(cfg.n_layers as usize);
+        let t_layers = std::time::Instant::now();
+        let stderr_tty = std::io::IsTerminal::is_terminal(&std::io::stderr());
         for i in 0..cfg.n_layers as usize {
             let p = |s: &str| format!("model.layers.{i}.{s}");
 
@@ -268,6 +270,17 @@ impl Model {
             };
 
             layers.push(Layer { in_ln, post_ln, attn, dsa, ffn });
+            // Per-layer progress (see kimi_k3::model's version): rewriting line in a TTY,
+            // newline-delimited when redirected so a log/monitor sees each layer.
+            if stderr_tty {
+                eprint!("\r  loaded layer {}/{} ({:.0}s)", i + 1, cfg.n_layers, t_layers.elapsed().as_secs_f32());
+                let _ = std::io::Write::flush(&mut std::io::stderr());
+            } else {
+                eprintln!("  loaded layer {}/{} ({:.0}s)", i + 1, cfg.n_layers, t_layers.elapsed().as_secs_f32());
+            }
+        }
+        if stderr_tty {
+            eprintln!();
         }
 
         Ok(Model { cfg, embed, lm_head, final_norm, layers, has_dsa, ebits, route_cfg: RouteConfig::default(), fused_gate_up })
diff --git a/src/glm52/moe.rs b/src/glm52/moe.rs
index c512b51..a00e27d 100644
--- a/src/glm52/moe.rs
+++ b/src/glm52/moe.rs
@@ -55,7 +55,11 @@ impl Activation {
     /// The single-element gating formula — `apply_single_expert`'s `Fused` gate_up branch reads
     /// gate/up from two positions within the SAME contiguous row buffer (not two separate
     /// slices), so it calls this directly per-element rather than through `apply` below.
-    fn combine(self, g: f32, u: f32) -> f32 {
+    ///
+    /// `pub(crate)` for the same reason as `unique_experts`/`apply_single_expert`/`expert_rows`
+    /// below: `kimi_k3::moe`'s row-blocked dispatch (Phase 5 v2) applies the activation to its own
+    /// transposed row blocks, and must do it with THIS formula rather than a second copy of it.
+    pub(crate) fn combine(self, g: f32, u: f32) -> f32 {
         match self {
             Activation::Silu => siluf(g) * u,
             Activation::Situ { beta, linear_beta } => {
@@ -71,8 +75,9 @@ impl Activation {
 
     /// `gate[i] = combine(gate[i], up[i])`, applied in place across two separate contiguous
     /// buffers — matches every existing call site's `g[k] = siluf(g[k]) * u[k]` convention
-    /// (result written into the gate buffer).
-    fn apply(self, gate: &mut [f32], up: &[f32]) {
+    /// (result written into the gate buffer). Element-wise with no cross-element dependency,
+    /// which is what lets `kimi_k3::moe` apply it to an arbitrary row block (see `combine`).
+    pub(crate) fn apply(self, gate: &mut [f32], up: &[f32]) {
         for (g, &u) in gate.iter_mut().zip(up) {
             *g = self.combine(*g, u);
         }
@@ -456,15 +461,25 @@ pub fn moe(
 /// calls this directly against its own down-projected `x`/narrower `d`/latent-width `out` buffer,
 /// getting the exact same tested `Separate`-vs-`Fused` gate_up matmul logic for free instead of
 /// duplicating it.
-#[allow(clippy::too_many_arguments)]
-pub(crate) fn apply_single_expert(slot: &ExpertSlot, routing: &Routing, x: &[f32], d: usize, i: usize, activation: Activation, out: &mut [f32]) {
-    let eid = slot.eid;
-    let rows: Vec<(usize, f32)> = routing
+/// Every `(sequence position, routing weight)` pair routed to `eid`, in ascending position order
+/// — the row list `apply_single_expert` gathers its activations by and scatters its results back
+/// through. At most one entry per position (a token picks any expert at most once).
+///
+/// `pub(crate)` for the same reason as `apply_single_expert` itself: `kimi_k3::moe`'s row-blocked
+/// dispatch (Phase 5 v2) needs the identical row list to reproduce that function's scatter order
+/// bit-for-bit, and re-deriving it there would be a second copy free to drift.
+pub(crate) fn expert_rows(routing: &Routing, eid: usize) -> Vec<(usize, f32)> {
+    routing
         .choices
         .iter()
         .enumerate()
         .filter_map(|(si, picks)| picks.iter().find(|&&(e, _)| e == eid).map(|&(_, wt)| (si, wt)))
-        .collect();
+        .collect()
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(crate) fn apply_single_expert(slot: &ExpertSlot, routing: &Routing, x: &[f32], d: usize, i: usize, activation: Activation, out: &mut [f32]) {
+    let rows = expert_rows(routing, slot.eid);
     if rows.is_empty() {
         return;
     }
diff --git a/src/kernels.rs b/src/kernels.rs
index 228ce17..e143e3f 100644
--- a/src/kernels.rs
+++ b/src/kernels.rs
@@ -12,7 +12,11 @@
 //! *less* rounding error than the AVX2/scalar order (colibrì measured 2-6x lower max relative
 //! error vs the scalar oracle), but for that same reason NOT bit-identical to them — parity
 //! tests for `matmul_i4`'s tiers are within-tolerance, not bit-exact, same as AVX2-vs-scalar
-//! already was for this function. `dot_i8i8`/`dot_i4i8` (the integer IDOT path) get
+//! already was for this function. `matmul_mxfp4` (K3's routed-expert MXFP4 path, added for the
+//! target-box work — `K3_OPTIMIZE_BRIEF.md`) follows the same split: a bit-exact scalar tier
+//! (`matmul_mxfp4_scalar`) and an AVX-512F/BW tier (`matmul_mxfp4_avx512` — a `permutexvar` E2M1
+//! decode with per-block E8M0 scale-fold and dual-chain FMA) that is within-tolerance, not
+//! bit-identical, for the same per-row reassociation reason. `dot_i8i8`/`dot_i4i8` (the integer IDOT path) get
 //! scalar/AVX2/AVX-512-VNNI: pure integer accumulation, so unlike either float path there's no
 //! reassociation to worry about — every tier must agree bit-for-bit there, which is exactly
 //! what this module's parity tests check.
@@ -35,10 +39,16 @@ use std::arch::x86_64::*;
 /// no cross-row dependency. The catch is `y`'s own layout (`y[si*O+oi]`, row-major by sequence
 /// position): a fixed `oi` touches `S` elements strided by `O`, which safe Rust can't split into
 /// disjoint mutable chunks across threads. `yt`'s `[O,S]` layout (row-major by output index)
-/// makes each `oi`'s slice contiguous instead — exactly what `par_chunks_mut(s)` needs — at the
+/// makes each `oi`'s slice contiguous instead — exactly what `par_chunks_mut` needs — at the
 /// cost of one sequential transpose back into `y` afterward. That transpose is O(S*O); the
 /// matmul itself is O(S*O*I), so for any realistic `I` (hidden/intermediate dims in the
 /// thousands) the transpose is noise.
+///
+/// Since Phase 5 v2 (`K3_OPTIMIZE_BRIEF.md`/`NUMA_AMX_BRIEF.md` N3) the fan-out is COARSENED:
+/// rows go to the pool in [`matmul_chunk_rows`]-sized blocks via [`par_rows`], not one task per
+/// row (see those docs — one-row tasks were measured as a scheduling floor, not a parallelism
+/// win). Callers that already hold coarser independent work (K3's expert dispatch) skip the
+/// internal fan-out entirely via the serial row-range entry point [`matmul_qt_rows`].
 fn transpose_so(y: &mut [f32], yt: &[f32], s: usize, o: usize) {
     for oi in 0..o {
         for si in 0..s {
@@ -64,15 +74,56 @@ thread_local! {
 /// generation (same shapes every layer/token) settles into zero further allocations after an
 /// initial warm-up. Callers get a raw `&mut [f32]`, not the `RefCell` itself, and always start
 /// from zero-filled — every existing caller already expected a fresh `vec![0f32; len]`.
+///
+/// Reentrancy: a `matmul_*` holds this thread-local borrow for the whole duration of its inner
+/// `par_chunks_mut` region, during which rayon can work-steal ANOTHER `matmul_*` onto this same
+/// thread (K3's Phase-5 across-expert parallelism nests an outer per-expert `par_iter` around the
+/// matmuls' inner fan-out — the single-level-parallelism assumption the original `borrow_mut()`
+/// encoded no longer holds). The inner (reentrant) call falls back to a one-shot heap buffer
+/// instead of panicking on the already-held borrow. The common non-nested path (GLM/Kimi, and
+/// K3's own non-matmul callers) still hits the zero-alloc thread-local fast path unchanged.
+/// Output rows per rayon task for a matmul over `o` rows. `par_chunks_mut(s)` — one task per
+/// output row — is the natural expression but a pathological schedule at batch-1 decode: it hands
+/// rayon thousands of tasks worth one dot product each, and the pool spends more time splitting,
+/// waking and joining than computing. Coarsening to a few tasks per pool thread keeps every
+/// thread fed and work-stealing able to even out a ragged tail, while collapsing the split tree by
+/// orders of magnitude. Bit-identity is unaffected: task granularity changes which thread computes
+/// a row, never the row's own accumulation order.
+///
+/// Measured on the 384-thread target box (`PERFORMANCE.md`, Phase 5 v2): with one task per row,
+/// K3 decode got *slower* the more threads it was given — 0.094 s/token at 48 threads against
+/// 1.02 s/token at 384, i.e. the pool was a liability well before it was a resource. The laptop
+/// this file was originally tuned on has 12 cores and never surfaced it.
+fn matmul_chunk_rows(o: usize) -> usize {
+    o.div_ceil((rayon::current_num_threads() * 4).max(1)).max(1)
+}
+
+/// Fans `row(oi, out)` — one output row's `s` values into its own `[S]` slice of the transposed
+/// `yt` — across the pool in [`matmul_chunk_rows`]-sized blocks. Every `matmul_*` below shares
+/// this rather than calling `par_chunks_mut` itself, so the granularity decision lives in exactly
+/// one place.
+fn par_rows(yt: &mut [f32], s: usize, o: usize, row: impl Fn(usize, &mut [f32]) + Sync + Send) {
+    let cr = matmul_chunk_rows(o);
+    yt.par_chunks_mut(s * cr).enumerate().for_each(|(ci, block)| {
+        for (k, out) in block.chunks_mut(s).enumerate() {
+            row(ci * cr + k, out);
+        }
+    });
+}
+
 fn with_yt_scratch<R>(len: usize, f: impl FnOnce(&mut [f32]) -> R) -> R {
     YT_SCRATCH.with(|cell| {
-        let mut buf = cell.borrow_mut();
-        if buf.len() < len {
-            buf.resize(len, 0.0);
+        if let Ok(mut buf) = cell.try_borrow_mut() {
+            if buf.len() < len {
+                buf.resize(len, 0.0);
+            } else {
+                buf[..len].fill(0.0);
+            }
+            f(&mut buf[..len])
         } else {
-            buf[..len].fill(0.0);
+            let mut buf = vec![0.0f32; len];
+            f(&mut buf)
         }
-        f(&mut buf[..len])
     })
 }
 
@@ -97,14 +148,22 @@ fn has_avx512_i4() -> bool {
     false
 }
 
+/// One output element of [`matmul`] — factored out of the inner loop so [`matmul_qt_rows`] can
+/// compute a row block through the SAME code (and therefore the same accumulation order) rather
+/// than a re-derived copy of it. Same reason the other `row_dot_*` helpers below exist; see
+/// [`matmul_qt_rows`]'s doc for the bit-identity contract they collectively make possible.
+#[inline]
+fn row_dot_f32(wr: &[f32], xs: &[f32]) -> f32 {
+    xs.iter().zip(wr).map(|(a, b)| a * b).sum()
+}
+
 /// y[S,O] = x[S,I] @ W^T, W[O,I] f32.
 pub fn matmul(y: &mut [f32], x: &[f32], w: &[f32], s: usize, i: usize, o: usize) {
     with_yt_scratch(o * s, |yt| {
-        yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+        par_rows(yt, s, o, |oi, out| {
             let wr = &w[oi * i..(oi + 1) * i];
-            for (si, slot) in row.iter_mut().enumerate() {
-                let xs = &x[si * i..(si + 1) * i];
-                *slot = xs.iter().zip(wr).map(|(a, b)| a * b).sum();
+            for (si, slot) in out.iter_mut().enumerate() {
+                *slot = row_dot_f32(wr, &x[si * i..(si + 1) * i]);
             }
         });
         transpose_so(y, yt, s, o);
@@ -123,10 +182,10 @@ pub fn matmul_q(y: &mut [f32], x: &[f32], q: &[i8], scale: &[f32], s: usize, i:
 
 fn matmul_q_scalar(y: &mut [f32], x: &[f32], q: &[i8], scale: &[f32], s: usize, i: usize, o: usize) {
     with_yt_scratch(o * s, |yt| {
-        yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+        par_rows(yt, s, o, |oi, out| {
             let w = &q[oi * i..(oi + 1) * i];
             let sc = scale[oi];
-            for (si, slot) in row.iter_mut().enumerate() {
+            for (si, slot) in out.iter_mut().enumerate() {
                 let xs = &x[si * i..(si + 1) * i];
                 let a: f32 = xs.iter().zip(w).map(|(&xv, &wv)| xv * wv as f32).sum();
                 *slot = a * sc;
@@ -152,30 +211,40 @@ pub fn matmul_i4(y: &mut [f32], x: &[f32], q4: &[u8], scale: &[f32], s: usize, i
     matmul_i4_scalar(y, x, q4, scale, s, i, o)
 }
 
+/// The unscaled dot behind [`matmul_i4_scalar`]'s inner loop. NOT interchangeable with
+/// [`dot_i4_f32_scalar`] despite computing the same mathematical quantity: this one adds each
+/// packed byte's TWO products together before folding them into the running sum
+/// (`a += xs[k]*lo + xs[k+1]*hi`), whereas `dot_i4_f32_scalar` — the reference
+/// `qt_addrow`/`qt_matvec_rows` are written against — accumulates strictly one element at a time.
+/// Different association, different bits. Keep them separate.
+#[inline]
+fn row_dot_i4_f32_pairs(w: &[u8], xs: &[f32], i: usize) -> f32 {
+    let mut a = 0f32;
+    let mut ii = 0;
+    while ii + 1 < i {
+        let byte = w[ii >> 1];
+        let lo = (byte & 0xF) as i32 - 8;
+        let hi = (byte >> 4) as i32 - 8;
+        a += xs[ii] * lo as f32 + xs[ii + 1] * hi as f32;
+        ii += 2;
+    }
+    if ii < i {
+        let byte = w[ii >> 1];
+        let lo = (byte & 0xF) as i32 - 8;
+        a += xs[ii] * lo as f32;
+    }
+    a
+}
+
 /// `pub` so `benches/kernels.rs` can compare tiers directly — same reason `dot_i8i8_scalar` is.
 pub fn matmul_i4_scalar(y: &mut [f32], x: &[f32], q4: &[u8], scale: &[f32], s: usize, i: usize, o: usize) {
     let rb = i.div_ceil(2);
     with_yt_scratch(o * s, |yt| {
-        yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+        par_rows(yt, s, o, |oi, out| {
             let w = &q4[oi * rb..(oi + 1) * rb];
             let sc = scale[oi];
-            for (si, slot) in row.iter_mut().enumerate() {
-                let xs = &x[si * i..(si + 1) * i];
-                let mut a = 0f32;
-                let mut ii = 0;
-                while ii + 1 < i {
-                    let byte = w[ii >> 1];
-                    let lo = (byte & 0xF) as i32 - 8;
-                    let hi = (byte >> 4) as i32 - 8;
-                    a += xs[ii] * lo as f32 + xs[ii + 1] * hi as f32;
-                    ii += 2;
-                }
-                if ii < i {
-                    let byte = w[ii >> 1];
-                    let lo = (byte & 0xF) as i32 - 8;
-                    a += xs[ii] * lo as f32;
-                }
-                *slot = a * sc;
+            for (si, slot) in out.iter_mut().enumerate() {
+                *slot = row_dot_i4_f32_pairs(w, &x[si * i..(si + 1) * i], i) * sc;
             }
         });
         transpose_so(y, yt, s, o);
@@ -191,22 +260,28 @@ pub fn matmul_i2(y: &mut [f32], x: &[f32], q2: &[u8], scale: &[f32], s: usize, i
     matmul_i2_scalar(y, x, q2, scale, s, i, o)
 }
 
+/// The unscaled dot behind [`matmul_i2_scalar`]'s inner loop — see [`row_dot_f32`] for why the
+/// per-row bodies are factored out at all.
+#[inline]
+fn row_dot_i2_f32_scalar(w: &[u8], xs: &[f32], i: usize) -> f32 {
+    let mut a = 0f32;
+    for ii in 0..i {
+        let byte = w[ii >> 2];
+        let sh = (ii & 3) * 2;
+        let v = ((byte >> sh) & 3) as i32 - 2;
+        a += xs[ii] * v as f32;
+    }
+    a
+}
+
 fn matmul_i2_scalar(y: &mut [f32], x: &[f32], q2: &[u8], scale: &[f32], s: usize, i: usize, o: usize) {
     let rb = i.div_ceil(4);
     with_yt_scratch(o * s, |yt| {
-        yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+        par_rows(yt, s, o, |oi, out| {
             let w = &q2[oi * rb..(oi + 1) * rb];
             let sc = scale[oi];
-            for (si, slot) in row.iter_mut().enumerate() {
-                let xs = &x[si * i..(si + 1) * i];
-                let mut a = 0f32;
-                for ii in 0..i {
-                    let byte = w[ii >> 2];
-                    let sh = (ii & 3) * 2;
-                    let v = ((byte >> sh) & 3) as i32 - 2;
-                    a += xs[ii] * v as f32;
-                }
-                *slot = a * sc;
+            for (si, slot) in out.iter_mut().enumerate() {
+                *slot = row_dot_i2_f32_scalar(w, &x[si * i..(si + 1) * i], i) * sc;
             }
         });
         transpose_so(y, yt, s, o);
@@ -215,32 +290,68 @@ fn matmul_i2_scalar(y: &mut [f32], x: &[f32], q2: &[u8], scale: &[f32], s: usize
 
 /// y[S,O] = x[S,I] @ W^T, W in OCP-MX FP4 (`QTKind::MxFp4` — see `quant.rs`'s doc): 4-bit E2M1
 /// elements, one E8M0 scale per 32-element block along the row (not per-row, unlike every other
-/// format here). Scalar only for now — correctness first, matching this project's own
-/// established order (see the module doc's history: SIMD tiers arrived after a proven scalar
-/// baseline for every other format too), and there's no real MXFP4 checkpoint to benchmark
-/// against yet regardless (see `rabbit-plan.md`'s Phase 2 entry).
+/// format here). Dispatches AVX-512F/BW (`matmul_mxfp4_avx512`) > scalar, same ladder as
+/// `matmul_i4` — see the module doc for why the AVX-512 tier is within-tolerance, not bit-exact.
 pub fn matmul_mxfp4(y: &mut [f32], x: &[f32], data: &[u8], block_scale: &[u8], s: usize, i: usize, o: usize) {
+    #[cfg(target_arch = "x86_64")]
+    if is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512bw") {
+        return unsafe { matmul_mxfp4_avx512(y, x, data, block_scale, s, i, o) };
+    }
+    matmul_mxfp4_scalar(y, x, data, block_scale, s, i, o)
+}
+
+/// Scalar tier of [`matmul_mxfp4`]. `pub` so `benches/kernels.rs` can compare tiers directly, same
+/// as `matmul_i4_scalar`. The inner loop iterates the row's **32-element scale blocks**:
+/// `e8m0_decode` is evaluated once per block (not once per element as the pre-Phase-2 loop did — it
+/// recomputed `2^(byte-127)` for all 32 elements sharing one scale byte), and each packed byte's
+/// two nibbles are decoded together, removing the per-element `k & 1` parity branch. The
+/// per-element arithmetic `xs[k] * e2m1 * scale` and the accumulation order (ascending `k`) are
+/// byte-for-byte the pre-Phase-2 element-at-a-time loop's, so this is **bit-identical** to it —
+/// pinned by `matmul_mxfp4_matches_the_pre_block_reference`. Deliberately NOT per-block partial
+/// sums (that reassociates); reassociation lives in the AVX-512 tier.
+pub fn matmul_mxfp4_scalar(y: &mut [f32], x: &[f32], data: &[u8], block_scale: &[u8], s: usize, i: usize, o: usize) {
     let rb = i.div_ceil(2);
     let bpr = i.div_ceil(32);
     with_yt_scratch(o * s, |yt| {
-        yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+        par_rows(yt, s, o, |oi, out| {
             let w = &data[oi * rb..(oi + 1) * rb];
             let bs = &block_scale[oi * bpr..(oi + 1) * bpr];
-            for (si, slot) in row.iter_mut().enumerate() {
-                let xs = &x[si * i..(si + 1) * i];
-                let mut a = 0f32;
-                for k in 0..i {
-                    let byte = w[k >> 1];
-                    let nibble = if k & 1 == 0 { byte & 0xF } else { byte >> 4 };
-                    a += xs[k] * e2m1_decode(nibble) * e8m0_decode(bs[k / 32]);
-                }
-                *slot = a;
+            for (si, slot) in out.iter_mut().enumerate() {
+                *slot = row_dot_mxfp4_f32_scalar(w, bs, &x[si * i..(si + 1) * i], i);
             }
         });
         transpose_so(y, yt, s, o);
     });
 }
 
+/// The dot behind [`matmul_mxfp4_scalar`]'s inner loop (MXFP4 carries its scale per 32-element
+/// block, so unlike the other formats there is nothing left to factor out afterward) — see
+/// [`row_dot_f32`] for why the per-row bodies are factored out at all.
+#[inline]
+fn row_dot_mxfp4_f32_scalar(w: &[u8], bs: &[u8], xs: &[f32], i: usize) -> f32 {
+    let mut a = 0f32;
+    let mut k = 0;
+    for &scale_byte in bs {
+        let scale = e8m0_decode(scale_byte);
+        let block_end = (k + 32).min(i);
+        // Blocks start on even `k` (multiples of 32), so within a block a packed byte
+        // holds element k (low nibble) then k+1 (high nibble) — same mapping the old
+        // `if k & 1 == 0` branch computed, just unrolled two at a time.
+        while k + 1 < block_end {
+            let byte = w[k >> 1];
+            a += xs[k] * e2m1_decode(byte & 0xF) * scale;
+            a += xs[k + 1] * e2m1_decode(byte >> 4) * scale;
+            k += 2;
+        }
+        if k < block_end {
+            let byte = w[k >> 1];
+            a += xs[k] * e2m1_decode(byte & 0xF) * scale;
+            k += 1;
+        }
+    }
+    a
+}
+
 /// y[S,O] = x[S,I] @ W^T, W int4-packed[O,ceil(I/2)] (2 values/byte) with a GROUPED scale
 /// (`QTKind::I4Grouped` — one `f32` per `group_size`-element run along each row, not one per
 /// whole row). Scalar only for now, same "correctness first" precedent as `matmul_mxfp4` (no
@@ -252,24 +363,30 @@ pub fn matmul_i4_grouped(y: &mut [f32], x: &[f32], q4: &[u8], scale: &[f32], gro
     let rb = i.div_ceil(2);
     let ngroups = i.div_ceil(group_size);
     with_yt_scratch(o * s, |yt| {
-        yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+        par_rows(yt, s, o, |oi, out| {
             let w = &q4[oi * rb..(oi + 1) * rb];
             let sc = &scale[oi * ngroups..(oi + 1) * ngroups];
-            for (si, slot) in row.iter_mut().enumerate() {
-                let xs = &x[si * i..(si + 1) * i];
-                let mut a = 0f32;
-                for k in 0..i {
-                    let byte = w[k >> 1];
-                    let nibble = if k & 1 == 0 { byte & 0xF } else { byte >> 4 };
-                    a += xs[k] * (nibble as i32 - 8) as f32 * sc[k / group_size];
-                }
-                *slot = a;
+            for (si, slot) in out.iter_mut().enumerate() {
+                *slot = row_dot_i4_grouped_scalar(w, sc, &x[si * i..(si + 1) * i], i, group_size);
             }
         });
         transpose_so(y, yt, s, o);
     });
 }
 
+/// The dot behind [`matmul_i4_grouped`]'s inner loop (the group scale varies within the row, so
+/// like MXFP4 it folds in per element) — see [`row_dot_f32`] for why these are factored out.
+#[inline]
+fn row_dot_i4_grouped_scalar(w: &[u8], sc: &[f32], xs: &[f32], i: usize, group_size: usize) -> f32 {
+    let mut a = 0f32;
+    for k in 0..i {
+        let byte = w[k >> 1];
+        let nibble = if k & 1 == 0 { byte & 0xF } else { byte >> 4 };
+        a += xs[k] * (nibble as i32 - 8) as f32 * sc[k / group_size];
+    }
+    a
+}
+
 /// Quantizes one activation row to int8 (absmax/127, Q8_0-style) for the IDOT kernels.
 /// Returns the row's scale; `q.len() == x.len()` required. Scalar only — the C never
 /// vectorized this either (it's a single absmax-then-round pass, not the hot inner loop).
@@ -476,8 +593,12 @@ mod simd {
     }
 
     /// int4-packed weight row · f32 activation row -> f32. `n` = logical (unpacked) length.
+    ///
+    /// # Safety
+    /// Caller must have verified `is_x86_feature_detected!("avx2")`. `w` must have length >=
+    /// `ceil(n/2)`, `xs` length >= `n`.
     #[target_feature(enable = "avx2")]
-    unsafe fn dot_i4_f32_avx2(w: &[u8], xs: &[f32], n: usize) -> f32 {
+    pub(super) unsafe fn dot_i4_f32_avx2(w: &[u8], xs: &[f32], n: usize) -> f32 {
         unsafe {
             let m4 = _mm_set1_epi8(0x0F);
             let b8 = _mm256_set1_epi32(8);
@@ -614,8 +735,12 @@ mod simd {
     }
 
     /// int2-packed weight row · f32 activation row -> f32. `n` = logical (unpacked) length.
+    ///
+    /// # Safety
+    /// Caller must have verified `is_x86_feature_detected!("avx2")`. `w` must have length >=
+    /// `ceil(n/4)`, `xs` length >= `n`.
     #[target_feature(enable = "avx2")]
-    unsafe fn dot_i2_f32_avx2(w: &[u8], xs: &[f32], n: usize) -> f32 {
+    pub(super) unsafe fn dot_i2_f32_avx2(w: &[u8], xs: &[f32], n: usize) -> f32 {
         unsafe {
             let m2 = _mm_set1_epi8(0x03);
             let b2 = _mm256_set1_epi32(2);
@@ -650,10 +775,10 @@ mod simd {
     #[target_feature(enable = "avx2")]
     pub(super) unsafe fn matmul_q_avx2(y: &mut [f32], x: &[f32], q: &[i8], scale: &[f32], s: usize, i: usize, o: usize) {
         with_yt_scratch(o * s, |yt| {
-            yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+            par_rows(yt, s, o, |oi, out| {
                 let w = &q[oi * i..(oi + 1) * i];
                 let sc = scale[oi];
-                for (si, slot) in row.iter_mut().enumerate() {
+                for (si, slot) in out.iter_mut().enumerate() {
                     let xs = &x[si * i..(si + 1) * i];
                     // Safety: caller of `matmul_q_avx2` already verified AVX2 at the dispatch
                     // site (`matmul_q`); that's a whole-machine capability, not per-thread
@@ -674,10 +799,10 @@ mod simd {
     pub unsafe fn matmul_i4_avx2(y: &mut [f32], x: &[f32], q4: &[u8], scale: &[f32], s: usize, i: usize, o: usize) {
         let rb = i.div_ceil(2);
         with_yt_scratch(o * s, |yt| {
-            yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+            par_rows(yt, s, o, |oi, out| {
                 let w = &q4[oi * rb..(oi + 1) * rb];
                 let sc = scale[oi];
-                for (si, slot) in row.iter_mut().enumerate() {
+                for (si, slot) in out.iter_mut().enumerate() {
                     let xs = &x[si * i..(si + 1) * i];
                     *slot = unsafe { dot_i4_f32_avx2(w, xs, i) } * sc;
                 }
@@ -696,10 +821,10 @@ mod simd {
     pub unsafe fn matmul_i4_avx512(y: &mut [f32], x: &[f32], q4: &[u8], scale: &[f32], s: usize, i: usize, o: usize) {
         let rb = i.div_ceil(2);
         with_yt_scratch(o * s, |yt| {
-            yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+            par_rows(yt, s, o, |oi, out| {
                 let w = &q4[oi * rb..(oi + 1) * rb];
                 let sc = scale[oi];
-                for (si, slot) in row.iter_mut().enumerate() {
+                for (si, slot) in out.iter_mut().enumerate() {
                     let xs = &x[si * i..(si + 1) * i];
                     *slot = unsafe { dot_i4_f32_avx512(w, xs, i) } * sc;
                 }
@@ -708,14 +833,98 @@ mod simd {
         });
     }
 
+    /// One row of MXFP4 weights · f32 activation row -> f32, AVX-512F/BW. The format is nearly
+    /// designed for this: one 32-element E8M0 scale block = 16 packed bytes = two 16-lane f32
+    /// vectors, and E2M1 has exactly 16 code points, so the whole sign×magnitude decode table fits
+    /// one `zmm` addressed by `_mm512_permutexvar_ps` (which uses each index's low 4 bits — exactly
+    /// the nibble). Per full block: unpack the 16 bytes' low/high nibbles into two i32 index
+    /// vectors (the same `unpacklo/unpackhi` interleave `dot_i4_f32_avx512` uses so element order
+    /// is preserved), gather E2M1 values via `permutexvar`, fold the block's scalar-decoded E8M0
+    /// scale in, and FMA into two independent accumulator chains reduced by one `_mm512_reduce_add_ps`
+    /// tree-sum at the end. That scale-fold + two-chain reduction reassociates relative to the
+    /// scalar `x*e2m1*scale` running sum, so this is **within-tolerance, NOT bit-identical** — the
+    /// same accuracy tradeoff `dot_i4_f32_avx512` documents (colibrì measured such reordering as
+    /// *lower* max error than scalar, but different bits). `n` = logical row length; the `cols % 32`
+    /// tail (a final partial block) is done scalar, matching the scalar decode exactly.
+    ///
+    /// `pub` so `benches/kernels.rs` can measure it directly, same as the other tier kernels.
+    ///
+    /// # Safety
+    /// Caller must have verified `is_x86_feature_detected!("avx512f")` and `"avx512bw"`. `data`
+    /// must have length >= `ceil(n/2)`, `bs` length >= `ceil(n/32)`, `xs` length >= `n`.
+    #[target_feature(enable = "avx512f,avx512bw")]
+    pub unsafe fn dot_mxfp4_f32_avx512(data: &[u8], bs: &[u8], xs: &[f32], n: usize) -> f32 {
+        unsafe {
+            // table[nibble] = e2m1_decode(nibble): bit 3 = sign, low 3 bits index the magnitudes.
+            let table = _mm512_setr_ps(
+                0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0,
+            );
+            let m4 = _mm_set1_epi8(0x0F);
+            let mut acc0 = _mm512_setzero_ps();
+            let mut acc1 = _mm512_setzero_ps();
+            let mut k = 0;
+            let mut b = 0;
+            while k + 32 <= n {
+                let scale = _mm512_set1_ps(e8m0_decode(bs[b]));
+                let by = _mm_loadu_si128(data.as_ptr().add(k >> 1) as *const __m128i);
+                let lo = _mm_and_si128(by, m4);
+                let hi = _mm_and_si128(_mm_srli_epi16::<4>(by), m4);
+                let n0 = _mm_unpacklo_epi8(lo, hi); // nibbles for elements [k, k+16)
+                let n1 = _mm_unpackhi_epi8(lo, hi); // nibbles for elements [k+16, k+32)
+                let w0 = _mm512_mul_ps(_mm512_permutexvar_ps(_mm512_cvtepu8_epi32(n0), table), scale);
+                let w1 = _mm512_mul_ps(_mm512_permutexvar_ps(_mm512_cvtepu8_epi32(n1), table), scale);
+                acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(xs.as_ptr().add(k)), w0, acc0);
+                acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(xs.as_ptr().add(k + 16)), w1, acc1);
+                k += 32;
+                b += 1;
+            }
+            let mut a = _mm512_reduce_add_ps(_mm512_add_ps(acc0, acc1));
+            if k < n {
+                // Final partial block (< 32 elements): scalar, same decode/order as the scalar tier.
+                let scale = e8m0_decode(bs[b]);
+                while k < n {
+                    let byte = data[k >> 1];
+                    let nibble = if k & 1 == 0 { byte & 0xF } else { byte >> 4 };
+                    a += xs[k] * e2m1_decode(nibble) * scale;
+                    k += 1;
+                }
+            }
+            a
+        }
+    }
+
+    /// AVX-512F/BW tier of `matmul_mxfp4` — parallelizes over output rows exactly like the scalar
+    /// tier and `matmul_i4_avx512`, calling `dot_mxfp4_f32_avx512` per (row, seq) dot. Within-
+    /// tolerance vs scalar (per-row reassociation), see `dot_mxfp4_f32_avx512`.
+    ///
+    /// # Safety
+    /// Caller must have verified `is_x86_feature_detected!("avx512f")` and `"avx512bw"`. Slice
+    /// lengths as for `matmul_mxfp4`.
+    #[target_feature(enable = "avx512f,avx512bw")]
+    pub unsafe fn matmul_mxfp4_avx512(y: &mut [f32], x: &[f32], data: &[u8], block_scale: &[u8], s: usize, i: usize, o: usize) {
+        let rb = i.div_ceil(2);
+        let bpr = i.div_ceil(32);
+        with_yt_scratch(o * s, |yt| {
+            par_rows(yt, s, o, |oi, out| {
+                let w = &data[oi * rb..(oi + 1) * rb];
+                let bs = &block_scale[oi * bpr..(oi + 1) * bpr];
+                for (si, slot) in out.iter_mut().enumerate() {
+                    let xs = &x[si * i..(si + 1) * i];
+                    *slot = unsafe { dot_mxfp4_f32_avx512(w, bs, xs, i) };
+                }
+            });
+            transpose_so(y, yt, s, o);
+        });
+    }
+
     #[target_feature(enable = "avx2")]
     pub(super) unsafe fn matmul_i2_avx2(y: &mut [f32], x: &[f32], q2: &[u8], scale: &[f32], s: usize, i: usize, o: usize) {
         let rb = i.div_ceil(4);
         with_yt_scratch(o * s, |yt| {
-            yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+            par_rows(yt, s, o, |oi, out| {
                 let w = &q2[oi * rb..(oi + 1) * rb];
                 let sc = scale[oi];
-                for (si, slot) in row.iter_mut().enumerate() {
+                for (si, slot) in out.iter_mut().enumerate() {
                     let xs = &x[si * i..(si + 1) * i];
                     *slot = unsafe { dot_i2_f32_avx2(w, xs, i) } * sc;
                 }
@@ -874,7 +1083,7 @@ use simd::*;
 #[cfg(target_arch = "x86_64")]
 pub use simd::{
     axpy_f32_avx2, axpy_i4_f32_avx512, dot_f32_avx2, dot_i4_f32_avx512, dot_i4i8_avx2, dot_i4i8_avx512vnni, dot_i8i8_avx2,
-    dot_i8i8_avx512vnni, matmul_i4_avx2, matmul_i4_avx512,
+    dot_i8i8_avx512vnni, dot_mxfp4_f32_avx512, matmul_i4_avx2, matmul_i4_avx512, matmul_mxfp4_avx512,
 };
 
 // mirrors matmul_q_idot/matmul_i4_idot's C signature 1:1; both are private, single-call-site
@@ -882,10 +1091,10 @@ pub use simd::{
 #[allow(clippy::too_many_arguments)]
 fn matmul_q_idot(y: &mut [f32], xq: &[i8], sx: &[f32], q: &[i8], scale: &[f32], s: usize, i: usize, o: usize) {
     with_yt_scratch(o * s, |yt| {
-        yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+        par_rows(yt, s, o, |oi, out| {
             let w = &q[oi * i..(oi + 1) * i];
             let sc = scale[oi];
-            for (si, slot) in row.iter_mut().enumerate() {
+            for (si, slot) in out.iter_mut().enumerate() {
                 let xrow = &xq[si * i..(si + 1) * i];
                 *slot = dot_i8i8(w, xrow, i) as f32 * sc * sx[si];
             }
@@ -898,10 +1107,10 @@ fn matmul_q_idot(y: &mut [f32], xq: &[i8], sx: &[f32], q: &[i8], scale: &[f32],
 fn matmul_i4_idot(y: &mut [f32], xq: &[i8], sx: &[f32], q4: &[u8], scale: &[f32], s: usize, i: usize, o: usize) {
     let rb = i.div_ceil(2);
     with_yt_scratch(o * s, |yt| {
-        yt.par_chunks_mut(s).enumerate().for_each(|(oi, row)| {
+        par_rows(yt, s, o, |oi, out| {
             let w = &q4[oi * rb..(oi + 1) * rb];
             let sc = scale[oi];
-            for (si, slot) in row.iter_mut().enumerate() {
+            for (si, slot) in out.iter_mut().enumerate() {
                 let xrow = &xq[si * i..(si + 1) * i];
                 *slot = dot_i4i8(w, xrow, i) as f32 * sc * sx[si];
             }
@@ -939,6 +1148,214 @@ pub fn matmul_qt(y: &mut [f32], x: &[f32], w: &QT, s: usize) {
     }
 }
 
+/// Activations prepared once, ahead of a row-blocked fan-out over one weight matrix.
+///
+/// [`matmul_qt`] parallelizes over output rows INTERNALLY (`par_chunks_mut`) and quantizes `x`
+/// once per call for its IDOT tiers. [`matmul_qt_rows`] inverts that: the caller owns the
+/// parallelism and calls it once per row block, so the quantization would otherwise be repeated
+/// per block. Preparing it here hoists that work back out to once per (`x`, weight) pair.
+///
+/// `x` is always retained, so an `acts` prepared against a different-kind weight is still
+/// *correct* (the IDOT arms fall back to quantizing inline) — just slower than intended.
+pub struct RowActs<'a> {
+    x: &'a [f32],
+    s: usize,
+    /// `Some` exactly when `matmul_qt` would take an int8-IDOT tier for the weight this was
+    /// prepared against: `QTKind::I8` always, `QTKind::I4` at `s >= I4_IDOT_MIN_S`.
+    q: Option<(Vec<i8>, Vec<f32>)>,
+}
+
+impl<'a> RowActs<'a> {
+    /// Prepares `x` (`[s, w.cols]`) for row-blocked matmuls against `w`.
+    pub fn prepare(x: &'a [f32], w: &QT, s: usize) -> RowActs<'a> {
+        let idot = match &w.kind {
+            QTKind::I8 { .. } => true,
+            QTKind::I4 { .. } => s >= I4_IDOT_MIN_S,
+            _ => false,
+        };
+        let q = idot.then(|| quantize_activations(x, s, w.cols));
+        RowActs { x, s, q }
+    }
+}
+
+/// `yt[j*s + si] = <W[r0+j, :], x[si, :]>` for `j` in `0..nrows` — ONE ROW BLOCK of what
+/// [`matmul_qt`] computes, evaluated serially on the calling thread and written in `matmul_qt`'s
+/// own internal `[O, S]` transposed layout (its `yt`), which is what makes a row block a
+/// contiguous, independently-writable slice.
+///
+/// The point is inverted parallelism: `matmul_qt` forks internally over output rows and joins,
+/// which at batch-1 decode divides thousands of one-element tasks over the pool and spends more
+/// time scheduling than computing (`PERFORMANCE.md`, Phase 3 and Phase 5 v2). A caller that
+/// already has coarser work to spread — K3's `latent_moe`, which has 16 experts × 3 matmuls of
+/// independent rows available at once — calls this instead and forks exactly once itself.
+///
+/// **Bit-identity contract:** tier selection mirrors `matmul_qt`'s dispatch exactly, and every
+/// per-row body is the same `#[inline] row_dot_*` helper (or the same `dot_*` SIMD kernel) the
+/// whole-matrix kernels call, in the same per-element order. So assembling every row block and
+/// transposing `[O,S]` → `[S,O]` reproduces `matmul_qt`'s output **bit for bit** — pinned across
+/// all six `QTKind`s by `matmul_qt_rows_reassembled_is_bit_identical_to_matmul_qt`. (That is a
+/// contract about the two *paths* agreeing; where the tier itself is within-tolerance rather than
+/// exact against scalar — AVX-512 int4/MXFP4 — both paths are equally within-tolerance, because
+/// they run the identical kernel.)
+///
+/// `acts` should come from [`RowActs::prepare`] against this same `w` and `x`; `yt.len()` must be
+/// `nrows * s`, and `r0 + nrows` must not exceed `w.rows`.
+pub fn matmul_qt_rows(yt: &mut [f32], acts: &RowActs, w: &QT, r0: usize, nrows: usize) {
+    let i = w.cols;
+    let s = acts.s;
+    let x = acts.x;
+    debug_assert_eq!(yt.len(), nrows * s, "matmul_qt_rows: yt must be [nrows, s]");
+    debug_assert!(r0 + nrows <= w.rows, "matmul_qt_rows: row block past the end of the weight");
+    match &w.kind {
+        QTKind::F32(data) => {
+            for (j, out) in yt.chunks_mut(s).enumerate() {
+                let row = r0 + j;
+                let wr = &data[row * i..(row + 1) * i];
+                for (si, slot) in out.iter_mut().enumerate() {
+                    *slot = row_dot_f32(wr, &x[si * i..(si + 1) * i]);
+                }
+            }
+        }
+        QTKind::I8 { data, scale } => {
+            let owned;
+            let (xq, sx) = match &acts.q {
+                Some((xq, sx)) => (xq, sx),
+                None => {
+                    owned = quantize_activations(x, s, i);
+                    (&owned.0, &owned.1)
+                }
+            };
+            for (j, out) in yt.chunks_mut(s).enumerate() {
+                let row = r0 + j;
+                let wr = &data[row * i..(row + 1) * i];
+                let sc = scale[row];
+                for (si, slot) in out.iter_mut().enumerate() {
+                    *slot = dot_i8i8(wr, &xq[si * i..(si + 1) * i], i) as f32 * sc * sx[si];
+                }
+            }
+        }
+        QTKind::I4 { data, scale } => {
+            let rb = i.div_ceil(2);
+            // Mirrors `matmul_qt`'s own `s >= I4_IDOT_MIN_S` split, then `matmul_i4`'s tier ladder.
+            if s >= I4_IDOT_MIN_S {
+                let owned;
+                let (xq, sx) = match &acts.q {
+                    Some((xq, sx)) => (xq, sx),
+                    None => {
+                        owned = quantize_activations(x, s, i);
+                        (&owned.0, &owned.1)
+                    }
+                };
+                for (j, out) in yt.chunks_mut(s).enumerate() {
+                    let row = r0 + j;
+                    let wr = &data[row * rb..(row + 1) * rb];
+                    let sc = scale[row];
+                    for (si, slot) in out.iter_mut().enumerate() {
+                        *slot = dot_i4i8(wr, &xq[si * i..(si + 1) * i], i) as f32 * sc * sx[si];
+                    }
+                }
+                return;
+            }
+            #[cfg(target_arch = "x86_64")]
+            {
+                if is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512bw") {
+                    for (j, out) in yt.chunks_mut(s).enumerate() {
+                        let row = r0 + j;
+                        let wr = &data[row * rb..(row + 1) * rb];
+                        let sc = scale[row];
+                        for (si, slot) in out.iter_mut().enumerate() {
+                            // Safety: AVX-512F/BW just verified; a whole-machine capability.
+                            *slot = unsafe { dot_i4_f32_avx512(wr, &x[si * i..(si + 1) * i], i) } * sc;
+                        }
+                    }
+                    return;
+                }
+                if has_avx2() {
+                    for (j, out) in yt.chunks_mut(s).enumerate() {
+                        let row = r0 + j;
+                        let wr = &data[row * rb..(row + 1) * rb];
+                        let sc = scale[row];
+                        for (si, slot) in out.iter_mut().enumerate() {
+                            // Safety: AVX2 just verified; a whole-machine capability.
+                            *slot = unsafe { dot_i4_f32_avx2(wr, &x[si * i..(si + 1) * i], i) } * sc;
+                        }
+                    }
+                    return;
+                }
+            }
+            for (j, out) in yt.chunks_mut(s).enumerate() {
+                let row = r0 + j;
+                let wr = &data[row * rb..(row + 1) * rb];
+                let sc = scale[row];
+                for (si, slot) in out.iter_mut().enumerate() {
+                    *slot = row_dot_i4_f32_pairs(wr, &x[si * i..(si + 1) * i], i) * sc;
+                }
+            }
+        }
+        QTKind::I2 { data, scale } => {
+            let rb = i.div_ceil(4);
+            #[cfg(target_arch = "x86_64")]
+            if has_avx2() {
+                for (j, out) in yt.chunks_mut(s).enumerate() {
+                    let row = r0 + j;
+                    let wr = &data[row * rb..(row + 1) * rb];
+                    let sc = scale[row];
+                    for (si, slot) in out.iter_mut().enumerate() {
+                        // Safety: AVX2 just verified; a whole-machine capability.
+                        *slot = unsafe { dot_i2_f32_avx2(wr, &x[si * i..(si + 1) * i], i) } * sc;
+                    }
+                }
+                return;
+            }
+            for (j, out) in yt.chunks_mut(s).enumerate() {
+                let row = r0 + j;
+                let wr = &data[row * rb..(row + 1) * rb];
+                let sc = scale[row];
+                for (si, slot) in out.iter_mut().enumerate() {
+                    *slot = row_dot_i2_f32_scalar(wr, &x[si * i..(si + 1) * i], i) * sc;
+                }
+            }
+        }
+        QTKind::I4Grouped { data, scale, group_size } => {
+            let rb = i.div_ceil(2);
+            let ngroups = i.div_ceil(*group_size);
+            for (j, out) in yt.chunks_mut(s).enumerate() {
+                let row = r0 + j;
+                let wr = &data[row * rb..(row + 1) * rb];
+                let sc = &scale[row * ngroups..(row + 1) * ngroups];
+                for (si, slot) in out.iter_mut().enumerate() {
+                    *slot = row_dot_i4_grouped_scalar(wr, sc, &x[si * i..(si + 1) * i], i, *group_size);
+                }
+            }
+        }
+        QTKind::MxFp4 { data, block_scale } => {
+            let rb = i.div_ceil(2);
+            let bpr = i.div_ceil(32);
+            #[cfg(target_arch = "x86_64")]
+            if is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512bw") {
+                for (j, out) in yt.chunks_mut(s).enumerate() {
+                    let row = r0 + j;
+                    let wr = &data[row * rb..(row + 1) * rb];
+                    let bs = &block_scale[row * bpr..(row + 1) * bpr];
+                    for (si, slot) in out.iter_mut().enumerate() {
+                        // Safety: AVX-512F/BW just verified; a whole-machine capability.
+                        *slot = unsafe { dot_mxfp4_f32_avx512(wr, bs, &x[si * i..(si + 1) * i], i) };
+                    }
+                }
+                return;
+            }
+            for (j, out) in yt.chunks_mut(s).enumerate() {
+                let row = r0 + j;
+                let wr = &data[row * rb..(row + 1) * rb];
+                let bs = &block_scale[row * bpr..(row + 1) * bpr];
+                for (si, slot) in out.iter_mut().enumerate() {
+                    *slot = row_dot_mxfp4_f32_scalar(wr, bs, &x[si * i..(si + 1) * i], i);
+                }
+            }
+        }
+    }
+}
+
 fn quantize_activations(x: &[f32], s: usize, cols: usize) -> (Vec<i8>, Vec<f32>) {
     let mut xq = vec![0i8; s * cols];
     let mut sx = vec![0f32; s];
@@ -948,6 +1365,141 @@ fn quantize_activations(x: &[f32], s: usize, cols: usize) -> (Vec<i8>, Vec<f32>)
     (xq, sx)
 }
 
+/// A dense weight row-sharded across the NUMA domains (Phase N4a, `NUMA_AMX_BRIEF.md`): shard
+/// `i`'s rows live in their own [`QT`] whose pages were first-touched inside domain `i`'s pinned
+/// pool, so a [`QTSharded::matvec`] fan-out has every domain reading domain-local weight bytes.
+/// Row-major `QT` buffers make the split trivial — a contiguous row range of any `QTKind` is a
+/// contiguous byte range in each of its buffers.
+///
+/// s=1 (decode matvec) only, deliberately: at s=1 `matmul_qt`'s internal transposed `yt` layout
+/// IS the output layout, so shard `i` writes `y[start_i..start_i+rows_i]` directly and
+/// reassembly is free. Every converted call site (lm_head, KDA projections) is structurally
+/// s=1; prefill's s>1 matmuls are not converted (see the N4 `PERFORMANCE.md` section for the
+/// fan-out-frequency arithmetic that also keeps the latent down/up unconverted).
+pub struct QTSharded {
+    shards: Vec<QT>,
+    /// `starts[i]` = first output row of shard `i`; `starts[n] == rows` (one-past-the-end
+    /// sentinel, so `starts[i+1] - starts[i]` is shard `i`'s row count).
+    starts: Vec<usize>,
+    pub rows: usize,
+    pub cols: usize,
+}
+
+impl QTSharded {
+    /// Splits `qt`'s rows into one contiguous block per domain of `pools`, COPYING each block
+    /// into a fresh `QT` allocated and filled inside that domain's pinned pool — the copy is
+    /// the placement mechanism (first touch on a policy-reset pinned worker), exactly N3b's
+    /// trick applied to a dense weight. The original allocation is dropped. Load-time cost:
+    /// one extra pass over the weight's bytes.
+    pub fn shard(qt: QT, pools: &crate::numa::NodePools) -> QTSharded {
+        let n = pools.n();
+        let (rows, cols) = (qt.rows, qt.cols);
+        let per = rows.div_ceil(n);
+        let starts: Vec<usize> = (0..=n).map(|i| (i * per).min(rows)).collect();
+        let mut shards: Vec<Option<QT>> = (0..n).map(|_| None).collect();
+        {
+            let slots: Vec<std::sync::Mutex<&mut Option<QT>>> = shards.iter_mut().map(std::sync::Mutex::new).collect();
+            let (qt, starts) = (&qt, &starts);
+            pools.run_all(|i| {
+                **slots[i].lock().unwrap() = Some(qt.copy_rows(starts[i], starts[i + 1] - starts[i]));
+            });
+        }
+        QTSharded { shards: shards.into_iter().map(|s| s.expect("every domain filled its shard")).collect(), starts, rows, cols }
+    }
+
+    /// `y[rows] = W @ x[cols]` — [`matmul_qt`] at s=1, as ONE cross-domain fan-out with each
+    /// domain computing its own shard's row range (row-block-parallel within its pool) straight
+    /// into its disjoint slice of `y`. Bit-identical to `matmul_qt` on the unsharded weight:
+    /// every row goes through the same [`matmul_qt_rows`] kernel bodies in the same per-element
+    /// order, and shard boundaries change only which thread computes a row (pinned by
+    /// `qt_sharded_matvec_is_bit_identical_to_matmul_qt` on this machine's real topology).
+    pub fn matvec(&self, y: &mut [f32], x: &[f32], pools: &crate::numa::NodePools) {
+        debug_assert_eq!(y.len(), self.rows);
+        matvec_sharded_batch(pools, x, &mut [(y, self)]);
+    }
+}
+
+/// A dense weight that is either today's plain [`QT`] or its NUMA-sharded form — the storage
+/// type for the handful of decode-hot dense weights Phase N4 converts (K3's lm_head and KDA
+/// q/k/v/o projections). Loaded `Plain` always; a post-load pass (`kimi_k3::model::Model::
+/// distribute_dense`) shards them iff `--numa` pools exist, so every non-NUMA configuration
+/// carries exactly the storage and code path it had before.
+pub enum DenseQT {
+    Plain(QT),
+    Sharded(QTSharded),
+}
+
+impl DenseQT {
+    /// s=1 matvec through whichever storage this is. `Sharded` fetches the singleton pools —
+    /// sharding only ever happens because the pools exist, and the singleton lives for the
+    /// process, so this cannot dangle.
+    pub fn matvec(&self, y: &mut [f32], x: &[f32]) {
+        match self {
+            DenseQT::Plain(qt) => matmul_qt(y, x, qt, 1),
+            DenseQT::Sharded(w) => w.matvec(y, x, crate::numa::NodePools::get().expect("sharded weights exist only when the node pools do")),
+        }
+    }
+
+    /// Consumes a `Plain` into `Sharded` across `pools` (no-op if already sharded).
+    pub fn shard(self, pools: &crate::numa::NodePools) -> DenseQT {
+        match self {
+            DenseQT::Plain(qt) => DenseQT::Sharded(QTSharded::shard(qt, pools)),
+            sharded => sharded,
+        }
+    }
+
+    /// In-place [`DenseQT::shard`] for weights living behind `&mut` in a loaded model (the
+    /// momentary placeholder is a zero-row `QT`, never observable — this is single-threaded
+    /// load-time code).
+    pub fn shard_in_place(&mut self, pools: &crate::numa::NodePools) {
+        if matches!(self, DenseQT::Plain(_)) {
+            let plain = std::mem::replace(self, DenseQT::Plain(QT::alloc(0, 0, 32, false)));
+            *self = plain.shard(pools);
+        }
+    }
+
+    pub fn rows(&self) -> usize {
+        match self {
+            DenseQT::Plain(qt) => qt.rows,
+            DenseQT::Sharded(w) => w.rows,
+        }
+    }
+}
+
+/// Several sharded matvecs over the SAME `x` in ONE cross-domain fan-out — the batched shape
+/// KDA's q/k/v projections want (three weights, one fan-out per layer instead of three; the
+/// fan-out itself is the dominant cost at this call frequency, see `PERFORMANCE.md`).
+pub fn matvec_sharded_batch(pools: &crate::numa::NodePools, x: &[f32], jobs: &mut [(&mut [f32], &QTSharded)]) {
+    // Split every output at its weight's shard boundaries up front, so the fan-out closure only
+    // ever sees its own domain's disjoint `&mut` slices (Mutex = `Fn`-boundary plumbing, one
+    // uncontended lock per (job, domain), never blocking).
+    let mut per_domain: Vec<Vec<(std::sync::Mutex<&mut [f32]>, &QT, usize)>> = (0..pools.n()).map(|_| Vec::new()).collect();
+    for (y, w) in jobs.iter_mut() {
+        debug_assert_eq!(y.len(), w.rows);
+        let mut rest: &mut [f32] = y;
+        for i in 0..pools.n() {
+            let nr = w.starts[i + 1] - w.starts[i];
+            let (mine, tail) = rest.split_at_mut(nr);
+            rest = tail;
+            per_domain[i].push((std::sync::Mutex::new(mine), &w.shards[i], nr));
+        }
+    }
+    pools.run_all(|i| {
+        let tpp = pools.threads_per_pool();
+        for (slot, shard, nr) in &per_domain[i] {
+            if *nr == 0 {
+                continue;
+            }
+            let mut out = slot.lock().unwrap();
+            let acts = RowActs::prepare(x, shard, 1);
+            let blk = nr.div_ceil(tpp * 4).max(1);
+            out.par_chunks_mut(blk).enumerate().for_each(|(b, seg)| {
+                matmul_qt_rows(seg, &acts, shard, b * blk, seg.len());
+            });
+        }
+    });
+}
+
 /// Scalar reference for `qt_addrow`'s int4 branch — factored out of the inline loop so
 /// `axpy_i4_f32_avx512` can be checked for bit-exactness against this SAME logic (not a
 /// differently-associated reference like `QT::row_f32`), and so the AVX-512 dispatch's fallback
@@ -1133,6 +1685,183 @@ mod tests {
         (0..n).map(|_| xorshift(&mut s)).collect()
     }
 
+    /// The contract `matmul_qt_rows` exists to make (and that K3's Phase 5 v2 dispatch is built
+    /// on): slicing a matmul into arbitrary row blocks and reassembling them reproduces
+    /// `matmul_qt`'s output **bit for bit**, for every `QTKind`, at `s = 1` (decode, where the
+    /// int4 tier takes its float branch) and `s > 1` (prefill, where it takes int8 IDOT).
+    ///
+    /// `assert_eq!` on the raw f32s, not a tolerance — the whole point is that the two paths run
+    /// the same per-row kernels in the same per-element order, so any drift between them (a tier
+    /// that stops mirroring `matmul_qt`'s dispatch, a `row_dot_*` helper edited on one side only)
+    /// must fail here rather than quietly change a checkpoint's output.
+    ///
+    /// Dims are deliberately awkward: `cols` not a multiple of 32 or 64 exercises the SIMD tiers'
+    /// scalar tails, `rows` prime means the row blocks below don't divide evenly.
+    #[test]
+    fn matmul_qt_rows_reassembled_is_bit_identical_to_matmul_qt() {
+        let (rows, cols) = (37usize, 100usize);
+        let w = random_vec(rows * cols, 7);
+        let gs = 32;
+        let variants: Vec<(&str, QT)> = vec![
+            ("f32", {
+                let mut t = QT::alloc(rows, cols, 32, false);
+                t.fill(&w);
+                t
+            }),
+            ("i8", {
+                let mut t = QT::alloc(rows, cols, 8, false);
+                t.fill(&w);
+                t
+            }),
+            ("i4", {
+                let mut t = QT::alloc(rows, cols, 4, false);
+                t.fill(&w);
+                t
+            }),
+            ("i2", {
+                let mut t = QT::alloc(rows, cols, 2, false);
+                t.fill(&w);
+                t
+            }),
+            ("i4grouped", {
+                let mut t = QT::alloc_grouped(rows, cols, 4, gs);
+                t.fill(&w);
+                t
+            }),
+            ("mxfp4", {
+                let mut t = QT::alloc_mxfp4(rows, cols);
+                t.fill(&w);
+                t
+            }),
+        ];
+
+        for s in [1usize, 3] {
+            let x = random_vec(s * cols, 91);
+            for (name, t) in &variants {
+                let mut expected = vec![0f32; s * rows];
+                matmul_qt(&mut expected, &x, t, s);
+
+                // Reassemble from row blocks of a size that leaves a short final block.
+                let acts = RowActs::prepare(&x, t, s);
+                let mut yt = vec![0f32; rows * s];
+                let blk = 8;
+                for (b, seg) in yt.chunks_mut(blk * s).enumerate() {
+                    matmul_qt_rows(seg, &acts, t, b * blk, seg.len() / s);
+                }
+                let mut got = vec![0f32; s * rows];
+                for (oi, col) in yt.chunks(s).enumerate() {
+                    for (si, &v) in col.iter().enumerate() {
+                        got[si * rows + oi] = v;
+                    }
+                }
+                assert_eq!(got, expected, "{name} at s={s}");
+            }
+        }
+    }
+
+    /// Phase N4's acceptance gate at the unit level: sharding a weight across real pinned
+    /// domain pools and computing the matvec as one cross-domain fan-out reproduces `matmul_qt`
+    /// **bit for bit**, for every `QTKind` — including a batched (multi-weight, one fan-out)
+    /// call and a row count that doesn't divide evenly across domains. Throwaway
+    /// `NodePools::build` pools (never the singleton); SKIPs on single-node machines.
+    #[test]
+    fn qt_sharded_matvec_is_bit_identical_to_matmul_qt() {
+        let Some(topo) = crate::numa::topology() else {
+            eprintln!("SKIP: no NUMA topology on this machine");
+            return;
+        };
+        let Some(pools) = crate::numa::NodePools::build(topo.nodes, 8) else {
+            eprintln!("SKIP: single NUMA node — nothing to shard across");
+            return;
+        };
+        let (rows, cols) = (37usize, 100usize);
+        let w = random_vec(rows * cols, 5);
+        let x = random_vec(cols, 55);
+        let variants: Vec<QT> = vec![
+            {
+                let mut t = QT::alloc(rows, cols, 32, false);
+                t.fill(&w);
+                t
+            },
+            {
+                let mut t = QT::alloc(rows, cols, 8, false);
+                t.fill(&w);
+                t
+            },
+            {
+                let mut t = QT::alloc(rows, cols, 4, false);
+                t.fill(&w);
+                t
+            },
+            {
+                let mut t = QT::alloc(rows, cols, 2, false);
+                t.fill(&w);
+                t
+            },
+            {
+                let mut t = QT::alloc_grouped(rows, cols, 4, 32);
+                t.fill(&w);
+                t
+            },
+            {
+                let mut t = QT::alloc_mxfp4(rows, cols);
+                t.fill(&w);
+                t
+            },
+        ];
+        let mut sharded = Vec::new();
+        for t in variants {
+            let mut expected = vec![0f32; rows];
+            matmul_qt(&mut expected, &x, &t, 1);
+            let s = QTSharded::shard(t, &pools);
+            let mut got = vec![0f32; rows];
+            s.matvec(&mut got, &x, &pools);
+            let a: Vec<u32> = got.iter().map(|v| v.to_bits()).collect();
+            let b: Vec<u32> = expected.iter().map(|v| v.to_bits()).collect();
+            assert_eq!(a, b, "sharded matvec must be bit-identical to matmul_qt");
+            sharded.push((s, expected));
+        }
+        // Batched: all six weights in ONE fan-out, same bits.
+        let mut outs: Vec<Vec<f32>> = sharded.iter().map(|(s, _)| vec![0f32; s.rows]).collect();
+        {
+            let mut jobs: Vec<(&mut [f32], &QTSharded)> = outs.iter_mut().map(|o| o.as_mut_slice()).zip(sharded.iter().map(|(s, _)| s)).collect();
+            matvec_sharded_batch(&pools, &x, &mut jobs);
+        }
+        for (out, (_, expected)) in outs.iter().zip(&sharded) {
+            let a: Vec<u32> = out.iter().map(|v| v.to_bits()).collect();
+            let b: Vec<u32> = expected.iter().map(|v| v.to_bits()).collect();
+            assert_eq!(a, b, "batched sharded matvec must be bit-identical too");
+        }
+    }
+
+    /// `RowActs` keeps `x` even when it prepared an int8 copy, so an `acts` built against the
+    /// wrong weight still computes the right answer (it re-quantizes inline) instead of silently
+    /// using a mismatched activation — the fallback documented on `RowActs`. Cheap insurance
+    /// against a caller that reuses one `acts` across differently-quantized weights.
+    #[test]
+    fn matmul_qt_rows_is_still_correct_when_acts_were_prepared_for_another_weight_kind() {
+        let (rows, cols, s) = (6usize, 64usize, 2usize);
+        let w = random_vec(rows * cols, 13);
+        let x = random_vec(s * cols, 17);
+        let mut i8t = QT::alloc(rows, cols, 8, false);
+        i8t.fill(&w);
+        let mut f32t = QT::alloc(rows, cols, 32, false);
+        f32t.fill(&w);
+
+        let mut expected = vec![0f32; s * rows];
+        matmul_qt(&mut expected, &x, &i8t, s);
+
+        // Prepared against the f32 weight (so: no int8 copy), used against the int8 one.
+        let mismatched = RowActs::prepare(&x, &f32t, s);
+        let mut yt = vec![0f32; rows * s];
+        matmul_qt_rows(&mut yt, &mismatched, &i8t, 0, rows);
+        for (oi, col) in yt.chunks(s).enumerate() {
+            for (si, &v) in col.iter().enumerate() {
+                assert_eq!(v, expected[si * rows + oi], "row {oi}, seq {si}");
+            }
+        }
+    }
+
     #[test]
     fn matmul_matches_hand_computed_dot_products() {
         // x: [S=2, I=3], W: [O=2, I=3]
@@ -1218,6 +1947,98 @@ mod tests {
         assert_eq!(y[0], 32.0 * 2.0 + 8.0 * -48.0);
     }
 
+    /// Phase 2's block-structured `matmul_mxfp4` must be **bit-identical** to the pre-Phase-2
+    /// element-at-a-time loop (`e8m0_decode` per element, `k & 1` nibble branch) — same inputs to
+    /// the same decode functions in the same per-element order, only the loop shape changed.
+    /// Verified across sizes including `i` not a multiple of 32/64 and odd `o`, at s=1 and s>1,
+    /// comparing raw f32 bits (not an epsilon) since the claim is exactness, not closeness.
+    #[test]
+    fn matmul_mxfp4_matches_the_pre_block_reference() {
+        // A straight port of the pre-Phase-2 inner loop, kept here as the exactness oracle.
+        fn reference(y: &mut [f32], x: &[f32], data: &[u8], block_scale: &[u8], s: usize, i: usize, o: usize) {
+            let rb = i.div_ceil(2);
+            let bpr = i.div_ceil(32);
+            for oi in 0..o {
+                let w = &data[oi * rb..(oi + 1) * rb];
+                let bs = &block_scale[oi * bpr..(oi + 1) * bpr];
+                for si in 0..s {
+                    let xs = &x[si * i..(si + 1) * i];
+                    let mut a = 0f32;
+                    for k in 0..i {
+                        let byte = w[k >> 1];
+                        let nibble = if k & 1 == 0 { byte & 0xF } else { byte >> 4 };
+                        a += xs[k] * e2m1_decode(nibble) * e8m0_decode(bs[k / 32]);
+                    }
+                    y[si * o + oi] = a;
+                }
+            }
+        }
+
+        let mut seed = 0x1234_5678u32;
+        let rnd = |seed: &mut u32| {
+            *seed ^= *seed << 13;
+            *seed ^= *seed >> 17;
+            *seed ^= *seed << 5;
+            *seed
+        };
+        // (s, i, o): i covers multiples of 32, non-multiples, odd, <32 tail; o includes odd rows.
+        for &(s, i, o) in &[(1usize, 32usize, 1usize), (1, 40, 3), (8, 96, 5), (1, 33, 7), (4, 3584, 6), (2, 3072, 4), (1, 17, 2), (3, 64, 1)] {
+            let rb = i.div_ceil(2);
+            let bpr = i.div_ceil(32);
+            let data: Vec<u8> = (0..o * rb).map(|_| (rnd(&mut seed) & 0xFF) as u8).collect();
+            // E8M0 bytes in [118,136] keep 2^(b-127) a modest finite factor (no inf poisoning).
+            let scale: Vec<u8> = (0..o * bpr).map(|_| 118 + (rnd(&mut seed) % 19) as u8).collect();
+            let x: Vec<f32> = (0..s * i).map(|_| (rnd(&mut seed) as f32 / u32::MAX as f32 - 0.5) * 2.0).collect();
+
+            let mut y_new = vec![0f32; s * o];
+            let mut y_ref = vec![0f32; s * o];
+            // Targets the SCALAR tier explicitly: `matmul_mxfp4` dispatches to the AVX-512 tier on
+            // capable CPUs, which is within-tolerance (reassociated), not bit-identical.
+            matmul_mxfp4_scalar(&mut y_new, &x, &data, &scale, s, i, o);
+            reference(&mut y_ref, &x, &data, &scale, s, i, o);
+            for (a, b) in y_new.iter().zip(&y_ref) {
+                assert_eq!(a.to_bits(), b.to_bits(), "mismatch at (s={s}, i={i}, o={o}): {a} vs {b}");
+            }
+        }
+    }
+
+    /// Phase 3's AVX-512 MXFP4 tier vs the scalar tier — **within-tolerance, not bit-exact**: the
+    /// tier folds the E8M0 scale into each block's decoded weights and reduces two FMA chains via a
+    /// tree-sum, reassociating the scalar `x*e2m1*scale` running sum (same tradeoff `matmul_i4`'s
+    /// AVX-512 tier's tests accept). Random weights/scales/activations across dims including `i` not
+    /// a multiple of 32/64 (partial tail block) and odd `o`. Tolerance is relative-with-floor at
+    /// 2e-3 — generous next to the sub-1e-4 relative error reassociation actually produces here, but
+    /// robust to the magnitudes larger `i` and E8M0 scale bytes create.
+    #[test]
+    fn matmul_mxfp4_avx512_matches_scalar_within_tolerance() {
+        if !(is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512bw")) {
+            eprintln!("SKIP: no AVX-512F/BW on this CPU");
+            return;
+        }
+        let mut seed = 0x9E37_79B1u32;
+        let rnd = |seed: &mut u32| {
+            *seed ^= *seed << 13;
+            *seed ^= *seed >> 17;
+            *seed ^= *seed << 5;
+            *seed
+        };
+        for &(s, i, o) in &[(1usize, 32usize, 4usize), (1, 64, 3), (8, 96, 5), (1, 40, 7), (2, 3584, 4), (1, 3072, 6), (1, 33, 2), (4, 128, 1)] {
+            let rb = i.div_ceil(2);
+            let bpr = i.div_ceil(32);
+            let data: Vec<u8> = (0..o * rb).map(|_| (rnd(&mut seed) & 0xFF) as u8).collect();
+            let scale: Vec<u8> = (0..o * bpr).map(|_| 122 + (rnd(&mut seed) % 11) as u8).collect(); // [122,132]
+            let x: Vec<f32> = (0..s * i).map(|_| (rnd(&mut seed) as f32 / u32::MAX as f32 - 0.5) * 2.0).collect();
+            let mut y_scalar = vec![0f32; s * o];
+            let mut y_avx = vec![0f32; s * o];
+            matmul_mxfp4_scalar(&mut y_scalar, &x, &data, &scale, s, i, o);
+            unsafe { matmul_mxfp4_avx512(&mut y_avx, &x, &data, &scale, s, i, o) };
+            for (a, b) in y_avx.iter().zip(&y_scalar) {
+                let tol = 2e-3 * (1.0 + b.abs());
+                assert!((a - b).abs() <= tol, "s={s} i={i} o={o}: {a} vs {b} (tol {tol})");
+            }
+        }
+    }
+
     #[test]
     fn matmul_i4_grouped_matches_manual_nibble_unpack_across_two_groups() {
         // 6 columns, group_size=3 -> group 0 = cols[0..3] (scale 2.0), group 1 = cols[3..6]
@@ -1400,7 +2221,13 @@ mod tests {
         }
         let mut y = vec![0.0f32; s * rows];
         matmul_qt(&mut y, &x, &t, s);
-        assert_eq!(y, expected);
+        // Was `assert_eq!` when the MxFp4 path was scalar-only. Phase 3 gave `matmul_mxfp4` an
+        // AVX-512 tier that `matmul_qt` dispatches to here; it reassociates the per-row sum
+        // (scale-fold + dual-chain tree reduce), so this is now WITHIN-TOLERANCE, exactly like this
+        // file's `matmul_qt` int4 test — whose i4 path has had an AVX-512 tier all along.
+        for (a, b) in y.iter().zip(&expected) {
+            assert!((a - b).abs() <= 1e-3 * (1.0 + b.abs()), "{a} vs {b}");
+        }
     }
 
     #[test]
diff --git a/src/kimi_k3/chat_template.rs b/src/kimi_k3/chat_template.rs
index d724bb8..f014cf5 100644
--- a/src/kimi_k3/chat_template.rs
+++ b/src/kimi_k3/chat_template.rs
@@ -37,7 +37,7 @@
 //! checkpoint's `config.json`'s `eos_token_id` — matches; `kimi_k3::config::Cfg` already reads
 //! this generically via `cfg.base.stop_ids`, no change needed there).
 
-use crate::chat::Role;
+use crate::chat::{Emit, Role};
 
 const OPEN: &str = "<|open|>";
 const CLOSE: &str = "<|close|>";
@@ -155,10 +155,130 @@ pub fn render_messages(messages: &[(Role, String)], think: bool) -> String {
     out
 }
 
+/// The output-side inverse of [`generation_prompt`]: strips the structural XTML envelope the
+/// model emits to close the turn that `generation_prompt` deliberately left open.
+///
+/// `generation_prompt` hands the model an UN-closed `<|open|>response<|sep|>` (or
+/// `<|open|>think<|sep|>` under `think`), so a well-behaved K3 completion ends by closing what
+/// was opened: `...<|close|>response<|sep|><|close|>message<|sep|><|end_of_msg|>`. Only
+/// `<|end_of_msg|>` (163586) is in `cfg.base.stop_ids`, and `generate` stops on it without
+/// emitting it — but `<|close|>` (163588) and `<|sep|>` (163589) are `"special": false` in the
+/// checkpoint's `added_tokens_decoder`, so the tokenizer decodes them to their literal text and
+/// they leak into the reply unless something strips them. That something is this.
+///
+/// Under `think` the model additionally switches channels mid-generation
+/// (`<|close|>think<|sep|><|open|>response<|sep|>`), which is exactly why the simpler "stop at
+/// the first `<|close|>`" rule is correct only for `!think`. Reasoning-channel text is passed
+/// through — rabbit's existing convention, GLM-5.2 likewise shows `<think>` content inline —
+/// the six control tokens of the transition are swallowed, and the reply continues with the
+/// response channel's body.
+pub struct ResponseFilter {
+    state: State,
+}
+
+enum State {
+    /// Emitting the think channel's body; its `<|close|>` begins the transition.
+    Think,
+    /// Swallowing `<|close|>think<|sep|><|open|>response<|sep|>` — two `<|sep|>`s wide. Counting
+    /// separators rather than matching the tag names keeps this independent of how `think` and
+    /// `response` happen to tokenize.
+    Transition { seps: u8 },
+    /// Emitting the response channel's body; its `<|close|>` ends the reply.
+    Response,
+}
+
+impl ResponseFilter {
+    /// `think` must be the same flag the turn's prompt was rendered with — it decides which
+    /// channel [`generation_prompt`] left open, and so which `<|close|>` terminates the reply.
+    pub fn new(think: bool) -> Self {
+        Self { state: if think { State::Think } else { State::Response } }
+    }
+
+    /// Classifies one freshly generated token by its decoded text.
+    pub fn step(&mut self, token: &str) -> Emit {
+        match &mut self.state {
+            State::Think => {
+                if token == CLOSE {
+                    self.state = State::Transition { seps: 0 };
+                    Emit::Skip
+                } else {
+                    Emit::Text
+                }
+            }
+            State::Transition { seps } => {
+                if token == SEP {
+                    *seps += 1;
+                    if *seps >= 2 {
+                        self.state = State::Response;
+                    }
+                }
+                Emit::Skip
+            }
+            State::Response => {
+                if token == CLOSE {
+                    Emit::Stop
+                } else {
+                    Emit::Text
+                }
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    /// Feeds `tokens` through the filter, returning the text that survives and whether the
+    /// filter asked to stop (mirroring `generate_reply`'s loop).
+    fn run(think: bool, tokens: &[&str]) -> (String, bool) {
+        let mut f = ResponseFilter::new(think);
+        let mut out = String::new();
+        for t in tokens {
+            match f.step(t) {
+                Emit::Text => out.push_str(t),
+                Emit::Skip => {}
+                Emit::Stop => return (out, true),
+            }
+        }
+        (out, false)
+    }
+
+    /// The exact token sequence the real checkpoint produced for "Can you tell me who you are
+    /// please?" (captured from a `--serve` SSE stream, 2026-08-01) — the bug this filter fixes.
+    #[test]
+    fn strips_the_closing_envelope_the_real_checkpoint_emits_without_thinking() {
+        let (text, stopped) = run(false, &["I", "’m", " Kim", "i", ".", CLOSE, "response", SEP, CLOSE, "message", SEP]);
+        assert_eq!(text, "I’m Kimi.");
+        assert!(stopped, "the response channel's <|close|> must end the turn, not be emitted");
+    }
+
+    #[test]
+    fn passes_through_a_reply_that_never_closes_its_channel() {
+        let (text, stopped) = run(false, &["still", " going"]);
+        assert_eq!(text, "still going");
+        assert!(!stopped, "max_tokens truncation must not look like a channel close");
+    }
+
+    /// Under `think` the first `<|close|>` is the think channel's, not the reply's — stopping
+    /// there would swallow the entire response.
+    #[test]
+    fn thinking_mode_keeps_generating_across_the_think_to_response_transition() {
+        let (text, stopped) = run(
+            true,
+            &["reason", "ing", CLOSE, "think", SEP, OPEN, "response", SEP, "the", " answer", CLOSE, "response", SEP],
+        );
+        assert_eq!(text, "reasoningthe answer", "transition markup is swallowed, both channels' bodies survive");
+        assert!(stopped);
+    }
+
+    #[test]
+    fn thinking_mode_does_not_stop_on_a_close_that_only_ends_the_think_channel() {
+        let (text, stopped) = run(true, &["hmm", CLOSE, "think", SEP, OPEN, "response", SEP, "hi"]);
+        assert_eq!(text, "hmmhi");
+        assert!(!stopped);
+    }
+
     #[test]
     fn first_turn_with_thinking_and_system_prompt_emits_preamble_then_system_then_user_then_open_think() {
         let out = render_turn("hi", true, true, Some("be nice"));
diff --git a/src/kimi_k3/generate.rs b/src/kimi_k3/generate.rs
index 9fcc647..d912ed4 100644
--- a/src/kimi_k3/generate.rs
+++ b/src/kimi_k3/generate.rs
@@ -24,7 +24,7 @@ use crate::expert_cache::{ExpertCache, ExpertNaming};
 use crate::generate::{Phases, StepProfile};
 use crate::glm52::attention::{self, rmsnorm, OutputGate};
 use crate::glm52::moe::{self, Activation};
-use crate::kernels::matmul_qt;
+use crate::kernels::{matmul_qt, matvec_sharded_batch, DenseQT};
 use crate::kimi_k3::attn_res::{self, AttnResState};
 use crate::kimi_k3::config::Cfg;
 use crate::kimi_k3::model::{self, Attn, Ffn, KdaOutputGate, KdaWeights, Model, ModelError};
@@ -127,7 +127,7 @@ impl ExpertCaches {
         // `.weight_packed`/`.weight_scale` naming; every synthetic test fixture in this crate
         // uses plain `.weight` float tensors and needs the original `KimiK3` naming.
         let naming = if model.cfg.mxfp4_experts { ExpertNaming::KimiK3Mxfp4 } else { ExpertNaming::KimiK3 };
-        let v = model.layers.iter().map(|l| if matches!(l.ffn, Ffn::Moe(_)) { Some(ExpertCache::for_family(capacity, naming)) } else { None }).collect();
+        let v = model.layers.iter().map(|l| if matches!(l.ffn, Ffn::Moe(_)) { Some(ExpertCache::for_family(capacity, model.cfg.base.n_experts as usize, naming)) } else { None }).collect();
         ExpertCaches(v)
     }
 
@@ -139,6 +139,22 @@ impl ExpertCaches {
         self.0.iter().flatten().map(|c| c.io_wait_nanos).sum()
     }
 
+    /// Whether any layer's cache loads through an `io_uring` ring. `false` (every MXFP4/K3 run,
+    /// whose naming never gets a ring) means `io_wait_nanos_total` is structurally zero, so the
+    /// CLI must not present it as "actual disk wait" (Phase 4c) — see `ExpertCache::has_ring`.
+    pub fn any_has_ring(&self) -> bool {
+        self.0.iter().flatten().any(|c| c.has_ring())
+    }
+
+    /// Phase 4b: preload every MoE layer's routed experts up front. K3's experts live at the
+    /// LATENT width, so `to_glm_cfg_expert` (not `to_glm_cfg`) supplies the shapes — the same
+    /// config `latent_moe`'s own `ensure_loaded` uses. See `expert_cache::preload_layers`.
+    pub fn preload(&mut self, model: &Model, shards: &Shards) -> Result<(), ModelError> {
+        let cfg = crate::kimi_k3::model::to_glm_cfg_expert(&model.cfg);
+        let n_experts = cfg.n_experts as usize;
+        Ok(crate::expert_cache::preload_layers(&mut self.0, shards, &cfg, model.ebits, n_experts)?)
+    }
+
     pub fn warm_start(&mut self, model_dir: &std::path::Path, cache_capacity: usize) -> crate::generate::WarmStartStats {
         let loaded = crate::usage_cache::load(&crate::usage_cache::usage_path(model_dir));
         let hist: u64 = loaded.values().sum();
@@ -195,18 +211,38 @@ fn embed_tokens(model: &Model, ids: &[usize]) -> Vec<f32> {
 /// checkpoints that don't set it — see `kda.rs::decay_gate`'s doc), and the output gate's `g2`
 /// input comes from EITHER `w.output_gate`'s low-rank or full-rank projection (the math
 /// consuming `g2`, `head_output_gate`, is unchanged either way).
-fn kda_step(cfg: &Cfg, w: &KdaWeights, state: &mut KdaLayerState, x: &[f32], out: &mut [f32]) {
+fn kda_step(cfg: &Cfg, w: &KdaWeights, state: &mut KdaLayerState, x: &[f32], out: &mut [f32], mut phases: Option<&mut Phases>) {
     let head_dim = cfg.base.kda_head_dim as usize;
     let n_heads = cfg.base.kda_n_heads as usize;
     let d_inner = head_dim * n_heads;
     let eps = cfg.base.eps;
 
+    // Phase N5a instrumentation (timing only — every operation below is byte-for-byte what it
+    // was): the step's cost is bucketed into its projection matmuls vs everything recurrent
+    // (convs, activations/norms, the 96-head state update). A handful of `Instant::now()` calls
+    // against a multi-ms step; noise. See `Phases::attn_kda_proj_s`'s doc.
+    let mut proj_ns = 0u128;
+    let mut recur_ns = 0u128;
+    let mut t = std::time::Instant::now();
+
     let mut q_pre = vec![0f32; d_inner];
-    matmul_qt(&mut q_pre, x, &w.q_proj, 1);
     let mut k_pre = vec![0f32; d_inner];
-    matmul_qt(&mut k_pre, x, &w.k_proj, 1);
     let mut v_pre = vec![0f32; d_inner];
-    matmul_qt(&mut v_pre, x, &w.v_proj, 1);
+    // N4: sharded q/k/v run as ONE cross-domain fan-out (three weights amortizing one fan-out's
+    // wake cost — at 69 KDA layers/token the fan-out count is the price that matters). Plain
+    // weights (no --numa) take exactly the pre-N4 three matmuls.
+    match (&w.q_proj, &w.k_proj, &w.v_proj, crate::numa::NodePools::get()) {
+        (DenseQT::Sharded(q), DenseQT::Sharded(k), DenseQT::Sharded(v), Some(pools)) => {
+            matvec_sharded_batch(pools, x, &mut [(&mut q_pre, q), (&mut k_pre, k), (&mut v_pre, v)]);
+        }
+        _ => {
+            w.q_proj.matvec(&mut q_pre, x);
+            w.k_proj.matvec(&mut k_pre, x);
+            w.v_proj.matvec(&mut v_pre, x);
+        }
+    }
+    proj_ns += t.elapsed().as_nanos();
+    t = std::time::Instant::now();
 
     let mut q = vec![0f32; d_inner];
     state.q_conv.step(&q_pre, &w.q_conv, &mut q);
@@ -233,6 +269,8 @@ fn kda_step(cfg: &Cfg, w: &KdaWeights, state: &mut KdaLayerState, x: &[f32], out
             *c *= q_scale;
         }
     }
+    recur_ns += t.elapsed().as_nanos();
+    t = std::time::Instant::now();
 
     let mut f_a = vec![0f32; head_dim];
     matmul_qt(&mut f_a, x, &w.f_a_proj, 1);
@@ -253,6 +291,8 @@ fn kda_step(cfg: &Cfg, w: &KdaWeights, state: &mut KdaLayerState, x: &[f32], out
             matmul_qt(&mut g2, &g_a, g_b_proj, 1);
         }
     }
+    proj_ns += t.elapsed().as_nanos();
+    t = std::time::Instant::now();
 
     let mut o = vec![0f32; d_inner];
     state.heads.par_iter_mut().zip(o.par_chunks_mut(head_dim)).enumerate().for_each(|(h, (head_state, o_slot))| {
@@ -263,8 +303,16 @@ fn kda_step(cfg: &Cfg, w: &KdaWeights, state: &mut KdaLayerState, x: &[f32], out
         head_state.step(&q[sl.clone()], &k[sl.clone()], &v[sl.clone()], &alpha, beta, o_slot);
         head_output_gate(o_slot, &w.o_norm, &g2[sl], eps);
     });
+    recur_ns += t.elapsed().as_nanos();
+    t = std::time::Instant::now();
 
-    matmul_qt(out, &o, &w.o_proj, 1);
+    w.o_proj.matvec(out, &o);
+    proj_ns += t.elapsed().as_nanos();
+
+    if let Some(p) = phases.as_deref_mut() {
+        p.attn_kda_proj_s += proj_ns as f32 / 1e9;
+        p.attn_kda_recur_s += recur_ns as f32 / 1e9;
+    }
 }
 
 /// One transformer layer, Attention-Residuals-aware: when `cfg.attn_res_block > 0` (and
@@ -309,7 +357,7 @@ fn layer_forward(
     match (&layer.attn, layer_state) {
         (Attn::Kda(w), LayerState::Kda(state)) => {
             for si in 0..s {
-                kda_step(cfg, w, state, &nrm[si * d..(si + 1) * d], &mut attn_delta[si * d..(si + 1) * d]);
+                kda_step(cfg, w, state, &nrm[si * d..(si + 1) * d], &mut attn_delta[si * d..(si + 1) * d], phases.as_deref_mut());
             }
         }
         (Attn::Mla(w), LayerState::Mla(kv)) => {
@@ -319,6 +367,9 @@ fn layer_forward(
                 None => OutputGate::Off,
             };
             attention::attention(&glm_cfg, &w.attn, kv, &nrm, s, pos_base, dsa, absorb, rope, qproj, output_gate, &mut attn_delta);
+            if let Some(p) = phases.as_deref_mut() {
+                p.attn_mla_s += attn_t.elapsed().as_secs_f32();
+            }
         }
         _ => unreachable!("Attn/LayerState variant mismatch -- KvState::new always pairs them per layer"),
     }
@@ -459,7 +510,7 @@ pub fn step(model: &Model, shards: &Shards, caches: &mut ExpertCaches, kv: &mut
     let mut last = x[(s - 1) * d..s * d].to_vec();
     rmsnorm(&mut last, &model.final_norm, model.cfg.base.eps);
     let mut logit = vec![0f32; model.cfg.base.vocab as usize];
-    matmul_qt(&mut logit, &last, &model.lm_head, 1);
+    model.lm_head.matvec(&mut logit, &last);
     Ok(logit)
 }
 
@@ -474,7 +525,7 @@ pub fn step_profiled(model: &Model, shards: &Shards, caches: &mut ExpertCaches,
     rmsnorm(&mut last, &model.final_norm, model.cfg.base.eps);
     let mut logit = vec![0f32; model.cfg.base.vocab as usize];
     let t = std::time::Instant::now();
-    matmul_qt(&mut logit, &last, &model.lm_head, 1);
+    model.lm_head.matvec(&mut logit, &last);
     let lm_head_s = t.elapsed().as_secs_f32();
     Ok((logit, StepProfile { phases, lm_head_s }))
 }
@@ -490,7 +541,7 @@ pub fn step_all(model: &Model, shards: &Shards, caches: &mut ExpertCaches, kv: &
     for si in 0..s {
         let mut row = x[si * d..(si + 1) * d].to_vec();
         rmsnorm(&mut row, &model.final_norm, model.cfg.base.eps);
-        matmul_qt(&mut lo[si * v..(si + 1) * v], &row, &model.lm_head, 1);
+        model.lm_head.matvec(&mut lo[si * v..(si + 1) * v], &row);
     }
     Ok(lo)
 }
diff --git a/src/kimi_k3/model.rs b/src/kimi_k3/model.rs
index 6689a7b..07df62c 100644
--- a/src/kimi_k3/model.rs
+++ b/src/kimi_k3/model.rs
@@ -35,6 +35,7 @@ use crate::glm52::moe::{Activation, RouteConfig};
 use crate::kimi_k3::attn_res::AttnResWeights;
 use crate::kimi_k3::config::{Cfg, ConfigError};
 use crate::kimi_k3::moe::LatentMoeWeights;
+use crate::kernels::DenseQT;
 use crate::quant::{PackedFormatError, QT};
 use crate::safetensors::{SafetensorsError, Shards};
 use std::fmt;
@@ -51,9 +52,14 @@ pub enum KdaOutputGate {
 /// `output_gate` (see `KdaOutputGate`) replacing that struct's always-low-rank
 /// `g_a_proj`/`g_b_proj` pair.
 pub struct KdaWeights {
-    pub q_proj: QT,
-    pub k_proj: QT,
-    pub v_proj: QT,
+    /// q/k/v/o are `DenseQT` — the ready mechanism for Phase N4's KDA-projection row-sharding,
+    /// currently always `Plain`: sharding them was built, gated, and reverted on measurement
+    /// (the per-layer cross-domain fan-out cost exceeded the locality win; see
+    /// `Model::distribute_dense` and `PERFORMANCE.md`, N4). The small projections (f/b/gates)
+    /// stay plain `QT` outright: their per-token cost could never repay a fan-out.
+    pub q_proj: DenseQT,
+    pub k_proj: DenseQT,
+    pub v_proj: DenseQT,
     /// `[d_inner, kernel]` row-major — see `kimi_linear::model::KdaWeights::q_conv`'s doc.
     pub q_conv: Vec<f32>,
     pub k_conv: Vec<f32>,
@@ -66,7 +72,7 @@ pub struct KdaWeights {
     pub b_proj: QT,
     pub output_gate: KdaOutputGate,
     pub o_norm: Vec<f32>,
-    pub o_proj: QT,
+    pub o_proj: DenseQT,
 }
 
 /// One MLA layer's weights: `glm52::model::AttnWeights`, reused unchanged (K3's MLA tensor
@@ -119,7 +125,9 @@ pub struct Layer {
 pub struct Model {
     pub cfg: Cfg,
     pub embed: QT,
-    pub lm_head: QT,
+    /// `DenseQT` for the same N4 reason as `KdaWeights`' q/k/v/o — the [163840, 7168] decode
+    /// matvec is the brief's row-sharding poster child.
+    pub lm_head: DenseQT,
     pub final_norm: Vec<f32>,
     pub layers: Vec<Layer>,
     /// The model-level final Attention-Residuals pooling (`KimiLinearModel._apply_output_attn_res`)
@@ -303,7 +311,7 @@ impl Model {
         // that's a genuinely different real top-level module, not a bug in the oracle.
         let io_bits = if dbits >= 8 { 16 } else { dbits };
         let embed = qt_load(&shards, b.group_size as usize, "language_model.model.embed_tokens.weight", b.vocab as usize, d, io_bits)?;
-        let lm_head = qt_load(&shards, b.group_size as usize, "language_model.lm_head.weight", b.vocab as usize, d, io_bits)?;
+        let lm_head = DenseQT::Plain(qt_load(&shards, b.group_size as usize, "language_model.lm_head.weight", b.vocab as usize, d, io_bits)?);
         let final_norm = ld(&shards, "language_model.model.norm.weight")?;
 
         let output_attn_res = if cfg.attn_res_block > 0 {
@@ -316,6 +324,11 @@ impl Model {
         };
 
         let mut layers = Vec::with_capacity(b.n_layers as usize);
+        let t_layers = std::time::Instant::now();
+        // A rewriting `\r` line is nice in a real terminal but writes NO newlines, so a redirected
+        // log / background monitor sees nothing until the load finishes. Emit newline-delimited
+        // progress when stderr isn't a TTY so any log consumer sees each layer land.
+        let stderr_tty = std::io::IsTerminal::is_terminal(&std::io::stderr());
         for i in 0..b.n_layers as usize {
             let p = |s: &str| format!("language_model.model.layers.{i}.{s}");
             let ap = |s: &str| format!("language_model.model.layers.{i}.self_attn.{s}");
@@ -333,9 +346,9 @@ impl Model {
                     }
                 };
                 Attn::Kda(Box::new(KdaWeights {
-                    q_proj: qt_load(&shards, b.group_size as usize, &ap("q_proj.weight"), d_inner, d, dbits)?,
-                    k_proj: qt_load(&shards, b.group_size as usize, &ap("k_proj.weight"), d_inner, d, dbits)?,
-                    v_proj: qt_load(&shards, b.group_size as usize, &ap("v_proj.weight"), d_inner, d, dbits)?,
+                    q_proj: DenseQT::Plain(qt_load(&shards, b.group_size as usize, &ap("q_proj.weight"), d_inner, d, dbits)?),
+                    k_proj: DenseQT::Plain(qt_load(&shards, b.group_size as usize, &ap("k_proj.weight"), d_inner, d, dbits)?),
+                    v_proj: DenseQT::Plain(qt_load(&shards, b.group_size as usize, &ap("v_proj.weight"), d_inner, d, dbits)?),
                     q_conv: ld(&shards, &ap("q_conv1d.weight"))?,
                     k_conv: ld(&shards, &ap("k_conv1d.weight"))?,
                     v_conv: ld(&shards, &ap("v_conv1d.weight"))?,
@@ -346,7 +359,7 @@ impl Model {
                     b_proj: qt_load(&shards, b.group_size as usize, &ap("b_proj.weight"), kda_n_heads, d, dbits)?,
                     output_gate,
                     o_norm: ld(&shards, &ap("o_norm.weight"))?,
-                    o_proj: qt_load(&shards, b.group_size as usize, &ap("o_proj.weight"), d, d_inner, dbits)?,
+                    o_proj: DenseQT::Plain(qt_load(&shards, b.group_size as usize, &ap("o_proj.weight"), d, d_inner, dbits)?),
                 }))
             } else {
                 let attn = if q_lora > 0 {
@@ -418,9 +431,48 @@ impl Model {
             };
 
             layers.push(Layer { in_ln, post_ln, attn, ffn, attn_res });
+            // Per-layer progress -- the dense load is otherwise silent for minutes on the real
+            // checkpoint (93 layers, ~13 min), indistinguishable from a hang. Elapsed-so-far makes
+            // a slow load visible, not just a count.
+            if stderr_tty {
+                eprint!("\r  loaded layer {}/{} ({:.0}s)", i + 1, b.n_layers, t_layers.elapsed().as_secs_f32());
+                let _ = std::io::Write::flush(&mut std::io::stderr()); // `\r` doesn't auto-flush
+            } else {
+                eprintln!("  loaded layer {}/{} ({:.0}s)", i + 1, b.n_layers, t_layers.elapsed().as_secs_f32());
+            }
+        }
+        if stderr_tty {
+            eprintln!(); // terminate the rewriting line before chat.rs's "model loaded in ..." prints
         }
 
-        Ok(Model { cfg, embed, lm_head, final_norm, layers, output_attn_res, ebits, route_cfg: RouteConfig::default() })
+        let mut model = Model { cfg, embed, lm_head, final_norm, layers, output_attn_res, ebits, route_cfg: RouteConfig::default() };
+        // Phase N4 (`NUMA_AMX_BRIEF.md`): with `--numa` pools active, row-shard the decode-hot
+        // dense weights across the domains — see `distribute_dense`'s doc for what ships sharded
+        // and what was measured and reverted. Pure storage/placement at load time; the math per
+        // row is unchanged (see `QTSharded`).
+        if let Some(pools) = crate::numa::NodePools::get() {
+            let t = std::time::Instant::now();
+            model.distribute_dense(pools);
+            eprintln!("  --numa: sharded lm_head across {} domains in {:.1}s", pools.n(), t.elapsed().as_secs_f32());
+        }
+        Ok(model)
+    }
+
+    /// Phase N4 (`NUMA_AMX_BRIEF.md`): row-shards the decode-hot dense weights across the NUMA
+    /// domains. Storage/placement only: each row's math is byte-identical through
+    /// `matmul_qt_rows` (see `QTSharded`'s doc for the bit-identity argument).
+    ///
+    /// **What ships sharded: lm_head only.** The KDA q/k/v/o projections — N5a's 72%-of-attention
+    /// target — were built, gated, and REVERTED to plain on the measurement: at 2 cross-domain
+    /// fan-outs per KDA layer (138/token) the fan-out wake/distribution cost exceeded the
+    /// locality win on synth AND real (real kda_proj 0.346 → 0.443 s/token; `PERFORMANCE.md`,
+    /// N4). The `DenseQT` fields, the batched q/k/v fan-out in `kda_step`, and this pass are the
+    /// ready mechanism for re-enabling them the moment the "pools stay hot" follow-up removes
+    /// that per-fan-out cost — flip the four `shard_in_place` calls back on and re-gate. The
+    /// latent down/up and small dense weights stay plain for the same arithmetic (their whole
+    /// per-token cost is smaller than the fan-outs would be).
+    pub fn distribute_dense(&mut self, pools: &crate::numa::NodePools) {
+        self.lm_head.shard_in_place(pools);
     }
 
     pub fn embed_row(&self, tok: usize) -> Vec<f32> {
diff --git a/src/kimi_k3/moe.rs b/src/kimi_k3/moe.rs
index 9e02282..a695e40 100644
--- a/src/kimi_k3/moe.rs
+++ b/src/kimi_k3/moe.rs
@@ -21,20 +21,29 @@
 //! trick `moe()` uses; that's a real, deliberately deferred optimization opportunity once this is
 //! proven correct against the real checkpoint, not an oversight.
 //!
+//! Dispatch shape (Phase 5 v2 + Phase N3): the apply stage runs as (expert × row-block) tasks
+//! through the serial `matmul_qt_rows` — two fan-outs per chunk on the global pool, or one
+//! cross-node fan-out with each expert computed on its home node's pinned pool under `--numa`
+//! (`dispatch_numa`, bit-identical either way — see `ExpertJob`'s and `dispatch_numa`'s docs for
+//! the full argument and `PERFORMANCE.md` for the measurements).
+//!
 //! `shared_experts(identity)` (computed at the model's real hidden size, on the pre-down-proj
 //! hidden states) is each caller's own job, same as `modeling_kimi_linear.py`'s
 //! `KimiSparseMoeBlock.forward` adds it AFTER this wrapper's up-projected output — not folded in
 //! here, since it needs no down/up-projection at all and `glm52::moe.rs`'s existing shared-expert
 //! matmul code is exactly what would be duplicated to inline it.
 
-use crate::expert_cache::ExpertCache;
+use crate::expert_cache::{ExpertCache, ExpertSlot, GateUp};
 use crate::glm52::config::Cfg;
 use crate::glm52::model::{ModelError, MoeWeights};
 use crate::glm52::moe::{self, RouteConfig};
-use crate::kernels::matmul_qt;
+use crate::kernels::{matmul_qt, matmul_qt_rows, RowActs};
 use crate::kimi_linear::ops::rmsnorm;
 use crate::quant::QT;
 use crate::safetensors::Shards;
+use rayon::prelude::*;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::OnceLock;
 
 /// The three extra tensors K3's latent-MoE wrapper needs, on top of the ordinary `MoeWeights`
 /// (router + shared experts, unchanged) and whatever `ExpertCache` streams in per routed expert.
@@ -92,18 +101,46 @@ pub fn latent_moe(
     let mut x_latent = vec![0f32; s * moe_hidden];
     matmul_qt(&mut x_latent, x, &lw.down_proj, s);
 
-    let mut routed = vec![0f32; s * moe_hidden];
+    // `routed` is accumulated in TRANSPOSED `[moe_hidden, s]` layout for the duration of the
+    // dispatch (see `ExpertJob`'s doc for why), then transposed back to the `[s, moe_hidden]` the
+    // norm and up-projection expect.
+    let mut routed_t = vec![0f32; moe_hidden * s];
     let uniq = moe::unique_experts(&routing);
     let chunk_size = cache.capacity().max(1);
+    let threads = rayon::current_num_threads();
     for chunk in uniq.chunks(chunk_size) {
         cache.ensure_loaded(shards, cfg_expert, layer, chunk, ebits)?;
-        for &eid in chunk {
-            if let Some(slot) = cache.get(eid) {
-                moe::apply_single_expert(slot, &routing, &x_latent, moe_hidden, inter, activation, &mut routed);
+        // The chunk's resident slots, in chunk order — immutable refs, because `ensure_loaded`
+        // already stamped LRU recency in `begin_loading`, so no `&mut` borrow is needed here
+        // (contra the brief's K5 note that `get` takes `&mut self`).
+        let jobs: Vec<ExpertJob> = chunk
+            .iter()
+            .filter_map(|&eid| cache.get(eid))
+            .filter_map(|slot| ExpertJob::new(slot, &routing, &x_latent, moe_hidden))
+            .collect();
+        if jobs.is_empty() {
+            continue;
+        }
+        // Phase N3: with `--numa` active (node pools built), each expert's chain runs on its home
+        // node's pinned pool — bit-identical to the global-pool path by `dispatch_numa`'s
+        // argument. Otherwise, the two-fan-out global-pool dispatch (Phase 5 v2).
+        match crate::numa::NodePools::get() {
+            Some(pools) => dispatch_numa(pools, &jobs, layer, cfg_full.n_experts as usize, inter, moe_hidden, s, activation, &mut routed_t),
+            None => {
+                let refs: Vec<&ExpertJob> = jobs.iter().collect();
+                let ggt = gate_up_blocked(&refs, inter, activation, threads);
+                down_blocked(&mut routed_t, &jobs, &ggt, moe_hidden, s, threads);
             }
         }
     }
 
+    let mut routed = vec![0f32; s * moe_hidden];
+    for (dd, col) in routed_t.chunks(s).enumerate() {
+        for (si, &v) in col.iter().enumerate() {
+            routed[si * moe_hidden + dd] = v;
+        }
+    }
+
     if let Some(norm_w) = &lw.norm {
         for row in routed.chunks_mut(moe_hidden) {
             rmsnorm(row, norm_w, eps);
@@ -115,6 +152,288 @@ pub fn latent_moe(
     Ok(())
 }
 
+/// Row-block tasks each fan-out below aims to produce per pool thread. Enough that work-stealing
+/// can even out a ragged tail, few enough that each task's own overhead (a heap scratch buffer
+/// plus a `matmul_qt_rows` dispatch) stays negligible against the thousands of MACs it then does.
+const TASKS_PER_THREAD: usize = 4;
+
+/// Rows per task when `total_rows` independent output rows are to be spread across the pool.
+fn block_rows(total_rows: usize, threads: usize) -> usize {
+    total_rows.div_ceil((threads * TASKS_PER_THREAD).max(1)).max(1)
+}
+
+/// One resident expert's share of a dispatch chunk: the token rows routed to it, and those rows'
+/// activations gathered contiguously — exactly the `rows`/`xg` pair `apply_single_expert` builds.
+///
+/// **Phase 5 v2** (`K3_OPTIMIZE_BRIEF.md`): v1 ran one task per expert and let each task's three
+/// `matmul_qt` calls fan out internally. That left the inner fork in place — at batch-1 decode
+/// every one of those matmuls splits ~3000 single-element tasks across the pool, so the pool
+/// spends its time scheduling rather than computing (measured: decode got *slower* the more
+/// threads it was given, `PERFORMANCE.md`). v2 inverts it. The chunk's work is flattened into
+/// (expert × row-block) tasks and dispatched as exactly **two** fan-outs per chunk — gate/up +
+/// activation, then down + accumulate — with each task calling the serial `matmul_qt_rows` on its
+/// own row range. No nested rayon at all, and every task does real vector work.
+///
+/// Both stages write **transposed** (`[rows, s]`, `matmul_qt`'s own internal `yt` layout) rather
+/// than `[s, rows]`: that is what makes one task's row block a contiguous, independently
+/// borrowable slice, so the split is `par_chunks_mut` in safe Rust with no aliasing tricks. The
+/// two transposes back (`[inter, nr]` → `[nr, inter]` per expert, `[moe_hidden, s]` → `[s,
+/// moe_hidden]` once per layer) are O(rows) against O(rows × cols) of matmul — noise, and the
+/// same trade `transpose_so` already makes inside every kernel.
+struct ExpertJob<'a> {
+    slot: &'a ExpertSlot,
+    /// `(sequence position, routing weight)`, ascending — `moe::expert_rows`' list verbatim, so
+    /// the scatter below reproduces `apply_single_expert`'s exactly.
+    rows: Vec<(usize, f32)>,
+    /// `[nr, moe_hidden]` — this expert's token rows gathered out of the down-projected input.
+    xg: Vec<f32>,
+}
+
+impl<'a> ExpertJob<'a> {
+    /// `None` for an expert no token routed to — the same early-out `apply_single_expert` takes,
+    /// keeping such experts out of the accumulation entirely rather than adding zeros.
+    fn new(slot: &'a ExpertSlot, routing: &moe::Routing, x_latent: &[f32], d: usize) -> Option<ExpertJob<'a>> {
+        let rows = moe::expert_rows(routing, slot.eid);
+        if rows.is_empty() {
+            return None;
+        }
+        let mut xg = vec![0f32; rows.len() * d];
+        for (r, &(si, _)) in rows.iter().enumerate() {
+            xg[r * d..(r + 1) * d].copy_from_slice(&x_latent[si * d..(si + 1) * d]);
+        }
+        Some(ExpertJob { slot, rows, xg })
+    }
+
+    fn nr(&self) -> usize {
+        self.rows.len()
+    }
+}
+
+/// A job's gate/up weights paired with activations prepared for them once (`RowActs::prepare`
+/// hoists the int8-IDOT tiers' per-call quantization out of the per-block calls — see its doc).
+/// Mirrors `GateUp` so the fan-out below matches on this alone and can't reach an impossible
+/// weights/activations pairing.
+enum GateUpActs<'a> {
+    Separate { gate: &'a QT, up: &'a QT, xg_gate: RowActs<'a>, xg_up: RowActs<'a> },
+    Fused { gate_up: &'a QT, xg: RowActs<'a> },
+}
+
+impl<'j> GateUpActs<'j> {
+    fn prepare(job: &'j ExpertJob<'_>) -> GateUpActs<'j> {
+        let nr = job.nr();
+        match &job.slot.gate_up {
+            GateUp::Separate { gate, up } => GateUpActs::Separate {
+                gate,
+                up,
+                xg_gate: RowActs::prepare(&job.xg, gate, nr),
+                xg_up: RowActs::prepare(&job.xg, up, nr),
+            },
+            GateUp::Fused { gate_up } => GateUpActs::Fused { gate_up, xg: RowActs::prepare(&job.xg, gate_up, nr) },
+        }
+    }
+}
+
+/// One (expert × row-block) task of stage A: rows `[r0, r0 + seg.len()/nr)` of one job's gate
+/// and up matmuls plus the activation, written transposed (`[n, nr]`) into `seg`. Factored out
+/// of [`gate_up_blocked`] so the NUMA per-node dispatch ([`dispatch_numa`]) runs the byte-exact
+/// same task body — two call sites, one arithmetic path, no way to drift.
+fn gate_up_block(acts: &GateUpActs, nr: usize, inter: usize, activation: moe::Activation, r0: usize, seg: &mut [f32]) {
+    let n = seg.len() / nr;
+    match acts {
+        GateUpActs::Separate { gate, up, xg_gate, xg_up } => {
+            matmul_qt_rows(seg, xg_gate, gate, r0, n);
+            let mut uu = vec![0f32; n * nr];
+            matmul_qt_rows(&mut uu, xg_up, up, r0, n);
+            activation.apply(seg, &uu);
+        }
+        // The fused weight's row `k` is gate's, row `inter + k` is up's — the same split
+        // `apply_single_expert`'s `Fused` arm makes within one wide output row.
+        GateUpActs::Fused { gate_up, xg } => {
+            let mut g = vec![0f32; n * nr];
+            let mut u = vec![0f32; n * nr];
+            matmul_qt_rows(&mut g, xg, gate_up, r0, n);
+            matmul_qt_rows(&mut u, xg, gate_up, inter + r0, n);
+            for (o, (&gv, &uv)) in seg.iter_mut().zip(g.iter().zip(&u)) {
+                *o = activation.combine(gv, uv);
+            }
+        }
+    }
+}
+
+/// `[rows, nr]`-transposed stage output → the `[nr, rows]` layout `apply_single_expert`'s `gg`
+/// buffer has (see `ExpertJob`'s doc for why the stages compute transposed in the first place).
+fn untranspose(t: &[f32], nr: usize) -> Vec<f32> {
+    let rows = t.len() / nr.max(1);
+    let mut out = vec![0f32; t.len()];
+    for (k, col) in t.chunks(nr).enumerate() {
+        for (r, &v) in col.iter().enumerate() {
+            out[r * rows + k] = v;
+        }
+    }
+    out
+}
+
+/// Stage A: every job's gate and up matmuls plus the activation, as ONE fan-out over
+/// (expert × `inter` row-block) tasks. Returns each expert's `[nr, inter]` activated intermediate
+/// — the buffer `apply_single_expert` calls `gg`, in the same layout, element for element.
+///
+/// Takes `&[&ExpertJob]` rather than `&[ExpertJob]` so [`dispatch_numa`] can hand it one home
+/// domain's (non-contiguous) subset of the chunk's jobs and run the SAME flattened fan-out
+/// inside that domain's pool — per-expert-sequential dispatch inside a wide pool was measured
+/// re-creating the exact micro-task pathology v2 exists to kill (`PERFORMANCE.md`, D1 flip).
+fn gate_up_blocked(jobs: &[&ExpertJob], inter: usize, activation: moe::Activation, threads: usize) -> Vec<Vec<f32>> {
+    let acts: Vec<GateUpActs> = jobs.iter().map(|j| GateUpActs::prepare(j)).collect();
+    let mut ggt: Vec<Vec<f32>> = jobs.iter().map(|j| vec![0f32; j.nr() * inter]).collect();
+    let blk = block_rows(inter * jobs.len(), threads);
+    jobs.par_iter()
+        .zip(acts.par_iter())
+        .zip(ggt.par_iter_mut())
+        .flat_map(|((job, acts), buf)| {
+            let nr = job.nr();
+            buf.par_chunks_mut(blk * nr).enumerate().map(move |(b, seg)| (acts, nr, b * blk, seg))
+        })
+        .for_each(|(acts, nr, r0, seg)| gate_up_block(acts, nr, inter, activation, r0, seg));
+    jobs.iter().zip(&ggt).map(|(job, t)| untranspose(t, job.nr())).collect()
+}
+
+/// The down-projections of every job's `gg`, as ONE flattened fan-out over (expert ×
+/// `moe_hidden` row-block) tasks, each expert's result in its own TRANSPOSED `[moe_hidden, nr]`
+/// buffer. Used by [`dispatch_numa`], which scatters these into `routed_t` sequentially
+/// afterward; the `--numa`-off path keeps [`down_blocked`] instead, which fuses the down matmul
+/// and the accumulation into one fan-out (it can, because it owns ALL experts — a single domain
+/// owning a subset cannot accumulate shared output rows without racing other domains).
+fn down_transposed(jobs: &[&ExpertJob], ggs: &[Vec<f32>], moe_hidden: usize, threads: usize) -> Vec<Vec<f32>> {
+    let acts: Vec<RowActs> = jobs.iter().zip(ggs).map(|(j, gg)| RowActs::prepare(gg, &j.slot.down, j.nr())).collect();
+    let mut hhs: Vec<Vec<f32>> = jobs.iter().map(|j| vec![0f32; moe_hidden * j.nr()]).collect();
+    let blk = block_rows(moe_hidden * jobs.len(), threads);
+    jobs.par_iter()
+        .zip(acts.par_iter())
+        .zip(hhs.par_iter_mut())
+        .flat_map(|((job, acts), buf)| {
+            let nr = job.nr();
+            buf.par_chunks_mut(blk * nr).enumerate().map(move |(b, seg)| (job, acts, nr, b * blk, seg))
+        })
+        .for_each(|(job, acts, nr, r0, seg)| matmul_qt_rows(seg, acts, &job.slot.down, r0, seg.len() / nr));
+    hhs
+}
+
+/// Stage B: the down-projection and its routing-weighted accumulation into `routed_t`
+/// (`[moe_hidden, s]`), as ONE fan-out over `moe_hidden` row-blocks.
+///
+/// Note which axis is parallel: **rows, not experts**. Each task walks every job in chunk order
+/// for its own row range, so each output element accumulates the experts' contributions in the
+/// order the sequential loop applied them — no per-expert partial buffers and no reduction pass,
+/// and the float addition order is unchanged, which is what makes this bit-identical.
+fn down_blocked(routed_t: &mut [f32], jobs: &[ExpertJob], ggs: &[Vec<f32>], moe_hidden: usize, s: usize, threads: usize) {
+    let acts: Vec<RowActs> = jobs.iter().zip(ggs).map(|(j, gg)| RowActs::prepare(gg, &j.slot.down, j.nr())).collect();
+    let nr_max = jobs.iter().map(ExpertJob::nr).max().unwrap_or(0);
+    let blk = block_rows(moe_hidden, threads);
+    routed_t.par_chunks_mut(blk * s).enumerate().for_each(|(b, seg)| {
+        let r0 = b * blk;
+        let n = seg.len() / s;
+        let mut hh = vec![0f32; n * nr_max];
+        for (job, acts) in jobs.iter().zip(&acts) {
+            let nr = job.nr();
+            matmul_qt_rows(&mut hh[..n * nr], acts, &job.slot.down, r0, n);
+            for (r, &(si, wgt)) in job.rows.iter().enumerate() {
+                for j in 0..n {
+                    seg[j * s + si] += wgt * hh[j * nr + r];
+                }
+            }
+        }
+    });
+}
+
+/// Phase N3c (`NUMA_AMX_BRIEF.md`): the same two stages as [`gate_up_blocked`]/[`down_blocked`],
+/// but each home domain's pinned pool runs them over ITS OWN experts — the ones whose weight
+/// pages `expert_cache`'s `numa_homed_load` first-touched onto that domain (`numa::home_node` is
+/// the single agreement point). One [`NodePools::run_all`] fan-out per chunk; domains proceed
+/// independently (a domain's stage B needs only its own experts' stage-A output), no cross-domain
+/// barrier between stages, no work stealing across domains (brief §2: routing skew is per-token
+/// noise — log it via [`numa_moe_stats`], don't fight it).
+///
+/// Within a domain the work is FLATTENED across its experts — the same
+/// [`gate_up_blocked`]/[`down_transposed`] fan-outs the global path uses, just scoped to the
+/// domain's job subset and its pool width. The first build of this function ran the domain's
+/// experts sequentially (each expert's row-blocks split across the whole pool), which re-created
+/// v2's micro-task pathology the moment D1 widened the pools to per-socket: `block_rows(3072,
+/// 192×4)` is a 4-row task. Measured as a straight regression and rebuilt this way (see
+/// `PERFORMANCE.md`, D1 flip).
+///
+/// **Bit-identity vs the `--numa`-off path** (the N3 acceptance gate): each row's dot runs the
+/// same `matmul_qt_rows` kernel over the same elements ([`gate_up_block`] is shared code;
+/// [`down_transposed`] computes the same per-element values [`down_blocked`] does, row-blocking
+/// never changes a row's own accumulation), and the routing-weighted accumulation into
+/// `routed_t` happens HERE, sequentially, walking jobs in chunk order — the same per-element
+/// addition order `down_blocked` produces by walking jobs in order inside each row-block task.
+/// Only WHICH thread computes each value and WHEN the accumulation happens differ; no float
+/// operation is reordered.
+#[allow(clippy::too_many_arguments)]
+fn dispatch_numa(
+    pools: &crate::numa::NodePools,
+    jobs: &[ExpertJob],
+    layer: usize,
+    n_experts: usize,
+    inter: usize,
+    moe_hidden: usize,
+    s: usize,
+    activation: moe::Activation,
+    routed_t: &mut [f32],
+) {
+    let tpp = pools.threads_per_pool();
+    let mut by_domain: Vec<Vec<usize>> = vec![Vec::new(); pools.n()];
+    for (k, job) in jobs.iter().enumerate() {
+        by_domain[crate::numa::home_node(layer, n_experts, job.slot.eid, pools.n())].push(k);
+    }
+    let mut hhs: Vec<Vec<f32>> = vec![Vec::new(); jobs.len()];
+    {
+        // One slot per job, each written by exactly one domain — the mutexes are uncontended and
+        // exist only to hand `&mut` slots across the `Fn` closure boundary in safe Rust.
+        let slots: Vec<std::sync::Mutex<&mut Vec<f32>>> = hhs.iter_mut().map(std::sync::Mutex::new).collect();
+        let stats = numa_stats(pools.n());
+        pools.run_all(|i| {
+            if by_domain[i].is_empty() {
+                return;
+            }
+            let t = std::time::Instant::now();
+            let mine: Vec<&ExpertJob> = by_domain[i].iter().map(|&k| &jobs[k]).collect();
+            let ggs = gate_up_blocked(&mine, inter, activation, tpp);
+            for (&k, hh) in by_domain[i].iter().zip(down_transposed(&mine, &ggs, moe_hidden, tpp)) {
+                **slots[k].lock().unwrap() = hh;
+            }
+            stats[i].0.fetch_add(t.elapsed().as_nanos() as u64, Ordering::Relaxed);
+            stats[i].1.fetch_add(by_domain[i].len() as u64, Ordering::Relaxed);
+        });
+    }
+    for (job, hh_t) in jobs.iter().zip(&hhs) {
+        let nr = job.nr();
+        for (j, col) in hh_t.chunks(nr).enumerate() {
+            for (r, &(si, wgt)) in job.rows.iter().enumerate() {
+                routed_t[j * s + si] += wgt * col[r];
+            }
+        }
+    }
+}
+
+/// Cumulative per-node MoE dispatch counters since process start (Phase N3d): `(busy_nanos,
+/// experts_dispatched)` per node, updated by [`dispatch_numa`], monotonic. This is the data D1
+/// (pool granularity) and the no-cross-node-rebalancing decision are judged on — busy-time
+/// imbalance across nodes IS the measured cost of routing skew.
+static NUMA_MOE_STATS: OnceLock<Vec<(AtomicU64, AtomicU64)>> = OnceLock::new();
+
+fn numa_stats(n_nodes: usize) -> &'static [(AtomicU64, AtomicU64)] {
+    NUMA_MOE_STATS.get_or_init(|| (0..n_nodes).map(|_| (AtomicU64::new(0), AtomicU64::new(0))).collect())
+}
+
+/// Snapshot of [`NUMA_MOE_STATS`] as `(busy_seconds, experts_dispatched)` per node — `None`
+/// until the first NUMA-dispatched MoE layer has run (which also means: always `None` with
+/// `--numa` off). Served by `GET /profile`; totals since process start, so a dashboard diffs
+/// consecutive polls for rates.
+pub fn numa_moe_stats() -> Option<Vec<(f64, u64)>> {
+    let stats = NUMA_MOE_STATS.get()?;
+    Some(stats.iter().map(|(ns, n)| (ns.load(Ordering::Relaxed) as f64 / 1e9, n.load(Ordering::Relaxed))).collect())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -276,6 +595,163 @@ mod tests {
         }
     }
 
+    /// Phase 5 v2's acceptance gate: the row-blocked dispatch is **bit-identical** to applying
+    /// the experts one whole expert at a time through `apply_single_expert` — which is both what
+    /// this function did before v2 and what `glm52::moe` still does.
+    ///
+    /// Not a tolerance comparison. v2 changes only WHICH thread computes which output rows, never
+    /// the arithmetic: each row's dot runs the same kernel over the same elements in the same
+    /// order (`matmul_qt_rows`' own contract, pinned in `kernels.rs`), and each output element
+    /// still accumulates its experts in chunk order because stage B parallelizes over rows rather
+    /// than experts. `s > 1` and `topk > 1` so experts genuinely share token rows — with `s = 1`
+    /// the transposed layouts would be trivially identical and the test would prove much less.
+    #[test]
+    fn latent_moe_row_blocked_dispatch_is_bit_identical_to_applying_experts_one_at_a_time() {
+        let (hidden, moe_hidden, inter, n_experts, topk, n_shared) = (6usize, 4usize, 3usize, 5usize, 3i32, 1i32);
+        let mut seed = 909u32;
+        let dir = TempDir::new("rabbit_test_k3_latent_moe_row_blocked");
+        let shards = build_expert_fixture(&dir.0, n_experts, inter, moe_hidden, &mut seed);
+        let cfg_full = tiny_cfg(n_experts as i32, topk, inter as i32, hidden as i32, n_shared);
+        let cfg_expert = tiny_cfg(n_experts as i32, topk, inter as i32, moe_hidden as i32, n_shared);
+        let w = MoeWeights {
+            router: random_vec(n_experts * hidden, &mut seed),
+            router_bias: random_vec(n_experts, &mut seed),
+            sh_gate: random_qt_f32(inter, hidden, &mut seed),
+            sh_up: random_qt_f32(inter, hidden, &mut seed),
+            sh_down: random_qt_f32(hidden, inter, &mut seed),
+        };
+        let lw = LatentMoeWeights {
+            down_proj: random_qt_f32(moe_hidden, hidden, &mut seed),
+            up_proj: random_qt_f32(hidden, moe_hidden, &mut seed),
+            norm: None,
+        };
+        let s = 4;
+        let x = random_vec(s * hidden, &mut seed);
+
+        let mut cache = ExpertCache::new(n_experts);
+        let mut out = vec![0f32; s * hidden];
+        latent_moe(&cfg_full, &cfg_expert, &w, &lw, &mut cache, &shards, 0, 32, &RouteConfig::default(), &x, s, 1e-5, moe::Activation::Silu, &mut out).unwrap();
+
+        // The pre-v2 shape, verbatim: one `apply_single_expert` per expert, accumulating into a
+        // single `[s, moe_hidden]` buffer in expert-id (= chunk) order.
+        let routing: Routing = moe::route(&cfg_full, &w, &x, s);
+        let mut x_latent = vec![0f32; s * moe_hidden];
+        matmul_qt(&mut x_latent, &x, &lw.down_proj, s);
+        let mut routed_ref = vec![0f32; s * moe_hidden];
+        let mut ref_cache = ExpertCache::new(n_experts);
+        for eid in moe::unique_experts(&routing) {
+            ref_cache.ensure_loaded(&shards, &cfg_expert, 0, &[eid], 32).unwrap();
+            let slot = ref_cache.get(eid).unwrap();
+            moe::apply_single_expert(slot, &routing, &x_latent, moe_hidden, inter, moe::Activation::Silu, &mut routed_ref);
+        }
+        let mut expected = vec![0f32; s * hidden];
+        matmul_qt(&mut expected, &routed_ref, &lw.up_proj, s);
+
+        let got: Vec<u32> = out.iter().map(|v| v.to_bits()).collect();
+        let want: Vec<u32> = expected.iter().map(|v| v.to_bits()).collect();
+        assert_eq!(got, want, "row-blocked dispatch must be bit-identical to the per-expert loop");
+    }
+
+    /// Phase N3's acceptance gate at the unit level: `dispatch_numa` (each expert's chain on its
+    /// home node's pinned pool, sequential chunk-order accumulation afterward) produces
+    /// **bit-identical** output to the global-pool two-fan-out dispatch, on the same jobs. Runs
+    /// against the machine's real topology via throwaway `NodePools::build` pools (NOT the
+    /// process singleton — that would flip every concurrent test onto the NUMA paths); SKIPs on
+    /// single-node machines, where there is nothing to pin to.
+    ///
+    /// `n_experts = 5` against 6 pools on the target box also exercises nodes with no work; the
+    /// `s = 4`, `topk = 3` shape makes experts share token rows, same reasoning as the v2 test.
+    #[test]
+    fn numa_dispatch_is_bit_identical_to_the_global_pool_dispatch() {
+        let Some(topo) = crate::numa::topology() else {
+            eprintln!("SKIP: no NUMA topology on this machine");
+            return;
+        };
+        let Some(pools) = crate::numa::NodePools::build(topo.nodes, 12) else {
+            eprintln!("SKIP: single NUMA node — nothing to pin to");
+            return;
+        };
+        let (hidden, moe_hidden, inter, n_experts, topk, n_shared) = (6usize, 4usize, 3usize, 5usize, 3i32, 1i32);
+        let mut seed = 4242u32;
+        let dir = TempDir::new("rabbit_test_k3_numa_dispatch");
+        let shards = build_expert_fixture(&dir.0, n_experts, inter, moe_hidden, &mut seed);
+        let cfg_full = tiny_cfg(n_experts as i32, topk, inter as i32, hidden as i32, n_shared);
+        let cfg_expert = tiny_cfg(n_experts as i32, topk, inter as i32, moe_hidden as i32, n_shared);
+        let w = MoeWeights {
+            router: random_vec(n_experts * hidden, &mut seed),
+            router_bias: random_vec(n_experts, &mut seed),
+            sh_gate: random_qt_f32(inter, hidden, &mut seed),
+            sh_up: random_qt_f32(inter, hidden, &mut seed),
+            sh_down: random_qt_f32(hidden, inter, &mut seed),
+        };
+        let down_proj = random_qt_f32(moe_hidden, hidden, &mut seed);
+        let s = 4;
+        let x = random_vec(s * hidden, &mut seed);
+
+        let routing: Routing = moe::route(&cfg_full, &w, &x, s);
+        let mut x_latent = vec![0f32; s * moe_hidden];
+        matmul_qt(&mut x_latent, &x, &down_proj, s);
+        let mut cache = ExpertCache::new(n_experts);
+        cache.ensure_loaded(&shards, &cfg_expert, 0, &(0..n_experts).collect::<Vec<_>>(), 32).unwrap();
+        let jobs: Vec<ExpertJob> = (0..n_experts)
+            .filter_map(|eid| cache.get(eid))
+            .filter_map(|slot| ExpertJob::new(slot, &routing, &x_latent, moe_hidden))
+            .collect();
+        assert!(!jobs.is_empty());
+
+        let threads = rayon::current_num_threads();
+        let mut routed_global = vec![0f32; moe_hidden * s];
+        let refs: Vec<&ExpertJob> = jobs.iter().collect();
+        let ggt = gate_up_blocked(&refs, inter, moe::Activation::Silu, threads);
+        down_blocked(&mut routed_global, &jobs, &ggt, moe_hidden, s, threads);
+
+        let mut routed_numa = vec![0f32; moe_hidden * s];
+        dispatch_numa(&pools, &jobs, 0, n_experts, inter, moe_hidden, s, moe::Activation::Silu, &mut routed_numa);
+
+        let a: Vec<u32> = routed_global.iter().map(|v| v.to_bits()).collect();
+        let b: Vec<u32> = routed_numa.iter().map(|v| v.to_bits()).collect();
+        assert_eq!(a, b, "NUMA dispatch must be bit-identical to the global-pool dispatch");
+    }
+
+    /// Phase 5 determinism gate: two identical warm runs of `latent_moe` produce **bit-identical**
+    /// output. Both fan-outs write each output element from exactly one task and accumulate the
+    /// experts in fixed chunk order (not nondeterministic completion order), so there is no
+    /// run-to-run variation — which is what makes a bit-identical acceptance gate valid at all
+    /// for a parallelized path.
+    #[test]
+    fn latent_moe_is_bit_identical_across_two_warm_runs() {
+        let (hidden, moe_hidden, inter, n_experts, topk, n_shared) = (6usize, 4usize, 3usize, 4usize, 2i32, 1i32);
+        let mut seed = 77u32;
+        let dir = TempDir::new("rabbit_test_k3_latent_moe_determinism");
+        let shards = build_expert_fixture(&dir.0, n_experts, inter, moe_hidden, &mut seed);
+        let cfg_full = tiny_cfg(n_experts as i32, topk, inter as i32, hidden as i32, n_shared);
+        let cfg_expert = tiny_cfg(n_experts as i32, topk, inter as i32, moe_hidden as i32, n_shared);
+        let w = MoeWeights {
+            router: random_vec(n_experts * hidden, &mut seed),
+            router_bias: random_vec(n_experts, &mut seed),
+            sh_gate: random_qt_f32(inter, hidden, &mut seed),
+            sh_up: random_qt_f32(inter, hidden, &mut seed),
+            sh_down: random_qt_f32(hidden, inter, &mut seed),
+        };
+        let lw = LatentMoeWeights {
+            down_proj: random_qt_f32(moe_hidden, hidden, &mut seed),
+            up_proj: random_qt_f32(hidden, moe_hidden, &mut seed),
+            norm: None,
+        };
+        let s = 4;
+        let x = random_vec(s * hidden, &mut seed);
+
+        let run = || {
+            let mut cache = ExpertCache::new(n_experts);
+            let mut out = vec![0f32; s * hidden];
+            latent_moe(&cfg_full, &cfg_expert, &w, &lw, &mut cache, &shards, 0, 32, &RouteConfig::default(), &x, s, 1e-5, moe::Activation::Silu, &mut out).unwrap();
+            out
+        };
+        let a: Vec<u32> = run().iter().map(|v| v.to_bits()).collect();
+        let b: Vec<u32> = run().iter().map(|v| v.to_bits()).collect();
+        assert_eq!(a, b, "latent_moe must be bit-identical across identical warm runs");
+    }
+
     /// Same setup as the no-norm test, but with `norm: Some(..)` — proves the norm is genuinely
     /// applied to the SUMMED per-token routed output (post weighted-sum-over-experts) before
     /// `up_proj`, rather than, say, silently skipped or applied per-expert (RMSNorm isn't linear,
diff --git a/src/kimi_linear/generate.rs b/src/kimi_linear/generate.rs
index a1281c5..d4277c2 100644
--- a/src/kimi_linear/generate.rs
+++ b/src/kimi_linear/generate.rs
@@ -136,7 +136,7 @@ impl ExpertCaches {
         let v = model
             .layers
             .iter()
-            .map(|l| if matches!(l.ffn, Ffn::Moe(_)) { Some(ExpertCache::for_family(capacity, ExpertNaming::KimiLinear)) } else { None })
+            .map(|l| if matches!(l.ffn, Ffn::Moe(_)) { Some(ExpertCache::for_family(capacity, model.cfg.n_experts as usize, ExpertNaming::KimiLinear)) } else { None })
             .collect();
         ExpertCaches(v)
     }
@@ -153,6 +153,22 @@ impl ExpertCaches {
         self.0.iter().flatten().map(|c| c.io_wait_nanos).sum()
     }
 
+    /// Whether any layer's cache loads through an `io_uring` ring. `false` (every MXFP4/K3 run,
+    /// whose naming never gets a ring) means `io_wait_nanos_total` is structurally zero, so the
+    /// CLI must not present it as "actual disk wait" (Phase 4c) — see `ExpertCache::has_ring`.
+    pub fn any_has_ring(&self) -> bool {
+        self.0.iter().flatten().any(|c| c.has_ring())
+    }
+
+    /// Phase 4b: preload every MoE layer's routed experts up front (see
+    /// `expert_cache::preload_layers`). `to_glm_cfg` supplies the expert shapes, same as the
+    /// dispatch path's `moe::moe` call.
+    pub fn preload(&mut self, model: &Model, shards: &Shards) -> Result<(), ModelError> {
+        let cfg = crate::kimi_linear::model::to_glm_cfg(&model.cfg);
+        let n_experts = cfg.n_experts as usize;
+        Ok(crate::expert_cache::preload_layers(&mut self.0, shards, &cfg, model.ebits, n_experts)?)
+    }
+
     /// Seeds usage counters from `<model_dir>/.rabbit_usage` and marks pin candidates once
     /// confidence crosses the threshold — see `crate::generate::ExpertCaches::warm_start`'s doc
     /// (identical logic, ported verbatim).
diff --git a/src/kimi_linear/model.rs b/src/kimi_linear/model.rs
index 64de0f7..bc24b6b 100644
--- a/src/kimi_linear/model.rs
+++ b/src/kimi_linear/model.rs
@@ -228,6 +228,8 @@ impl Model {
         let final_norm = ld(&shards, "model.norm.weight")?;
 
         let mut layers = Vec::with_capacity(cfg.n_layers as usize);
+        let t_layers = std::time::Instant::now();
+        let stderr_tty = std::io::IsTerminal::is_terminal(&std::io::stderr());
         for i in 0..cfg.n_layers as usize {
             let p = |s: &str| format!("model.layers.{i}.{s}");
 
@@ -286,6 +288,17 @@ impl Model {
             };
 
             layers.push(Layer { in_ln, post_ln, attn, ffn });
+            // Per-layer progress (see kimi_k3::model's version): rewriting line in a TTY,
+            // newline-delimited when redirected so a log/monitor sees each layer.
+            if stderr_tty {
+                eprint!("\r  loaded layer {}/{} ({:.0}s)", i + 1, cfg.n_layers, t_layers.elapsed().as_secs_f32());
+                let _ = std::io::Write::flush(&mut std::io::stderr());
+            } else {
+                eprintln!("  loaded layer {}/{} ({:.0}s)", i + 1, cfg.n_layers, t_layers.elapsed().as_secs_f32());
+            }
+        }
+        if stderr_tty {
+            eprintln!();
         }
 
         Ok(Model { cfg, embed, lm_head, final_norm, layers, ebits, route_cfg: RouteConfig::default() })
diff --git a/src/lib.rs b/src/lib.rs
index ba31d94..f10c118 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,6 +10,7 @@ pub mod kimi_k3;
 pub mod kimi_linear;
 pub mod kv_session;
 pub mod model;
+pub mod numa;
 pub mod quant;
 pub mod safetensors;
 pub mod server;
diff --git a/src/main.rs b/src/main.rs
index 530cd91..ac49e58 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -12,7 +12,7 @@ use std::process::ExitCode;
 
 const USAGE: &str = "usage: rabbit --model <dir> (--prompt <text> | --chat | --serve) [--max-tokens N] \
 [--temperature F] [--nucleus F] [--seed N] [--dbits N] [--ebits N] [--expert-cache N] [--think] \
-[--session <path>] [--no-usage-cache] [--cache-route] [--threads N] [--host H] [--port N] [--api-key K] \
+[--session <path>] [--no-usage-cache] [--cache-route] [--preload-experts] [--threads N] [--numa] [--numa-threads N] [--host H] [--port N] [--api-key K] \
 [--shard-dirs <dir1,dir2,...>]";
 
 struct Args {
@@ -34,9 +34,22 @@ struct Args {
     session: Option<PathBuf>,
     no_usage_cache: bool,
     cache_route: bool,
+    /// `--preload-experts`: after warm-start, load every MoE layer's experts up front (Phase 4b)
+    /// so token 1 runs at steady state. Applies to `--prompt`/`--chat`/`--serve` alike.
+    preload_experts: bool,
     /// `None` = use the default (physical core count — see `main`'s call to
     /// `rayon::ThreadPoolBuilder`). `--threads` overrides it explicitly.
     threads: Option<usize>,
+    /// `--numa`: NUMA-affine execution (`NUMA_AMX_BRIEF.md`) — pinned per-node worker pools,
+    /// expert weights placed on per-expert home nodes, expert compute dispatched to where its
+    /// weights live. Default off; a no-op on single-node machines and non-Linux (the flag parses
+    /// everywhere so command lines stay portable, and degrades to current behavior).
+    numa: bool,
+    /// `--numa-threads`: total pinned threads across the node pools (default: the effective
+    /// `--threads`). Decoupled because the two pools want DIFFERENT widths (measured, synth):
+    /// attention/dense on the global pool degrades past ~96 threads, while the expert phase
+    /// keeps scaling on pinned pools — e.g. `--threads 48 --numa --numa-threads 384`.
+    numa_threads: Option<usize>,
     host: String,
     port: u16,
     api_key: Option<String>,
@@ -59,7 +72,10 @@ fn parse_args() -> Result<Args, String> {
     let mut session = None;
     let mut no_usage_cache = false;
     let mut cache_route = false;
+    let mut preload_experts = false;
     let mut threads = None;
+    let mut numa = false;
+    let mut numa_threads = None;
     let mut host = "127.0.0.1".to_string();
     let mut port = 8000u16;
     let mut api_key = None;
@@ -84,7 +100,10 @@ fn parse_args() -> Result<Args, String> {
             "--session" => session = Some(PathBuf::from(next("--session")?)),
             "--no-usage-cache" => no_usage_cache = true,
             "--cache-route" => cache_route = true,
+            "--preload-experts" => preload_experts = true,
             "--threads" => threads = Some(next("--threads")?.parse().map_err(|e| format!("--threads: {e}"))?),
+            "--numa" => numa = true,
+            "--numa-threads" => numa_threads = Some(next("--numa-threads")?.parse().map_err(|e| format!("--numa-threads: {e}"))?),
             "--host" => host = next("--host")?,
             "--port" => port = next("--port")?.parse().map_err(|e| format!("--port: {e}"))?,
             "--api-key" => api_key = Some(next("--api-key")?),
@@ -117,7 +136,10 @@ fn parse_args() -> Result<Args, String> {
         session,
         no_usage_cache,
         cache_route,
+        preload_experts,
         threads,
+        numa,
+        numa_threads,
         host,
         port,
         api_key,
@@ -137,23 +159,38 @@ fn load_args(args: &Args) -> LoadArgs {
         cache_capacity: args.cache_capacity,
         no_usage_cache: args.no_usage_cache,
         cache_route: args.cache_route,
+        preload_experts: args.preload_experts,
     }
 }
 
 /// Prints a `GenEvent` to stderr exactly as `main.rs` always has — a real-model forward can
 /// take seconds to tens of seconds per step, so silent output would be indistinguishable from
 /// a hang.
-fn print_progress(ev: GenEvent) {
+/// `show_disk_wait` is `caches.any_has_ring()`: without a ring (MXFP4/K3) `io_wait_seconds` is
+/// structurally always 0.0 and means nothing, so the `[... actual disk wait]` bracket is replaced
+/// with `[disk wait n/a]` rather than printed as if the disk were instant (Phase 4c — the 0.0
+/// readout misled the owner in the first minutes of the first real K3 run). `io_seconds`
+/// (`load_nanos`, the real sequential/parallel load time) stays shown either way.
+fn print_progress(ev: GenEvent, show_disk_wait: bool) {
+    let wait = |io_wait_seconds: f32| {
+        if show_disk_wait {
+            format!("[{io_wait_seconds:.1}s actual disk wait]")
+        } else {
+            "[disk wait n/a]".to_string()
+        }
+    };
     match ev {
         GenEvent::Prefill { tokens, seconds, hits, misses, io_seconds, io_wait_seconds } => {
             eprintln!("prefill ({tokens} tokens)...");
             eprintln!(
-                "  prefill done in {seconds:.1}s (expert cache: {hits} hits, {misses} misses, {io_seconds:.1}s in disk I/O [{io_wait_seconds:.1}s actual disk wait])"
+                "  prefill done in {seconds:.1}s (expert cache: {hits} hits, {misses} misses, {io_seconds:.1}s in disk I/O {})",
+                wait(io_wait_seconds)
             );
         }
         GenEvent::Token { index, max, seconds, hits, misses, io_seconds, io_wait_seconds, .. } => {
             eprintln!(
-                "  token {index}/{max} in {seconds:.1}s ({io_seconds:.1}s in disk I/O [{io_wait_seconds:.1}s actual disk wait] this step; expert cache totals: {hits} hits, {misses} misses)"
+                "  token {index}/{max} in {seconds:.1}s ({io_seconds:.1}s in disk I/O {} this step; expert cache totals: {hits} hits, {misses} misses)",
+                wait(io_wait_seconds)
             );
         }
     }
@@ -166,7 +203,11 @@ fn run_single_shot(args: &Args, prompt: &str) -> Result<(), Box<dyn std::error::
 
     let mut kv = KvState::new(&sess.model);
     let t1 = std::time::Instant::now();
-    let (text, _pos, n, _profile) = chat::generate_reply(&mut sess, &mut kv, &prompt_ids, 0, print_progress)?;
+    let show_disk_wait = sess.caches.any_has_ring();
+    // Passthrough, not `output_filter`: `prompt` was encoded raw above (no `render_turn`), so
+    // there is no chat envelope open for the model to close and nothing to strip.
+    let (text, _pos, n, _profile) =
+        chat::generate_reply(&mut sess, &mut kv, &prompt_ids, 0, chat::OutputFilter::Passthrough, |ev| print_progress(ev, show_disk_wait))?;
     let elapsed = t1.elapsed().as_secs_f32();
     println!("{text}");
     eprintln!("\n{n} tokens in {elapsed:.1}s ({:.1} tok/s)", n as f32 / elapsed.max(0.001));
@@ -238,7 +279,9 @@ fn run_chat(args: &Args) -> Result<(), Box<dyn std::error::Error>> {
 
         let from_pos = pos;
         let t1 = std::time::Instant::now();
-        let (reply, new_pos, n, _profile) = chat::generate_reply(&mut sess, &mut kv, &turn_ids, pos, print_progress)?;
+        let show_disk_wait = sess.caches.any_has_ring();
+        let filter = chat::output_filter(&sess.model, args.think);
+        let (reply, new_pos, n, _profile) = chat::generate_reply(&mut sess, &mut kv, &turn_ids, pos, filter, |ev| print_progress(ev, show_disk_wait))?;
         let elapsed = t1.elapsed().as_secs_f32();
         eprintln!("{n} tokens in {elapsed:.1}s ({:.1} tok/s)", n as f32 / elapsed.max(0.001));
         pos = new_pos;
@@ -283,13 +326,24 @@ fn run_serve(args: &Args) -> Result<(), Box<dyn std::error::Error>> {
 /// anyone who wants to test otherwise on different hardware. Only ever called once, before any
 /// generation work starts, so a `build_global` failure (already-initialized pool) can't happen
 /// in practice; logged rather than panicking just in case.
-fn configure_thread_pool(threads: Option<usize>) {
+fn configure_thread_pool(threads: Option<usize>, numa: bool, numa_threads: Option<usize>) {
     let n = threads.unwrap_or_else(|| num_cpus::get_physical().max(1));
     if let Err(e) = rayon::ThreadPoolBuilder::new().num_threads(n).build_global() {
         eprintln!("warning: failed to configure {n}-thread rayon pool ({e}), using the default");
     } else {
         eprintln!("using {n} threads ({} physical cores detected)", num_cpus::get_physical());
     }
+    // Phase N2 (`NUMA_AMX_BRIEF.md`): pinned per-node pools, sized from the SAME effective
+    // `--threads` total as the global pool above (total/n_nodes each) so the thread sweep stays
+    // meaningful with --numa on. The global pool remains for everything that isn't
+    // NUMA-dispatched (other architectures, attention, dense matmuls) — the two never run
+    // concurrently, so this is not oversubscription.
+    if numa {
+        match rabbit::numa::NodePools::init(numa_threads.unwrap_or(n)) {
+            Ok(p) => eprintln!("--numa: {} pinned node pools x {} threads", p.n(), p.threads_per_pool()),
+            Err(why) => eprintln!("--numa: {why}"),
+        }
+    }
 }
 
 fn main() -> ExitCode {
@@ -300,7 +354,7 @@ fn main() -> ExitCode {
             return ExitCode::FAILURE;
         }
     };
-    configure_thread_pool(args.threads);
+    configure_thread_pool(args.threads, args.numa, args.numa_threads);
     let result = if args.serve {
         run_serve(&args)
     } else if args.chat {
diff --git a/src/model.rs b/src/model.rs
index 1d1428d..df8a1d1 100644
--- a/src/model.rs
+++ b/src/model.rs
@@ -249,6 +249,29 @@ impl ExpertCaches {
         }
     }
 
+    /// Whether any layer loads through an `io_uring` ring — see `ExpertCache::has_ring`. When
+    /// `false` (MXFP4/K3), `io_wait_nanos_total` is structurally zero and the CLI suppresses its
+    /// "actual disk wait" readout (Phase 4c).
+    pub fn any_has_ring(&self) -> bool {
+        match self {
+            ExpertCaches::Glm52(c) => c.any_has_ring(),
+            ExpertCaches::KimiLinear(c) => c.any_has_ring(),
+            ExpertCaches::KimiK3(c) => c.any_has_ring(),
+        }
+    }
+
+    /// Phase 4b: preload every MoE layer's experts up front (see `expert_cache::preload_layers`).
+    /// Same dispatch/panic contract as `step` — `caches` and `model` always come from the same
+    /// `Model::load` in any correctly-threaded session.
+    pub fn preload(&mut self, model: &Model, shards: &crate::safetensors::Shards) -> Result<(), ModelError> {
+        match (self, model) {
+            (ExpertCaches::Glm52(c), Model::Glm52(m)) => Ok(c.preload(m, shards)?),
+            (ExpertCaches::KimiLinear(c), Model::KimiLinear(m)) => Ok(c.preload(m, shards)?),
+            (ExpertCaches::KimiK3(c), Model::KimiK3(m)) => Ok(c.preload(m, shards)?),
+            _ => unreachable!("ExpertCaches/Model family mismatch -- always construct ExpertCaches from the same Model"),
+        }
+    }
+
     pub fn warm_start(&mut self, model_dir: &Path, cache_capacity: usize) -> crate::generate::WarmStartStats {
         match self {
             ExpertCaches::Glm52(c) => c.warm_start(model_dir, cache_capacity),
diff --git a/src/numa.rs b/src/numa.rs
new file mode 100644
index 0000000..93af261
--- /dev/null
+++ b/src/numa.rs
@@ -0,0 +1,582 @@
+//! NUMA topology discovery + placement primitives (Phase N1 of `NUMA_AMX_BRIEF.md`).
+//!
+//! The design principle this module serves (brief §1): all tensors of one computation group live
+//! on one NUMA node, and the compute for that group runs on that node's cores. This is
+//! *bandwidth* locality, not cache locality — in decode every expert-weight byte is read exactly
+//! once per token, so what placement buys is concurrent use of all nodes' DRAM channels instead
+//! of hammering the one or two nodes first-touch happened to fill (measured on the target box:
+//! 183 GB/s node-local reads vs 87 GB/s cross-socket — a 2.1× split, `PERFORMANCE.md` Phase N0).
+//!
+//! Everything here is `libc`-only (house rule: no `hwloc`, no `numa`/`numactl` crates —
+//! `mbind`/`get_mempolicy` have no glibc wrapper at all, they only exist as wrappers in the
+//! banned libnuma, so they are raw `libc::syscall` calls). The whole module is compiled twice:
+//! the real implementation for Linux, and a no-op twin for everything else, so the owner's Mac
+//! and the single-node dev laptop build and behave exactly as before. Nothing here may hardcode
+//! node or core counts — the brief's provenance caveat (§1) is explicit that topology comes from
+//! [`topology()`] at runtime, never from documentation.
+//!
+//! Placement mechanism of choice is **first touch on a pinned thread** (an allocation's pages
+//! land on the node of the CPU that first writes them): wherever a load/init loop already runs
+//! on a thread we control — the expert preload path — pinning that thread is the entire
+//! placement story, zero syscalls per buffer. [`bind_region`]/[`interleave_region`] exist for
+//! buffers whose faulting thread can't be controlled (e.g. a reserve-then-grow KV cache, Phase
+//! N4c): `mbind` sets a **VMA policy**, so pages faulted *later* in the range follow it — but it
+//! does not migrate pages already touched (first-touch discipline instead of `MPOL_MF_MOVE`;
+//! moving pages is a rescue, not a design).
+
+#[cfg(target_os = "linux")]
+pub use linux::*;
+
+#[cfg(not(target_os = "linux"))]
+pub use stub::*;
+
+use std::sync::OnceLock;
+
+/// The home node of routed expert `eid` in layer `layer` (Phase N3a): static, deterministic,
+/// stateless — `(layer * n_experts + eid) % n_nodes`. Placement (which node's pool loads the
+/// expert, so first touch lands its pages there) and dispatch (which pool computes it) MUST both
+/// come through here: the whole point is that the two always agree, and a pure function of
+/// `(layer, eid)` can't drift the way persisted state could. The `layer *` term rotates the
+/// assignment so an expert id hot in many layers doesn't pile onto one node.
+pub fn home_node(layer: usize, n_experts: usize, eid: usize, n_nodes: usize) -> usize {
+    (layer * n_experts + eid) % n_nodes.max(1)
+}
+
+/// One pinned worker pool per NUMA **domain** (Phase N2, domain choice = decision D1). Built at
+/// most once, only when `--numa` is active AND the machine really has multiple nodes; everything
+/// else leaves the singleton `None` and every consumer takes its existing global-pool path
+/// unchanged.
+///
+/// **D1: domains are per-SOCKET** (2026-08-02, owner-approved flip from the initial per-node
+/// build) — see [`Topology::socket_domains`] for the grouping rule and the measured skew/locality
+/// argument. On a non-SNC machine where every node IS a socket, the grouping is the identity and
+/// nothing changes.
+///
+/// **Pool size derives from the effective `--threads` total, not the core count**
+/// (`total / n_domains`, min 1 per domain): the owner's measured 48-beats-192 result means the
+/// thread sweep must stay meaningful with `--numa` on — hardcoding cores-per-domain would
+/// silently override it (brief §6).
+///
+/// The hazard this design is shaped around (brief §6, trap #2): **never `pool.install(..)` from
+/// inside another pool's worker** — it blocks a worker thread waiting on a different pool, which
+/// deadlocks under load. [`NodePools::run_all`] is the only cross-pool fan-out and it always
+/// orchestrates from a NON-pool thread (debug-asserted).
+pub struct NodePools {
+    pools: Vec<(NumaNode, rayon::ThreadPool)>,
+}
+
+static NODE_POOLS: OnceLock<Option<NodePools>> = OnceLock::new();
+
+impl NodePools {
+    /// Builds the singleton from [`topology()`] with `total_threads` workers spread evenly
+    /// across nodes, pinning each pool's workers to its node's CPUs. Returns the reason as a
+    /// human-readable `Err` when NUMA execution isn't possible (single node, no topology,
+    /// non-Linux, or a pool failing to build) — the caller logs it once; all consumers then see
+    /// [`node_pools()`] `== None` and degrade to the global pool.
+    pub fn init(total_threads: usize) -> Result<&'static NodePools, String> {
+        let built = NODE_POOLS.get_or_init(|| {
+            let topo = topology()?;
+            if topo.n_nodes() < 2 {
+                return None;
+            }
+            Self::build(topo.socket_domains(sys_distance_row), total_threads)
+        });
+        built.as_ref().ok_or_else(|| "no multi-node NUMA topology (or pool build failed) — running without node pools".to_string())
+    }
+
+    /// The non-singleton constructor behind [`NodePools::init`] — public so tests and benches can
+    /// build throwaway pools against explicit domains (per-socket via
+    /// [`Topology::socket_domains`], per-node via `topo.nodes`, or synthetic ones) WITHOUT
+    /// flipping the process-global singleton, which would silently switch every concurrently
+    /// running test onto the NUMA code paths.
+    pub fn build(domains: Vec<NumaNode>, total_threads: usize) -> Option<NodePools> {
+        if domains.len() < 2 {
+            return None;
+        }
+        let per = (total_threads / domains.len()).max(1);
+        let mut pools = Vec::with_capacity(domains.len());
+        for domain in domains {
+            let pin_to = domain.clone();
+            let pool = rayon::ThreadPoolBuilder::new()
+                .num_threads(per)
+                .start_handler(move |_| pin_current_thread(&pin_to))
+                .build()
+                .ok()?;
+            pools.push((domain, pool));
+        }
+        Some(NodePools { pools })
+    }
+
+    /// The pools if `--numa` was requested and [`NodePools::init`] succeeded, else `None`. This
+    /// is the one switch every NUMA-aware code path keys off.
+    pub fn get() -> Option<&'static NodePools> {
+        NODE_POOLS.get().and_then(|p| p.as_ref())
+    }
+
+    pub fn n(&self) -> usize {
+        self.pools.len()
+    }
+
+    pub fn threads_per_pool(&self) -> usize {
+        self.pools[0].1.current_num_threads()
+    }
+
+    /// Runs `f(i)` INSIDE pool `i`, for all pools concurrently, returning when every call is
+    /// done — the one fork/join a NUMA-dispatched MoE layer pays. `f(i)` may freely use rayon
+    /// parallelism; it executes on pool `i`'s pinned workers, so nested `par_iter`s stay on that
+    /// domain. A pool with nothing to do should just return immediately from its `f(i)`.
+    ///
+    /// Orchestration is NESTED `in_place_scope`s, one per pool, recursed on the calling thread:
+    /// each scope's `spawn` injects `f(i)` into ITS pool, the innermost recursion step then
+    /// unwinds, and each scope exit blocks (on the orchestrator thread only — never a pool
+    /// worker, so the install-from-another-pool deadlock stays structurally impossible) until
+    /// its pool's task is done. No OS threads are spawned and no `'static` bound is needed —
+    /// this replaced the original one-scoped-thread-per-pool shape after the N3 serve gate
+    /// measured the per-call cost: ~0.4 ms/call × 92 MoE layers ≈ 35 ms/token of pure spawn
+    /// overhead. (The larger cost of waking a SLEEPING pool's workers remains either way; see
+    /// `PERFORMANCE.md`'s N3 serve-gate decomposition.)
+    ///
+    /// Must not be called from inside any rayon pool (that would re-create the blocked-worker
+    /// hazard one level up); debug-asserted.
+    pub fn run_all<F: Fn(usize) + Sync>(&self, f: F) {
+        debug_assert!(rayon::current_thread_index().is_none(), "NodePools::run_all must be orchestrated from a non-pool thread");
+        fn nest<F: Fn(usize) + Sync>(pools: &[(NumaNode, rayon::ThreadPool)], i: usize, f: &F) {
+            let Some((_, pool)) = pools.get(i) else { return };
+            pool.in_place_scope(|s| {
+                s.spawn(move |_| f(i));
+                nest(pools, i + 1, f);
+            });
+        }
+        nest(&self.pools, 0, &f);
+    }
+}
+
+/// One NUMA node: its kernel id and the logical CPUs it owns, as parsed from
+/// `/sys/devices/system/node/node<id>/cpulist`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct NumaNode {
+    pub id: usize,
+    pub cpus: Vec<usize>,
+}
+
+/// The machine's NUMA layout, nodes sorted by id. Obtained via [`topology()`]; a machine (or OS)
+/// where that returns `None` — or a topology with fewer than 2 nodes — means every downstream
+/// consumer degrades to exactly the current single-pool behavior.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Topology {
+    pub nodes: Vec<NumaNode>,
+}
+
+impl Topology {
+    pub fn n_nodes(&self) -> usize {
+        self.nodes.len()
+    }
+
+    /// Groups the nodes into per-socket domains — one synthetic [`NumaNode`] per socket holding
+    /// the union of its nodes' CPUs (ids renumbered 0..n_sockets) — using the kernel's node
+    /// distance matrix: two nodes share a socket iff their mutual distance is below 20 (SNC
+    /// sub-NUMA domains report 10–17 to each other, 21+ across a socket boundary; the same
+    /// threshold `numactl --hardware` output makes obvious on this box). Falls back to the
+    /// per-node identity when a distance row is unreadable or the grouping merges nothing.
+    ///
+    /// This is decision **D1 resolved to per-socket** (2026-08-02, owner-approved), on N3's own
+    /// measured data: per-token expert wall was dominated by per-LAYER routing skew — with 6
+    /// per-node pools the busiest node draws ~4.5–5 of 16 experts against a 2.7 mean (~1.7×),
+    /// while 2 per-socket domains draw ~9.5 of 16 against an 8 mean (~1.19×) — and the N0a probe
+    /// measured intra-socket-remote reads at only −4% vs node-local, so widening the domain
+    /// buys back most of the skew cost for almost no locality cost. (First touch inside a
+    /// socket-pinned pool lands pages on whichever of the socket's nodes the faulting CPU
+    /// belongs to — still socket-local, which is what the −4% number says is enough.)
+    pub fn socket_domains(&self, distance_row: impl Fn(usize) -> Option<String>) -> Vec<NumaNode> {
+        let mut groups: Vec<Vec<&NumaNode>> = Vec::new();
+        'outer: for node in &self.nodes {
+            let Some(row) = distance_row(node.id) else { return self.nodes.clone() };
+            let dists: Vec<usize> = row.split_whitespace().filter_map(|t| t.parse().ok()).collect();
+            for g in &mut groups {
+                if dists.get(g[0].id).is_some_and(|&d| d < 20) {
+                    g.push(node);
+                    continue 'outer;
+                }
+            }
+            groups.push(vec![node]);
+        }
+        groups
+            .into_iter()
+            .enumerate()
+            .map(|(i, g)| NumaNode { id: i, cpus: g.iter().flat_map(|n| n.cpus.iter().copied()).collect() })
+            .collect()
+    }
+}
+
+/// Parses one `cpulist` string (`"0-31,192-223"`, `"7"`, `"0-3,8"`) into the CPU ids it names.
+/// Kernel format: comma-separated decimal ranges, each either `a` or `a-b` inclusive. Pure
+/// function so the parser is unit-testable without a `/sys` to read; `None` on anything
+/// malformed rather than a guess.
+fn parse_cpulist(s: &str) -> Option<Vec<usize>> {
+    let mut cpus = Vec::new();
+    for part in s.trim().split(',') {
+        if part.is_empty() {
+            return None;
+        }
+        match part.split_once('-') {
+            Some((a, b)) => {
+                let (a, b) = (a.parse::<usize>().ok()?, b.parse::<usize>().ok()?);
+                if b < a {
+                    return None;
+                }
+                cpus.extend(a..=b);
+            }
+            None => cpus.push(part.parse().ok()?),
+        }
+    }
+    Some(cpus)
+}
+
+#[cfg(target_os = "linux")]
+mod linux {
+    use super::{parse_cpulist, NumaNode, Topology};
+
+    /// Reads the machine's NUMA layout from `/sys/devices/system/node/node*/cpulist`. `None` if
+    /// the directory doesn't exist or any node's cpulist fails to parse — the caller treats that
+    /// exactly like a single-node machine (no placement, current behavior).
+    pub fn topology() -> Option<Topology> {
+        let mut nodes = Vec::new();
+        for entry in std::fs::read_dir("/sys/devices/system/node").ok()? {
+            let entry = entry.ok()?;
+            let name = entry.file_name();
+            let name = name.to_str()?;
+            let Some(id) = name.strip_prefix("node").and_then(|n| n.parse::<usize>().ok()) else {
+                continue; // `has_cpu`, `possible`, etc. — not node directories
+            };
+            let cpulist = std::fs::read_to_string(entry.path().join("cpulist")).ok()?;
+            let cpus = parse_cpulist(&cpulist)?;
+            if !cpus.is_empty() {
+                // CPU-less (memory-only) nodes exist on some machines; they can't host a pinned
+                // pool, so they are simply not schedulable entities here.
+                nodes.push(NumaNode { id, cpus });
+            }
+        }
+        if nodes.is_empty() {
+            return None;
+        }
+        nodes.sort_by_key(|n| n.id);
+        Some(Topology { nodes })
+    }
+
+    /// Pins the calling thread to `node`'s CPUs (`sched_setaffinity` on the current thread).
+    /// Best-effort: on failure the thread simply stays where the scheduler put it — placement
+    /// quality degrades, correctness doesn't.
+    ///
+    /// Also resets the calling thread's memory policy to the kernel default (first-touch-local)
+    /// via `set_mempolicy(MPOL_DEFAULT)`. Memory policy is inherited per-thread from the
+    /// process: under the canonical `numactl --interleave=all` launch, every thread starts with
+    /// `MPOL_INTERLEAVE`, which would spread a pinned worker's allocations across ALL nodes and
+    /// silently defeat the entire first-touch placement scheme this function exists for. The
+    /// reset is thread-local — the main thread (dense-weight loads, which N4 hasn't sharded yet)
+    /// keeps whatever policy the launch prefix set.
+    pub fn pin_current_thread(node: &NumaNode) {
+        unsafe {
+            let mut set: libc::cpu_set_t = std::mem::zeroed();
+            for &cpu in &node.cpus {
+                if cpu < 8 * std::mem::size_of::<libc::cpu_set_t>() {
+                    libc::CPU_SET(cpu, &mut set);
+                }
+            }
+            // 0 = the calling thread. Failure is not worth even a log line per thread; the
+            // pool builder logs once if the topology looks unusable.
+            libc::sched_setaffinity(0, std::mem::size_of::<libc::cpu_set_t>(), &set);
+            libc::syscall(libc::SYS_set_mempolicy, libc::MPOL_DEFAULT, std::ptr::null::<libc::c_ulong>(), 0 as libc::c_ulong);
+        }
+    }
+
+    /// `maxnode` for the raw `mbind`/`get_mempolicy` syscalls: the kernel's `get_nodes()`
+    /// internally does `--maxnode` before `BITS_TO_LONGS(maxnode)`, so passing one u64 mask word
+    /// as 65 declared bits makes it read exactly that one word (bits 0–63 — machines with more
+    /// than 64 NUMA nodes are out of scope, checked at the call sites).
+    const MAXNODE: libc::c_ulong = 65;
+
+    /// The whole-page subrange of `[addr, addr+len)` — `mbind` demands a page-aligned start and
+    /// rejects anything else with `EINVAL`, and heap pointers are NOT page-aligned in general
+    /// (glibc's mmap'd large chunks carry a 16-byte header, so even a fresh 16 MiB `Vec` starts
+    /// 16 bytes past the page boundary). The sliver pages outside the aligned range keep the
+    /// default policy and land wherever first touch puts them: at the multi-MB buffer sizes this
+    /// module exists for, two stray pages are noise.
+    fn page_aligned(addr: *mut u8, len: usize) -> Option<(usize, usize)> {
+        let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;
+        let start = (addr as usize).next_multiple_of(page);
+        let end = (addr as usize + len) / page * page;
+        (end > start).then_some((start, end - start))
+    }
+
+    /// Sets an `MPOL_BIND`-to-`node` VMA policy on the whole pages of `[addr, addr+len)`. Pages
+    /// already faulted stay where they are (no `MPOL_MF_MOVE` — see the module doc); pages
+    /// faulted after this call land on `node`. Returns `false` (having done nothing) on syscall
+    /// failure, a node id past the one-word mask, or a range without a single whole page in it.
+    ///
+    /// # Safety
+    /// `addr..addr+len` must be a valid mapping owned by the caller for the policy's lifetime.
+    pub unsafe fn bind_region(addr: *mut u8, len: usize, node: usize) -> bool {
+        let Some((start, alen)) = page_aligned(addr, len) else {
+            return false;
+        };
+        if node >= 64 {
+            return false;
+        }
+        let mask: libc::c_ulong = 1 << node;
+        unsafe { libc::syscall(libc::SYS_mbind, start, alen, libc::MPOL_BIND, &mask, MAXNODE, 0) == 0 }
+    }
+
+    /// Sets an `MPOL_INTERLEAVE`-across-all-nodes VMA policy on the whole pages of
+    /// `[addr, addr+len)` — the in-process equivalent of launching under
+    /// `numactl --interleave=all`, for one buffer. Same fault-time semantics and failure
+    /// behavior as [`bind_region`].
+    ///
+    /// # Safety
+    /// Same contract as [`bind_region`].
+    pub unsafe fn interleave_region(addr: *mut u8, len: usize, topo: &Topology) -> bool {
+        let Some((start, alen)) = page_aligned(addr, len) else {
+            return false;
+        };
+        let mut mask: libc::c_ulong = 0;
+        for n in &topo.nodes {
+            if n.id >= 64 {
+                return false;
+            }
+            mask |= 1 << n.id;
+        }
+        unsafe { libc::syscall(libc::SYS_mbind, start, alen, libc::MPOL_INTERLEAVE, &mask, MAXNODE, 0) == 0 }
+    }
+
+    /// One node's row of the kernel distance matrix, for [`super::Topology::socket_domains`] —
+    /// e.g. `"10 15 17 21 28 26"` for node 0 on the target box.
+    pub fn sys_distance_row(node_id: usize) -> Option<String> {
+        std::fs::read_to_string(format!("/sys/devices/system/node/node{node_id}/distance")).ok()
+    }
+
+    /// Which node the page containing `addr` is resident on, via
+    /// `get_mempolicy(MPOL_F_NODE | MPOL_F_ADDR)` — `None` if the syscall fails (e.g. the page
+    /// was never faulted). Diagnostic/test instrument, not a hot-path call.
+    pub fn node_of_page(addr: *const u8) -> Option<usize> {
+        // get_mempolicy query flags; not exported by the libc crate (they have no glibc wrapper
+        // either — values from linux/include/uapi/linux/mempolicy.h, stable ABI).
+        const MPOL_F_NODE: libc::c_ulong = 1 << 0;
+        const MPOL_F_ADDR: libc::c_ulong = 1 << 1;
+        let mut node: libc::c_int = -1;
+        let rc = unsafe {
+            libc::syscall(
+                libc::SYS_get_mempolicy,
+                &mut node,
+                std::ptr::null_mut::<libc::c_ulong>(),
+                0 as libc::c_ulong,
+                addr,
+                MPOL_F_NODE | MPOL_F_ADDR,
+            )
+        };
+        (rc == 0 && node >= 0).then_some(node as usize)
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+mod stub {
+    use super::{NumaNode, Topology};
+
+    /// Non-Linux twin: there is no NUMA API to speak to, so there is no topology — every
+    /// consumer takes its single-node degradation path. (macOS has no NUMA placement syscalls;
+    /// Apple silicon is UMA anyway.)
+    pub fn topology() -> Option<Topology> {
+        None
+    }
+
+    pub fn pin_current_thread(_node: &NumaNode) {}
+
+    /// # Safety
+    /// No-op; same signature as the Linux twin so call sites don't need their own cfg.
+    pub unsafe fn bind_region(_addr: *mut u8, _len: usize, _node: usize) -> bool {
+        false
+    }
+
+    /// # Safety
+    /// No-op; same signature as the Linux twin so call sites don't need their own cfg.
+    pub unsafe fn interleave_region(_addr: *mut u8, _len: usize, _topo: &Topology) -> bool {
+        false
+    }
+
+    pub fn node_of_page(_addr: *const u8) -> Option<usize> {
+        None
+    }
+
+    pub fn sys_distance_row(_node_id: usize) -> Option<String> {
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn cpulist_parser_handles_the_kernel_formats() {
+        assert_eq!(parse_cpulist("0-31,192-223\n").unwrap().len(), 64);
+        assert_eq!(parse_cpulist("0-31,192-223").unwrap()[32], 192);
+        assert_eq!(parse_cpulist("7").unwrap(), vec![7]);
+        assert_eq!(parse_cpulist("0-3,8").unwrap(), vec![0, 1, 2, 3, 8]);
+        assert_eq!(parse_cpulist("0"), Some(vec![0]));
+    }
+
+    /// The target box's real distance matrix must group its 6 SNC3 nodes into 2 sockets
+    /// {0,1,2} and {3,4,5}; a machine whose "nodes" are already sockets (all cross distances
+    /// ≥ 20) must come back unchanged; unreadable distances fall back to per-node identity.
+    #[test]
+    fn socket_domains_groups_snc_nodes_by_distance_and_falls_back_per_node() {
+        let topo = Topology {
+            nodes: (0..6).map(|id| NumaNode { id, cpus: vec![id * 10, id * 10 + 1] }).collect(),
+        };
+        let matrix = ["10 15 17 21 28 26", "15 10 15 23 26 23", "17 15 10 26 23 21", "21 28 26 10 15 17", "23 26 23 15 10 15", "26 23 21 17 15 10"];
+        let sockets = topo.socket_domains(|id| Some(matrix[id].to_string()));
+        assert_eq!(sockets.len(), 2);
+        assert_eq!(sockets[0].cpus, vec![0, 1, 10, 11, 20, 21]);
+        assert_eq!(sockets[1].cpus, vec![30, 31, 40, 41, 50, 51]);
+
+        let two = Topology { nodes: (0..2).map(|id| NumaNode { id, cpus: vec![id] }).collect() };
+        let same = two.socket_domains(|id| Some(["10 21", "21 10"][id].to_string()));
+        assert_eq!(same.len(), 2, "already-per-socket nodes must not merge");
+
+        assert_eq!(topo.socket_domains(|_| None), topo.nodes, "unreadable distances fall back to per-node");
+    }
+
+    #[test]
+    fn cpulist_parser_rejects_malformed_input_instead_of_guessing() {
+        assert_eq!(parse_cpulist(""), None);
+        assert_eq!(parse_cpulist("a-b"), None);
+        assert_eq!(parse_cpulist("3-1"), None);
+        assert_eq!(parse_cpulist("1,,2"), None);
+        assert_eq!(parse_cpulist("1-"), None);
+    }
+
+    /// `run_all(f)` must execute `f(i)` — and any rayon work `f(i)` fans out — on pool `i`'s
+    /// pinned CPUs, concurrently across pools. Checked with `sched_getcpu` from inside a
+    /// `par_iter` running in each pool: every observed CPU must belong to that pool's node.
+    /// SKIP-not-fail off Linux/multi-node, same as the placement test below.
+    #[test]
+    fn node_pools_run_work_on_their_own_nodes_cpus() {
+        #[cfg(target_os = "linux")]
+        {
+            let Some(topo) = topology() else {
+                eprintln!("SKIP: no NUMA topology on this machine");
+                return;
+            };
+            if topo.n_nodes() < 2 {
+                eprintln!("SKIP: single NUMA node — pinning is unobservable");
+                return;
+            }
+            // `build`, NOT `init`: setting the process-global singleton from a test would
+            // silently switch every other test in this binary onto the NUMA code paths.
+            // Per-node identity domains, so "pool i runs on node i's CPUs" stays assertable.
+            let pools = NodePools::build(topo.nodes.clone(), topo.n_nodes() * 2).expect("multi-node box must build pools");
+            use rayon::prelude::*;
+            let seen: Vec<std::sync::Mutex<Vec<usize>>> = (0..pools.n()).map(|_| std::sync::Mutex::new(Vec::new())).collect();
+            pools.run_all(|i| {
+                (0..64).into_par_iter().for_each(|_| {
+                    let cpu = unsafe { libc::sched_getcpu() };
+                    assert!(cpu >= 0);
+                    seen[i].lock().unwrap().push(cpu as usize);
+                });
+            });
+            for (i, node) in topo.nodes.iter().enumerate() {
+                for &cpu in seen[i].lock().unwrap().iter() {
+                    assert!(node.cpus.contains(&cpu), "pool {i} ran on CPU {cpu}, which is not on node {}", node.id);
+                }
+            }
+        }
+    }
+
+    /// The `numactl --interleave=all` hazard: memory policy is inherited by new threads from
+    /// their creator, so under the canonical interleaved launch every pool worker would START
+    /// with `MPOL_INTERLEAVE` — and first touch would spray each expert's pages across all nodes
+    /// instead of homing them. `pin_current_thread` must reset the worker's policy to default;
+    /// this reproduces the launch condition (interleave set on the spawning thread) and asserts
+    /// a pinned child's first-touched pages still land on ITS node.
+    #[test]
+    fn pin_current_thread_overrides_an_inherited_interleave_policy() {
+        #[cfg(target_os = "linux")]
+        {
+            let Some(topo) = topology() else {
+                eprintln!("SKIP: no NUMA topology on this machine");
+                return;
+            };
+            if topo.n_nodes() < 2 {
+                eprintln!("SKIP: single NUMA node — placement is unobservable");
+                return;
+            }
+            let target = topo.nodes.last().unwrap().clone();
+            let len = 4 << 20;
+            let buf = std::thread::scope(|s| {
+                // The spawning thread takes the interleave policy `numactl --interleave=all`
+                // would give the whole process; the child inherits it at spawn.
+                let mut mask: libc::c_ulong = 0;
+                for n in &topo.nodes {
+                    mask |= 1 << n.id;
+                }
+                unsafe { libc::syscall(libc::SYS_set_mempolicy, libc::MPOL_INTERLEAVE, &mask, 65 as libc::c_ulong) };
+                let buf = s
+                    .spawn(|| {
+                        pin_current_thread(&target);
+                        vec![1u8; len]
+                    })
+                    .join()
+                    .unwrap();
+                unsafe { libc::syscall(libc::SYS_set_mempolicy, libc::MPOL_DEFAULT, std::ptr::null::<libc::c_ulong>(), 0 as libc::c_ulong) };
+                buf
+            });
+            let page = 4096;
+            for off in (0..len).step_by(len / 8) {
+                let addr = unsafe { buf.as_ptr().add((off / page) * page) };
+                assert_eq!(node_of_page(addr), Some(target.id), "page at offset {off} escaped the pinned thread's node — the inherited interleave policy leaked through");
+            }
+        }
+    }
+
+    /// On a real multi-node Linux box: pinning a thread to a node and first-touching an
+    /// allocation from it must land the pages on that node (the exact mechanism Phase N3's
+    /// preload placement relies on), and `bind_region` must steer pages faulted after the call.
+    /// Prints a SKIP line (repo convention for absent-fixture tests) on single-node machines and
+    /// non-Linux — this is a placement test, not a parser test, and needs real NUMA to mean
+    /// anything.
+    #[test]
+    fn first_touch_on_a_pinned_thread_and_bind_region_place_pages_on_the_requested_node() {
+        let Some(topo) = topology() else {
+            eprintln!("SKIP: no NUMA topology on this machine");
+            return;
+        };
+        if topo.n_nodes() < 2 {
+            eprintln!("SKIP: single NUMA node — placement is unobservable");
+            return;
+        }
+        let target = topo.nodes.last().unwrap().clone();
+        let len = 4 << 20; // 4 MiB: large enough that malloc mmaps it fresh (own VMA, no reused pages)
+
+        // First-touch: fault the pages from a thread pinned to `target`.
+        let buf = std::thread::scope(|s| {
+            s.spawn(|| {
+                pin_current_thread(&target);
+                vec![1u8; len]
+            })
+            .join()
+            .unwrap()
+        });
+        let got = node_of_page(buf.as_ptr());
+        assert_eq!(got, Some(target.id), "first-touched pages must sit on the pinned thread's node");
+
+        // bind_region: reserve untouched memory, set the policy, THEN fault from an unpinned
+        // thread — the VMA policy, not the faulting CPU, must decide placement.
+        let first = &topo.nodes[0];
+        let mut reserved: Vec<u8> = Vec::with_capacity(len);
+        let ok = unsafe { bind_region(reserved.as_mut_ptr(), len, first.id) };
+        assert!(ok, "mbind on a fresh allocation should succeed");
+        reserved.resize(len, 1);
+        // Probe mid-buffer: the first page can straddle the alignment sliver `bind_region`
+        // leaves on the default policy (see `page_aligned`), the middle can't.
+        let mid = unsafe { reserved.as_ptr().add(len / 2) };
+        assert_eq!(node_of_page(mid), Some(first.id), "pages faulted after bind_region must follow the VMA policy");
+    }
+}
diff --git a/src/quant.rs b/src/quant.rs
index 2fdf56b..ae9a599 100644
--- a/src/quant.rs
+++ b/src/quant.rs
@@ -126,6 +126,39 @@ impl QT {
         }
     }
 
+    /// A fresh `QT` holding rows `[r0, r0+nr)` of this one, byte-for-byte — every variant's
+    /// buffers are row-major, so a contiguous row range is a contiguous byte range in each.
+    /// Exists for NUMA row-sharding (`kernels::QTSharded`): the FRESH allocations are the point
+    /// — called on a pinned thread, first touch places the copy's pages on that thread's node,
+    /// which a borrow of the original obviously couldn't do.
+    pub fn copy_rows(&self, r0: usize, nr: usize) -> QT {
+        assert!(r0 + nr <= self.rows, "copy_rows: row range past the end of the weight");
+        let cols = self.cols;
+        let kind = match &self.kind {
+            QTKind::F32(data) => QTKind::F32(data[r0 * cols..(r0 + nr) * cols].to_vec()),
+            QTKind::I8 { data, scale } => QTKind::I8 { data: data[r0 * cols..(r0 + nr) * cols].to_vec(), scale: scale[r0..r0 + nr].to_vec() },
+            QTKind::I4 { data, scale } => {
+                let rb = cols.div_ceil(2);
+                QTKind::I4 { data: data[r0 * rb..(r0 + nr) * rb].to_vec(), scale: scale[r0..r0 + nr].to_vec() }
+            }
+            QTKind::I2 { data, scale } => {
+                let rb = cols.div_ceil(4);
+                QTKind::I2 { data: data[r0 * rb..(r0 + nr) * rb].to_vec(), scale: scale[r0..r0 + nr].to_vec() }
+            }
+            QTKind::I4Grouped { data, scale, group_size } => {
+                let rb = cols.div_ceil(2);
+                let ngroups = cols.div_ceil(*group_size);
+                QTKind::I4Grouped { data: data[r0 * rb..(r0 + nr) * rb].to_vec(), scale: scale[r0 * ngroups..(r0 + nr) * ngroups].to_vec(), group_size: *group_size }
+            }
+            QTKind::MxFp4 { data, block_scale } => {
+                let rb = cols.div_ceil(2);
+                let bpr = cols.div_ceil(32);
+                QTKind::MxFp4 { data: data[r0 * rb..(r0 + nr) * rb].to_vec(), block_scale: block_scale[r0 * bpr..(r0 + nr) * bpr].to_vec() }
+            }
+        };
+        QT { rows: nr, cols, bits: self.bits, kind }
+    }
+
     /// Fills from row-major f32 weights `w[rows*cols]`, quantizing per the chosen format.
     pub fn fill(&mut self, w: &[f32]) {
         assert_eq!(w.len(), self.rows * self.cols);
diff --git a/src/server.rs b/src/server.rs
index bcd584b..3247303 100644
--- a/src/server.rs
+++ b/src/server.rs
@@ -140,7 +140,13 @@ fn push_profile(session: &mut Session, profile: chat::TurnProfile) {
 }
 
 fn handle_profile(request: Request, session: &Session) {
-    let body = json!({"seq": session.profile_seq, "turns": session.profile});
+    let mut body = json!({"seq": session.profile_seq, "turns": session.profile});
+    // Phase N3d (`NUMA_AMX_BRIEF.md`): per-node MoE dispatch totals since process start —
+    // present only when --numa dispatch has actually run. Monotonic; clients diff polls. The
+    // busy-time spread across nodes is the measured routing skew D1 is judged on.
+    if let Some(stats) = crate::kimi_k3::moe::numa_moe_stats() {
+        body["numa_moe"] = json!(stats.iter().enumerate().map(|(node, (busy_s, experts))| json!({"node": node, "busy_s": busy_s, "experts": experts})).collect::<Vec<_>>());
+    }
     respond_json(request, 200, body.to_string());
 }
 
@@ -258,7 +264,8 @@ fn handle_chat_completions(mut request: Request, session: &mut Session, cfg: &Se
     }
 
     let mut hit_stop = true;
-    let result = chat::generate_reply(session, &mut kv, &prompt_ids, 0, |ev| {
+    let filter = chat::output_filter(&session.model, cfg.think);
+    let result = chat::generate_reply(session, &mut kv, &prompt_ids, 0, filter, |ev| {
         if let GenEvent::Token { index, max, .. } = ev
             && index >= max
         {
@@ -330,7 +337,8 @@ fn stream_chat_completion(request: Request, session: &mut Session, kv: &mut KvSt
     let mut hit_stop = true;
     let mut io_failed = false;
 
-    let gen_result = chat::generate_reply(session, kv, prompt_ids, 0, |ev| {
+    let filter = chat::output_filter(&session.model, cfg.think);
+    let gen_result = chat::generate_reply(session, kv, prompt_ids, 0, filter, |ev| {
         if io_failed {
             return;
         }