gspark-etri · Oct 23, 2026 · Oct 23, 2026
diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl
@@ -3,3 +3,5 @@
 {"id": "4c7935b757264687b1310fbe1ec6399a", "ts": "2026-10-23T21:26:38+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Production factory union of generation-config/tokenizer EOS fixes real KONI HTTP turn end: baseline length, fixed stop, ignore_eos length, streaming stop, reuse stop. Baseline factory3fail/3pass ->22related tests pass. Preserved failed initial SSE checker log.", "next": "Publish scoped serving fix; verify NPU HTTP and retain long RNGD semantic validation.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"model": "KONI-8B", "backend": "gpu", "max_tokens": "8"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-http-final.json", "size": 5067, "sha256": "d5be9a4a52a6485f"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
 {"id": "e4993823527144b5a14e6441b1cd5c01", "ts": "2026-10-23T21:29:56+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Published production EOS fix and real KONI HTTP evidence in draft PR230 (e5fa087). Final factory/streaming/sampling/batched suite22passed; hardware checker5/5 cases passed; changed files sensitive-clean,4unchanged baseline findings in this branch.", "next": "Verify same HTTP factory on NPU; continue full four-model long semantic and heterogeneous PD validation.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "gpu"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-http-final.json", "size": 5067, "sha256": "d5be9a4a52a6485f"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
 {"id": "328b8349e9974a059ddc7e1a8a485e2c", "ts": "2026-10-23T21:32:23+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Real ATOM KONI HTTP also passes5/5 baseline/fixed/ignore/stream/reuse checks using same production factory: config-only length vs union stop. GPU+ATOM now10/10 expected outcomes; no SDK/artifact rebuild.", "next": "Update PR230 with NPU evidence; finish KONI medium-context RNGD handoff and broader PD work.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "atom", "kv_blocks": "1"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-atom-http.json", "size": 5199, "sha256": "4735255db379ff00"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
+{"id": "1020452aea8445f080108c1ed7f6d643", "ts": "2026-10-24T04:35:37+09:00", "task": "Dense hybrid GPU KV import (aicomp #258)", "result": "Dense hybrid receiver fix:31focused and505full CPU tests passed,34skipped,253subtests,151.69s. Preparing isolated PR and fresh Gemma live rerun; hardware and numerical acceptance not yet claimed.", "next": "Run corrected Gemma RNGD/GPU matrix preserving original failure; complete lifecycle and numerical checks", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [], "experiments": ["experiments/2026-10-24-dense-hybrid258.md"], "labels": ["bug"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
+{"id": "551af4e77bca4e67bf0512304142ac48", "ts": "2026-10-24T04:41:11+09:00", "task": "Dense hybrid GPU KV import (aicomp #258)", "result": "Corrected live Gemma completed24batches;20/20token checks and60/60timed texts exact to nativeGPU,84remote/0local/0errors. Shape crash fixed on hardware. Qwen0.6 GPU lifecycle passed; Qwen4 next. Owned ATOM Pod confirmed absent.", "next": "Publish hardware evidence PR259; finish remaining GPU fault coverage and numerical boundary work", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [{"path": "analysis/serving-matrix-201/local258-gemma-summary.json", "size": 1288, "sha256": "3afbcc7e61bb3239"}], "experiments": ["experiments/2026-10-24-dense-hybrid258.md"], "labels": ["bug"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md
@@ -37,3 +37,7 @@ A real terminated subprocess producer previously left /ready200 and new requests
 ## Batched producer timing (#249)
 
 Candidate exports airm_pd_producer_seconds_total{stage="prefill"|"transfer"} from existing accumulated batcher durations. Repeated scrapes do not add observations; missing timing keys are omitted. Tiny real-model admission test covers fixed worker-reported durations and legacy omissions. These counters are worker-reported admitted-handoff sums, not request latency or proof of zero cost for workers that do not measure stages. Full CPU suite:467passed,17skipped,135subtests,1existing warning. See pd-stage-metrics249 experiment.
+
+#258 dense hybrid KV: actual Gemma failure reproduced; mapping/group validation corrected.31focused and505full CPU tests pass; live rerun pending.
+
+#258 corrected live Gemma32ce5d5:24batches,20/20token checks,60/60timed texts exact;84remote/0local/0errors. Shape crash fixed in recorded hardware matrix; overall numerical/quality boundaries remain.
diff --git a/opt_prime/examples/pd/http_matrix/evidence258/dense258-focused-b.log b/opt_prime/examples/pd/http_matrix/evidence258/dense258-focused-b.log
@@ -0,0 +1,15 @@
+...............................                                          [100%]
+=============================== warnings summary ===============================
+tests/test_pd_dense_hybrid.py::test_dense_gemma_pd_matches_independent_hf_beyond_window
+  /home/gspark/venvs/pt213/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2481: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+  SOLUTION: Use torch.compile(flex_attention)(...)
+
+  If you want to debug your score_mod/mask_mod, you can set:
+  torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+  This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+    _warn_once(
+
+-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
+31 passed, 1 warning in 29.86s
diff --git a/opt_prime/examples/pd/http_matrix/evidence258/dense258-focused.log b/opt_prime/examples/pd/http_matrix/evidence258/dense258-focused.log
@@ -0,0 +1,109 @@
+............FF................                                           [100%]
+=================================== FAILURES ===================================
+_________________ test_recurrent_pd_hands_over_state[qwen3_5] __________________
+
+make = <function _qwen35 at 0x73d66db4f880>
+
+    @RECURRENT
+    def test_recurrent_pd_hands_over_state(make):
+        model = make()
+        engine = TorchPrefillEngine(copy.deepcopy(model), max_context=128, chunk=8, page_size=8, prefix_cache=False)
+        b = PDBatcher(model, PrefillWorker(engine), slots=2, max_context=128, page_size=8)
+        prompts = _prompts(4, 6, 30, seed=4)
+        fs = [b.submit(PDRequest(f"p{i}", p, 12)) for i, p in enumerate(prompts)]
+>       b.run_until_idle()
+
+tests/test_pd_hybrid.py:157:
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+opt_prime/accel/pd/batching.py:207: in run_until_idle
+    self.step()
+opt_prime/accel/pd/pd_batching.py:468: in step
+    self._admit_prefilled(block=not self.decoding)
+opt_prime/accel/pd/pd_batching.py:497: in _admit_prefilled
+    self.kv.write_span(r.slot, out.start, out.keys, out.values)
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+
+self = <opt_prime.accel.pd.paged.PagedKV object at 0x73d626af4b80>, slot = 0
+start = 0
+keys = tensor([[[[-0.0660,  2.0859,  0.4345, -1.4313,  0.4459, -0.0251, -0.4985,
+           -1.7052, -1.7932,  0.2561,  1.213...26, -0.4552,
+           -0.7853,  0.2788, -2.0608, -0.8472, -0.4423, -0.3608,  1.5063,
+           -1.4844,  0.3283]]]])
+values = tensor([[[[-2.4548e-01, -2.1482e-01,  7.9515e-03,  2.1531e-01, -9.8814e-03,
+           -1.9493e-02, -1.3410e-01,  1.33...3, -3.6229e-02,
+           -1.5983e-01,  8.6865e-02,  1.8228e-01,  1.4877e-02, -7.5556e-02,
+            2.9426e-01]]]])
+
+    def write_span(self, slot, start, keys, values):
+        """keys/values [layers, kv_heads, n, head_dim] for positions start .. start+n of `slot` (a list of those, one
+        per KV group, for hybrid models), each group in one indexed copy (the PD admission of KV that another engine
+        prefilled)"""
+        if isinstance(keys, torch.Tensor) and isinstance(values, torch.Tensor):
+            layers = max(self.layer_group) + 1
+            if keys.ndim != 4 or values.shape != keys.shape or keys.shape[0] != layers:
+>               raise ValueError("dense KV span must contain every model layer with matching K/V shapes")
+E               ValueError: dense KV span must contain every model layer with matching K/V shapes
+
+opt_prime/accel/pd/paged.py:454: ValueError
+_______________ test_recurrent_pd_hands_over_state[qwen3_5_moe] ________________
+
+make = <function _qwen35_moe at 0x73d66db4f910>
+
+    @RECURRENT
+    def test_recurrent_pd_hands_over_state(make):
+        model = make()
+        engine = TorchPrefillEngine(copy.deepcopy(model), max_context=128, chunk=8, page_size=8, prefix_cache=False)
+        b = PDBatcher(model, PrefillWorker(engine), slots=2, max_context=128, page_size=8)
+        prompts = _prompts(4, 6, 30, seed=4)
+        fs = [b.submit(PDRequest(f"p{i}", p, 12)) for i, p in enumerate(prompts)]
+>       b.run_until_idle()
+
+tests/test_pd_hybrid.py:157:
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+opt_prime/accel/pd/batching.py:207: in run_until_idle
+    self.step()
+opt_prime/accel/pd/pd_batching.py:468: in step
+    self._admit_prefilled(block=not self.decoding)
+opt_prime/accel/pd/pd_batching.py:497: in _admit_prefilled
+    self.kv.write_span(r.slot, out.start, out.keys, out.values)
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+
+self = <opt_prime.accel.pd.paged.PagedKV object at 0x73d6253e5f60>, slot = 0
+start = 0
+keys = tensor([[[[-1.3119e+00, -4.6383e-02,  4.9614e-01, -9.4488e-01, -4.3832e-01,
+            2.6910e-01,  6.3046e-01, -1.59...1,  1.3446e+00,
+            4.5665e-01,  1.3538e-01,  3.4665e-01, -1.7604e-01, -2.0072e-02,
+           -1.5641e+00]]]])
+values = tensor([[[[-3.2685e-01,  2.6668e-01,  4.5333e-03,  7.0952e-02,  1.8279e-01,
+            2.7009e-01,  1.4598e-01,  2.43...2, -9.8851e-02,
+           -5.3184e-02, -4.8559e-02,  9.7567e-02,  8.8385e-02,  8.4103e-02,
+           -3.2886e-02]]]])
+
+    def write_span(self, slot, start, keys, values):
+        """keys/values [layers, kv_heads, n, head_dim] for positions start .. start+n of `slot` (a list of those, one
+        per KV group, for hybrid models), each group in one indexed copy (the PD admission of KV that another engine
+        prefilled)"""
+        if isinstance(keys, torch.Tensor) and isinstance(values, torch.Tensor):
+            layers = max(self.layer_group) + 1
+            if keys.ndim != 4 or values.shape != keys.shape or keys.shape[0] != layers:
+>               raise ValueError("dense KV span must contain every model layer with matching K/V shapes")
+E               ValueError: dense KV span must contain every model layer with matching K/V shapes
+
+opt_prime/accel/pd/paged.py:454: ValueError
+=============================== warnings summary ===============================
+tests/test_pd_hybrid.py::test_batched_beyond_window_matches_hf[gemma3]
+  /home/gspark/venvs/pt213/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2481: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+  SOLUTION: Use torch.compile(flex_attention)(...)
+
+  If you want to debug your score_mod/mask_mod, you can set:
+  torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+  This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+    _warn_once(
+
+-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
+=========================== short test summary info ============================
+FAILED tests/test_pd_hybrid.py::test_recurrent_pd_hands_over_state[qwen3_5]
+FAILED tests/test_pd_hybrid.py::test_recurrent_pd_hands_over_state[qwen3_5_moe]
+2 failed, 28 passed, 1 warning in 26.53s
diff --git a/opt_prime/examples/pd/http_matrix/evidence258/dense258-full.log b/opt_prime/examples/pd/http_matrix/evidence258/dense258-full.log
@@ -0,0 +1,21 @@
+.sssssss.....................ssssssssss.....................................................s..... [ 18%]
+........................................................s.s............. [ 31%]
+....ss.....................ss....s...................................... [ 45%]
+........................................................................ [ 58%]
+....s.s...........................................s..................... [ 72%]
+................................................................... [ 84%]
+s.s...........................................................................s....    [100%]
+=============================== warnings summary ===============================
+tests/test_kvstore.py::test_second_engine_reuses_chunks_from_the_store[lmcache]
+  /home/gspark/venvs/pt213/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2481: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+  SOLUTION: Use torch.compile(flex_attention)(...)
+
+  If you want to debug your score_mod/mask_mod, you can set:
+  torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+  This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+    _warn_once(
+
+-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
+505 passed, 34 skipped, 1 warning, 253 subtests passed in 151.69s (0:02:31)
diff --git a/opt_prime/examples/pd/http_matrix/evidence258/dense258-red.log b/opt_prime/examples/pd/http_matrix/evidence258/dense258-red.log
@@ -0,0 +1,132 @@
+FFF                                                                      [100%]
+=================================== FAILURES ===================================
+______________ test_dense_span_maps_model_layers_at_nonzero_start ______________
+
+    def test_dense_span_maps_model_layers_at_nonzero_start():
+        cache = pool()
+        keys = torch.arange(30).reshape(3, 1, 5, 2).float()
+>       cache.write_span(0, 3, keys, keys + 100)
+
+tests/test_pd_dense_hybrid.py:19:
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+
+self = <opt_prime.accel.pd.paged.PagedKV object at 0x713eaa041540>, slot = 0
+start = 3
+keys = [tensor([[[[ 0.,  1.],
+          [ 2.,  3.],
+          [ 4.,  5.],
+          [ 6.,  7.],
+          [ 8.,  9.]]],
+
+
+   ...]],
+
+
+        [[[20., 21.],
+          [22., 23.],
+          [24., 25.],
+          [26., 27.],
+          [28., 29.]]]])]
+values = [tensor([[[[100., 101.],
+          [102., 103.],
+          [104., 105.],
+          [106., 107.],
+          [108., 109....    [[[120., 121.],
+          [122., 123.],
+          [124., 125.],
+          [126., 127.],
+          [128., 129.]]]])]
+
+    def write_span(self, slot, start, keys, values):
+        """keys/values [layers, kv_heads, n, head_dim] for positions start .. start+n of `slot` (a list of those, one
+        per KV group, for hybrid models), each group in one indexed copy (the PD admission of KV that another engine
+        prefilled)"""
+        keys = keys if isinstance(keys, (list, tuple)) else [keys]
+        values = values if isinstance(values, (list, tuple)) else [values]
+        idx = self.span_index(slot, start, start + keys[0].shape[2])
+        for pk, pv, k, v in zip(self.gk, self.gv, keys, values):
+>           pk[:, :, idx] = k.to(self.device, pk.dtype, non_blocking=True)
+E           RuntimeError: shape mismatch: value tensor of shape [3, 1, 5, 2] cannot be broadcast to indexing result of shape [2, 1, 5, 2]
+
+opt_prime/accel/pd/paged.py:455: RuntimeError
+______ test_invalid_grouped_span_rejected_before_any_write[missing_group] ______
+
+bad = 'missing_group'
+
+    @pytest.mark.parametrize('bad', ['missing_group', 'bad_later_shape'])
+    def test_invalid_grouped_span_rejected_before_any_write(bad):
+        cache = pool()
+        keys = [torch.ones(2, 1, 5, 2), torch.ones(1, 1, 5, 2)]
+        values = [x + 1 for x in keys]
+        if bad == 'missing_group':
+            keys, values = keys[:1], values[:1]
+        else:
+            values[1] = torch.ones(1, 1, 4, 2)
+>       with pytest.raises(ValueError):
+E       Failed: DID NOT RAISE ValueError
+
+tests/test_pd_dense_hybrid.py:37: Failed
+_____ test_invalid_grouped_span_rejected_before_any_write[bad_later_shape] _____
+
+bad = 'bad_later_shape'
+
+    @pytest.mark.parametrize('bad', ['missing_group', 'bad_later_shape'])
+    def test_invalid_grouped_span_rejected_before_any_write(bad):
+        cache = pool()
+        keys = [torch.ones(2, 1, 5, 2), torch.ones(1, 1, 5, 2)]
+        values = [x + 1 for x in keys]
+        if bad == 'missing_group':
+            keys, values = keys[:1], values[:1]
+        else:
+            values[1] = torch.ones(1, 1, 4, 2)
+        with pytest.raises(ValueError):
+>           cache.write_span(0, 0, keys, values)
+
+tests/test_pd_dense_hybrid.py:38:
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+
+self = <opt_prime.accel.pd.paged.PagedKV object at 0x713eaa0800a0>, slot = 0
+start = 0
+keys = [tensor([[[[1., 1.],
+          [1., 1.],
+          [1., 1.],
+          [1., 1.],
+          [1., 1.]]],
+
+
+        [[[1....  [1., 1.]]]]), tensor([[[[1., 1.],
+          [1., 1.],
+          [1., 1.],
+          [1., 1.],
+          [1., 1.]]]])]
+values = [tensor([[[[2., 2.],
+          [2., 2.],
+          [2., 2.],
+          [2., 2.],
+          [2., 2.]]],
+
+
+        [[[2....  [2., 2.],
+          [2., 2.]]]]), tensor([[[[1., 1.],
+          [1., 1.],
+          [1., 1.],
+          [1., 1.]]]])]
+
+    def write_span(self, slot, start, keys, values):
+        """keys/values [layers, kv_heads, n, head_dim] for positions start .. start+n of `slot` (a list of those, one
+        per KV group, for hybrid models), each group in one indexed copy (the PD admission of KV that another engine
+        prefilled)"""
+        keys = keys if isinstance(keys, (list, tuple)) else [keys]
+        values = values if isinstance(values, (list, tuple)) else [values]
+        idx = self.span_index(slot, start, start + keys[0].shape[2])
+        for pk, pv, k, v in zip(self.gk, self.gv, keys, values):
+            pk[:, :, idx] = k.to(self.device, pk.dtype, non_blocking=True)
+>           pv[:, :, idx] = v.to(self.device, pv.dtype, non_blocking=True)
+E           RuntimeError: shape mismatch: value tensor of shape [4, 2] cannot be broadcast to indexing result of shape [1, 1, 5, 2]
+
+opt_prime/accel/pd/paged.py:456: RuntimeError
+=========================== short test summary info ============================
+FAILED tests/test_pd_dense_hybrid.py::test_dense_span_maps_model_layers_at_nonzero_start
+FAILED tests/test_pd_dense_hybrid.py::test_invalid_grouped_span_rejected_before_any_write[missing_group]
+FAILED tests/test_pd_dense_hybrid.py::test_invalid_grouped_span_rejected_before_any_write[bad_later_shape]
+3 failed in 2.48s