test: evaluate held-out structured answers across four-model routes

opt_prime/examples/pd/http_matrix/.research/events.jsonl

@@ -8,3 +8,5 @@
 {"id": "c41ea1b2753146b091f85134832aa680", "ts": "2026-10-24T04:44:49+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "Prepared independent live GPU lifecycle validation branch on32f18ff with executed harness, native-call observer and completed Qwen0.6/Qwen4 raw evidence; KONI remains live. CLI help verified; production runtime unchanged.", "next": "Publish draft evidence PR and complete KONI/Gemma lifecycle runs", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [], "experiments": ["experiments/2026-10-24-live-gpu-fault201.md"], "labels": ["benchmark"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
 {"id": "2919d0420ffc49ceacc9aa8982bf4859", "ts": "2026-10-24T04:48:11+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "All4 live GPU lifecycle runs completed passed=true, including KONI/Gemma: cancellation/reuse, producer death503/readiness, slab unlink and replacement exact32-token controls. Started separate512-token concurrent-decode isolation suite after prior queue terminal and device-idle checks.", "next": "Publish remaining lifecycle evidence; verify currently decoding requests survive producer failure across4models", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [{"path": "analysis/serving-matrix-201/local201-koni-faults.json", "size": 1003, "sha256": "5ef1011fb2c2a4c6"}, {"path": "analysis/serving-matrix-201/local201-gemma-faults.json", "size": 1016, "sha256": "e707fb1f52adbb1a"}], "experiments": ["experiments/2026-10-24-live-gpu-fault201.md"], "labels": ["benchmark"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
 {"id": "eb53d939aca44b85ab4b6fd602754651", "ts": "2026-10-24T04:58:02+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "All4 GPU concurrent-decode isolation tests passed512exact tokens after producer killed at token2, plus cancellation/503/slab-unlink/replacement. KONI paddedGPU control completed: changes output but does NOT reproduce short RNGD output; padding-only explanation rejected.", "next": "Publish all4isolation evidence; capture short KONI KV for identical-prefix numerical analysis", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [{"path": "analysis/serving-matrix-201/koni-short-padding-diagnostic.json", "size": 5903, "sha256": "f67039817a78849a"}], "experiments": ["experiments/2026-10-24-live-gpu-fault201.md"], "labels": ["benchmark"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
+{"id": "ce47a4a232ae4f6dbb19f8738f8b2364", "ts": "2026-10-24T06:29:28+09:00", "task": "Structured answer compatibility (aicomp #263)", "result": "Completed all576 responses, normal EOS and no inference failures. Qwen3-4B allfour routes36/36 answers and token-exact; Q06 and Gemma retain cross-route content counterexamples. Final published artifact auditor reproduces summary exactly; owned Pod absent.", "next": "Review PR264; investigate content differences with matched-KV controls; complete source-side streaming in #265", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"models": "4", "routes": "4", "cases_per_model": "36"}, "artifacts": ["evidence263/quality263-summary.json"], "experiments": ["experiments/2026-10-24-heldout-quality263.md"], "labels": ["experiment", "rngd"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
+{"id": "57487a847f5f468cbfb29adf40acbdb9", "ts": "2026-10-24T08:05:12+09:00", "task": "Held-out answer quality (aicomp #263)", "result": "Post-hoc content diagnostic on unchanged576 responses: Gemma29/30/29/29 correct after outer-fence/arithmetic-int relaxation vs frozen primary0/0/0/0. Other models unchanged; one Gemma cross-route content difference remains. All16 files match PR264, self-checks and deterministic rerun pass; primary criteria not changed.", "next": "Publish diagnostic as exploratory in PR264; preserve native-parity failures; continue active long ATOM timing64332", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": ["evidence263/quality263-content-diagnostic.json"], "experiments": ["experiments/2026-10-24-heldout-quality263.md"], "labels": ["analysis", "rngd"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "analysis host", "branch": "main", "commit": "351d39d-dirty"}

opt_prime/examples/pd/http_matrix/README.md

@@ -51,3 +51,42 @@ Run from the repository root. Omit the producer arguments for native control. `p
 Subprocess producers expose a boolean `ready` property; pools require every configured member that implements it to be ready. Batched `/ready` and `/health` return503 after a known terminal failure/close, and new completions receive retryable503 before admission. A healthy replacement at an idle boundary restores availability without poisoning the decoder. Individual request errors do not make a still-running producer unready.
 
 No pool member is silently removed and no request is automatically retried. Workers without this optional property retain prior behavior. Existing requests keep their existing completion/error handling. See producer-readiness246 for the real subprocess/HTTP reproduction and regression results.
+
+### Held-out structured answers (#263)
+
+`heldout_quality.py` freezes 36 synthetic cases per model: four addition,
+passkey, and Korean table variants at short/~2K/~8K lengths. It renders the
+model's chat template with thinking disabled, then sends the same input IDs
+to each route. Inputs and hashes are frozen in `evidence263/` before inference.
+Decompress each `*-fixtures.json.gz` with `gzip -dk` to reuse the exact inputs.
+
+```bash
+python heldout_quality.py run --fixtures INPUT.json --endpoint http://localhost:8000 \
+  --route native-gpu --output RESULTS.jsonl
+```
+
+Use normal EOS, greedy sampling, output cap128, no prefix/host caching. The
+runner refuses output overwrite, retains raw text/token IDs, and stops on
+an HTTP/inference failure. Start the server using the same runtime/model
+configuration being compared; the runner does not launch or alter it.
+
+`answer_correct` requires an exact string answer in a parsed JSON object;
+`format_correct` separately requires exactly one `answer` key. Duplicate keys,
+non-string answers, trailing prose and malformed JSON fail. Thus this measures
+structured-answer adherence, not unrestricted semantic quality. `compare`
+requires matching case sets and input hashes and reports both correct→wrong
+and wrong→correct transitions plus exact token matches. Do not erase existing
+strict token failures or infer general equivalence from these 36 synthetic cases.
+
+Reproduce the recorded partial audit after unpacking the frozen inputs:
+
+```bash
+gzip -dk evidence263/*-fixtures.json.gz
+python evidence263/summarize_quality263.py
+```
+
+The auditor recomputes scores, checks fixture hashes and case correspondence,
+and leaves uncollected routes explicit. Gemma's strict score is zero on both
+GPU routes because outputs contain Markdown code fences; this does not mean
+zero content accuracy. Its four changed pairs comprise three whitespace changes
+and one `null`→correct-city answer. No scoring rules were changed after inference.

opt_prime/examples/pd/http_matrix/STATUS.md

@@ -47,3 +47,21 @@ Live GPU faults#201: Qwen0.6/Qwen4 cancellation, producer-exit503/readiness, SHM
 KONI/Gemma GPU lifecycle completed; allfour pass cancellation/reuse, producer death/readiness503, slab unlink and producer replacement. Concurrent-decode isolation is being tested separately and not claimed here.
 
 Allfour live GPU isolation runs pass: active512-token decode survives other-request producer death with exact output; all baseline lifecycle gates retained.
+
+#263 held-out suite: Qwen3-0.6B/Qwen3-4B/KONI-8B/Gemma3-1B inputs frozen (36/model); scorer distinguishes parsed answer, JSON format and exact token parity. Tests9passed after9expected failures. Hardware results pending; #256/#261 remain open.
+
+#263 evaluator verification:10focused passed; full prior collection533passed/15skipped/253subtests. Frozen fixtures hashed before inference; GPU route queue started, hardware results pending.
+
+#263 first GPU pairs complete: Qwen0.6 native32/36 versus RNGD34/36, no paired regression,34/36 token equality; Qwen4 both36/36, alltokens equal. KONI/Gemma and ATOM queues still running; old long-output failures are retained.
+
+#263 all288 GPU responses complete; structured scores native/RNGD:Q06 32/34,Q4 36/36,KONI31/31,Gemma0/0 (code fences, not zero content accuracy). Exacttoken pairs34/36,36/36,35/36,32/36. Q06 ATOM pair33/33 scores and36/36 tokens equal; Q4nativeATOM36/36. All source weight shards/config/tokenizer SHA256 match between hosts. Remaining fiveATOM routes active.
+
+#263 Qwen4allfour routes complete:each36/36 structured answers and all36 token sequences equal acrossGPU/RNGD/ATOM. KONI/GemmaATOM pending; historical#256 remains open.
+
+#263 six-pair audit: Q06 passkey-3-2048 correct on RNGD→GPU but wrong on bothATOM routes; no universal quality-equivalence claim. KONI allfour complete31/36 scores; nativeGPU/nativeATOM andRNGD-GPU/RNGD-ATOM each36/36 exacttokens. OnlyGemma RNGD-ATOM pending.
+
+## Final held-out evaluation (#263)
+
+All 576 responses completed normally. Qwen3-4B: 36/36 structured answers and exact tokens on all four routes. Qwen3-0.6B and Gemma retain cross-route content counterexamples; all six route pairs are audited. Final published artifact auditor reproduces the summary exactly. Owned evaluation Pod absent. See evidence263 and the heldout-quality263 experiment; numerical compatibility and source streaming remain open.
+
+Post-hoc #263 content diagnostic (primary unchanged): Gemma29/30/29/29 of36 correct after only outer-fence/arithmetic-int relaxation, vs frozen primary0/0/0/0. Six or seven content errors and one cross-route answer difference remain. Other model scores unchanged. All16 raw JSONL inputs byte-identical to published evidence; analyzer self-checks/reproduction pass. Exploratory, not an acceptance-rule change.

opt_prime/examples/pd/http_matrix/evidence263/analyze_quality263_content.py

@@ -0,0 +1,50 @@
+"""Post-hoc diagnostic only: preserve frozen primary scores, separate wrapper/type failures."""
+import json,re,sys,itertools
+from pathlib import Path
+
+def content(row):
+ text=row['text'].strip();fenced=False
+ match=re.fullmatch(r'```(?:json)?[ \t]*\n(.*?)\n```',text,flags=re.S)
+ if match:text=match[1];fenced=True
+ def unique(pairs):
+  if len(dict(pairs))!=len(pairs):raise ValueError("duplicate keys")
+  return dict(pairs)
+ try:obj=json.loads(text,object_pairs_hook=unique)
+ except (ValueError,TypeError):return None,fenced,False
+ if not isinstance(obj,dict) or set(obj)!={'answer'}:return None,fenced,False
+ answer=obj['answer'];numeric=type(answer) is int and row['case'].startswith('arithmetic-')
+ if numeric:answer=str(answer)
+ if not isinstance(answer,str):answer=None
+ return answer,fenced,numeric
+
+def check():
+ def row(text,case='arithmetic-0-0'):return dict(text=text,case=case)
+ assert content(row('```json\n{"answer": 96}\n```'))==('96',True,True)
+ assert content(row('{"answer": true}'))[0] is None
+ assert content(row('{"answer": 96.0}'))[0] is None
+ assert content(row('{"answer": 96}','passkey-0-0'))[0] is None
+ assert content(row('prefix ```json\n{"answer":"96"}\n```'))[0] is None
+ assert content(row('{"answer":"096"}'))[0]=='096'
+ assert content(row('{"answer":null}'))[0] is None
+ assert content(row('{"answer":"wrong","answer":"96"}'))[0] is None
+ assert content(row('{"answer":"96","extra":1}'))[0] is None
+
+check()
+if __name__=='__main__':
+ root=Path(sys.argv[1]);out={'scope':'Post-hoc exploratory diagnostic. Frozen primary scores unchanged; this is not a replacement acceptance criterion. Only an outer JSON/unlabelled fence and integer-to-string arithmetic answers are relaxed.','models':{}}
+ routes=('native-gpu','rngd-gpu','native-atom','rngd-atom')
+ for model in ('qwen06b','qwen4b','koni','gemma'):
+  rows={route:[json.loads(x) for x in (root/f'quality263-{model}-{route}.jsonl').read_text().splitlines()] for route in routes}
+  assert all(len(v)==36 and len({r['case'] for r in v})==36 for v in rows.values())
+  stats={};parsed={}
+  for route,records in rows.items():
+   parsed[route]={r['case']:content(r)[0] for r in records}
+   stats[route]=dict(cases=36,primary_correct=sum(r['answer_correct'] for r in records),content_correct=sum(content(r)[0]==r['expected'] for r in records),outer_fence=sum(content(r)[1] for r in records),arithmetic_integer=sum(content(r)[2] for r in records),content_errors=[dict(case=r['case'],expected=r['expected'],answer=content(r)[0],text=r['text']) for r in records if content(r)[0]!=r['expected']])
+  pairs={}
+  for a,b in itertools.combinations(routes,2):
+   aa={r['case']:r for r in rows[a]};bb={r['case']:r for r in rows[b]}
+   assert aa.keys()==bb.keys()
+   assert all(aa[k]['input_sha256']==bb[k]['input_sha256'] and aa[k]['expected']==bb[k]['expected'] for k in aa)
+   pairs[a+' vs '+b]=[dict(case=k,left=parsed[a][k],right=parsed[b][k],expected=aa[k]['expected']) for k in aa if parsed[a][k]!=parsed[b][k]]
+  out['models'][model]=dict(routes=stats,content_differences=pairs)
+ print(json.dumps(out,ensure_ascii=False,indent=2))

opt_prime/examples/pd/http_matrix/evidence263/fixture-manifest.json

@@ -0,0 +1,74 @@
+{
+  "quality263-gemma-fixtures.json": {
+    "sha256": "0c24b1e759148dd7de68e15fa435e03c77a85b947fd6e923eee46af8c87edf44",
+    "cases": 36,
+    "length_ranges": {
+      "0": [
+        38,
+        68
+      ],
+      "2048": [
+        2034,
+        2048
+      ],
+      "8192": [
+        8183,
+        8187
+      ]
+    }
+  },
+  "quality263-koni-fixtures.json": {
+    "sha256": "22f6e5b117913530ef6d58ed5352bb47684d55409535157b75f0462d64797eaf",
+    "cases": 36,
+    "length_ranges": {
+      "0": [
+        150,
+        182
+      ],
+      "2048": [
+        2040,
+        2045
+      ],
+      "8192": [
+        8186,
+        8191
+      ]
+    }
+  },
+  "quality263-qwen06b-fixtures.json": {
+    "sha256": "ed8adc8fe7bd7e6b5dff206ece812b74e398616a2b043092dcb3f5e26a20a27a",
+    "cases": 36,
+    "length_ranges": {
+      "0": [
+        40,
+        77
+      ],
+      "2048": [
+        2037,
+        2046
+      ],
+      "8192": [
+        8183,
+        8192
+      ]
+    }
+  },
+  "quality263-qwen4b-fixtures.json": {
+    "sha256": "701be76546e6fa3c7410475bc026a50c21c677abb7725565fb650b514fc71c98",
+    "cases": 36,
+    "length_ranges": {
+      "0": [
+        40,
+        77
+      ],
+      "2048": [
+        2037,
+        2046
+      ],
+      "8192": [
+        8183,
+        8192
+      ]
+    }
+  }
+}

opt_prime/examples/pd/http_matrix/evidence263/hash_models.py

@@ -0,0 +1,21 @@
+import hashlib,json,sys
+from pathlib import Path
+hub=Path.home()/'.cache/huggingface/hub'
+models=[('qwen06b','Qwen--Qwen3-0.6B',None),('qwen4b','Qwen--Qwen3-4B','Qwen3-4B'),('koni','KISTI-KONI--KONI-Llama3.1-8B-Instruct-20241024','KONI-8B'),('gemma','unsloth--gemma-3-1b-it','Gemma3-1B')]
+result={}
+for label,repo,alias in models:
+ cache=hub/('models--'+repo)
+ revision=(cache/'refs/main').read_text().strip() if (cache/'refs/main').exists() else None
+ if sys.argv[1]=='local' and alias: folder=Path('/tmp/serving-matrix-models')/alias
+ else: folder=cache/'snapshots'/revision
+ files=sorted(set(folder.glob('*.safetensors')) | {folder/'config.json',folder/'generation_config.json',folder/'tokenizer.json'})
+ row={}
+ for p in files:
+  if not p.exists():continue
+  h=hashlib.sha256()
+  with p.open('rb') as stream:
+   for chunk in iter(lambda:stream.read(8*1024*1024),b''): h.update(chunk)
+  row[p.name]={'bytes':p.stat().st_size,'sha256':h.hexdigest()}
+ assert any(name.endswith('.safetensors') for name in row)
+ result[label]={'revision_ref':revision,'files':row}
+print(json.dumps(result,indent=2))

opt_prime/examples/pd/http_matrix/evidence263/quality263-atom-source-sha256.txt

@@ -0,0 +1,8 @@
+c6674a711e3c071bd71a99fe2f155922b9b884340d16c7d25a7b9fdd8ec5b7e8  atom_batching253.py
+dbccc7a1d45671e46ea1ba413b11e4e4820ffc65e17566e0ad8b398e2fb37b95  pd_batching253.py
+81fa8c78648064c38a8146f52f5207553cf91fb5de8d9048f2de3340f4a8b316  batched253.py
+849165a74a2046f7bc201d346201bf1fa98ef4d7ec0fe4bd71498614d5ec47fd  llmd253.py
+7fe175b5c13fb8a3d913d5f42af906e7916de2d5cb1cba64905e5e2cea8ca935  worker_python253.sh
+f497a2c9adeabdd46a0fe2a62553dd6c3e9be632c09b82df3eef441557af0639  quality263_atom.py
+b4ab0f8c7810006bcbb6fb239a5d9f56d8a2ea9624d7174ec33c7344a57f4624  heldout_quality263.py
+587908a00c72826df0cb0b4175e1b0e219962b5a42a72200d0e42c2b9b89f1f0  run-quality263-atom.sh