diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index f883ccc0..61fb3f9b 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -8,3 +8,5 @@ {"id": "c41ea1b2753146b091f85134832aa680", "ts": "2026-10-24T04:44:49+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "Prepared independent live GPU lifecycle validation branch on32f18ff with executed harness, native-call observer and completed Qwen0.6/Qwen4 raw evidence; KONI remains live. CLI help verified; production runtime unchanged.", "next": "Publish draft evidence PR and complete KONI/Gemma lifecycle runs", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [], "experiments": ["experiments/2026-10-24-live-gpu-fault201.md"], "labels": ["benchmark"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} {"id": "2919d0420ffc49ceacc9aa8982bf4859", "ts": "2026-10-24T04:48:11+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "All4 live GPU lifecycle runs completed passed=true, including KONI/Gemma: cancellation/reuse, producer death503/readiness, slab unlink and replacement exact32-token controls. Started separate512-token concurrent-decode isolation suite after prior queue terminal and device-idle checks.", "next": "Publish remaining lifecycle evidence; verify currently decoding requests survive producer failure across4models", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [{"path": "analysis/serving-matrix-201/local201-koni-faults.json", "size": 1003, "sha256": "5ef1011fb2c2a4c6"}, {"path": "analysis/serving-matrix-201/local201-gemma-faults.json", "size": 1016, "sha256": "e707fb1f52adbb1a"}], "experiments": ["experiments/2026-10-24-live-gpu-fault201.md"], "labels": ["benchmark"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} {"id": "eb53d939aca44b85ab4b6fd602754651", "ts": "2026-10-24T04:58:02+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "All4 GPU concurrent-decode isolation tests passed512exact tokens after producer killed at token2, plus cancellation/503/slab-unlink/replacement. KONI paddedGPU control completed: changes output but does NOT reproduce short RNGD output; padding-only explanation rejected.", "next": "Publish all4isolation evidence; capture short KONI KV for identical-prefix numerical analysis", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [{"path": "analysis/serving-matrix-201/koni-short-padding-diagnostic.json", "size": 5903, "sha256": "f67039817a78849a"}], "experiments": ["experiments/2026-10-24-live-gpu-fault201.md"], "labels": ["benchmark"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "ce47a4a232ae4f6dbb19f8738f8b2364", "ts": "2026-10-24T06:29:28+09:00", "task": "Structured answer compatibility (aicomp #263)", "result": "Completed all576 responses, normal EOS and no inference failures. Qwen3-4B allfour routes36/36 answers and token-exact; Q06 and Gemma retain cross-route content counterexamples. Final published artifact auditor reproduces summary exactly; owned Pod absent.", "next": "Review PR264; investigate content differences with matched-KV controls; complete source-side streaming in #265", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"models": "4", "routes": "4", "cases_per_model": "36"}, "artifacts": ["evidence263/quality263-summary.json"], "experiments": ["experiments/2026-10-24-heldout-quality263.md"], "labels": ["experiment", "rngd"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "57487a847f5f468cbfb29adf40acbdb9", "ts": "2026-10-24T08:05:12+09:00", "task": "Held-out answer quality (aicomp #263)", "result": "Post-hoc content diagnostic on unchanged576 responses: Gemma29/30/29/29 correct after outer-fence/arithmetic-int relaxation vs frozen primary0/0/0/0. Other models unchanged; one Gemma cross-route content difference remains. All16 files match PR264, self-checks and deterministic rerun pass; primary criteria not changed.", "next": "Publish diagnostic as exploratory in PR264; preserve native-parity failures; continue active long ATOM timing64332", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": ["evidence263/quality263-content-diagnostic.json"], "experiments": ["experiments/2026-10-24-heldout-quality263.md"], "labels": ["analysis", "rngd"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "analysis host", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/examples/pd/http_matrix/README.md b/opt_prime/examples/pd/http_matrix/README.md index b260e35b..a7834601 100644 --- a/opt_prime/examples/pd/http_matrix/README.md +++ b/opt_prime/examples/pd/http_matrix/README.md @@ -51,3 +51,42 @@ Run from the repository root. Omit the producer arguments for native control. `p Subprocess producers expose a boolean `ready` property; pools require every configured member that implements it to be ready. Batched `/ready` and `/health` return503 after a known terminal failure/close, and new completions receive retryable503 before admission. A healthy replacement at an idle boundary restores availability without poisoning the decoder. Individual request errors do not make a still-running producer unready. No pool member is silently removed and no request is automatically retried. Workers without this optional property retain prior behavior. Existing requests keep their existing completion/error handling. See producer-readiness246 for the real subprocess/HTTP reproduction and regression results. + +### Held-out structured answers (#263) + +`heldout_quality.py` freezes 36 synthetic cases per model: four addition, +passkey, and Korean table variants at short/~2K/~8K lengths. It renders the +model's chat template with thinking disabled, then sends the same input IDs +to each route. Inputs and hashes are frozen in `evidence263/` before inference. +Decompress each `*-fixtures.json.gz` with `gzip -dk` to reuse the exact inputs. + +```bash +python heldout_quality.py run --fixtures INPUT.json --endpoint http://localhost:8000 \ + --route native-gpu --output RESULTS.jsonl +``` + +Use normal EOS, greedy sampling, output cap128, no prefix/host caching. The +runner refuses output overwrite, retains raw text/token IDs, and stops on +an HTTP/inference failure. Start the server using the same runtime/model +configuration being compared; the runner does not launch or alter it. + +`answer_correct` requires an exact string answer in a parsed JSON object; +`format_correct` separately requires exactly one `answer` key. Duplicate keys, +non-string answers, trailing prose and malformed JSON fail. Thus this measures +structured-answer adherence, not unrestricted semantic quality. `compare` +requires matching case sets and input hashes and reports both correct→wrong +and wrong→correct transitions plus exact token matches. Do not erase existing +strict token failures or infer general equivalence from these 36 synthetic cases. + +Reproduce the recorded partial audit after unpacking the frozen inputs: + +```bash +gzip -dk evidence263/*-fixtures.json.gz +python evidence263/summarize_quality263.py +``` + +The auditor recomputes scores, checks fixture hashes and case correspondence, +and leaves uncollected routes explicit. Gemma's strict score is zero on both +GPU routes because outputs contain Markdown code fences; this does not mean +zero content accuracy. Its four changed pairs comprise three whitespace changes +and one `null`→correct-city answer. No scoring rules were changed after inference. diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index 4664d251..d6484b8c 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -47,3 +47,21 @@ Live GPU faults#201: Qwen0.6/Qwen4 cancellation, producer-exit503/readiness, SHM KONI/Gemma GPU lifecycle completed; allfour pass cancellation/reuse, producer death/readiness503, slab unlink and producer replacement. Concurrent-decode isolation is being tested separately and not claimed here. Allfour live GPU isolation runs pass: active512-token decode survives other-request producer death with exact output; all baseline lifecycle gates retained. + +#263 held-out suite: Qwen3-0.6B/Qwen3-4B/KONI-8B/Gemma3-1B inputs frozen (36/model); scorer distinguishes parsed answer, JSON format and exact token parity. Tests9passed after9expected failures. Hardware results pending; #256/#261 remain open. + +#263 evaluator verification:10focused passed; full prior collection533passed/15skipped/253subtests. Frozen fixtures hashed before inference; GPU route queue started, hardware results pending. + +#263 first GPU pairs complete: Qwen0.6 native32/36 versus RNGD34/36, no paired regression,34/36 token equality; Qwen4 both36/36, alltokens equal. KONI/Gemma and ATOM queues still running; old long-output failures are retained. + +#263 all288 GPU responses complete; structured scores native/RNGD:Q06 32/34,Q4 36/36,KONI31/31,Gemma0/0 (code fences, not zero content accuracy). Exacttoken pairs34/36,36/36,35/36,32/36. Q06 ATOM pair33/33 scores and36/36 tokens equal; Q4nativeATOM36/36. All source weight shards/config/tokenizer SHA256 match between hosts. Remaining fiveATOM routes active. + +#263 Qwen4allfour routes complete:each36/36 structured answers and all36 token sequences equal acrossGPU/RNGD/ATOM. KONI/GemmaATOM pending; historical#256 remains open. + +#263 six-pair audit: Q06 passkey-3-2048 correct on RNGD→GPU but wrong on bothATOM routes; no universal quality-equivalence claim. KONI allfour complete31/36 scores; nativeGPU/nativeATOM andRNGD-GPU/RNGD-ATOM each36/36 exacttokens. OnlyGemma RNGD-ATOM pending. + +## Final held-out evaluation (#263) + +All 576 responses completed normally. Qwen3-4B: 36/36 structured answers and exact tokens on all four routes. Qwen3-0.6B and Gemma retain cross-route content counterexamples; all six route pairs are audited. Final published artifact auditor reproduces the summary exactly. Owned evaluation Pod absent. See evidence263 and the heldout-quality263 experiment; numerical compatibility and source streaming remain open. + +Post-hoc #263 content diagnostic (primary unchanged): Gemma29/30/29/29 of36 correct after only outer-fence/arithmetic-int relaxation, vs frozen primary0/0/0/0. Six or seven content errors and one cross-route answer difference remain. Other model scores unchanged. All16 raw JSONL inputs byte-identical to published evidence; analyzer self-checks/reproduction pass. Exploratory, not an acceptance-rule change. diff --git a/opt_prime/examples/pd/http_matrix/evidence263/analyze_quality263_content.py b/opt_prime/examples/pd/http_matrix/evidence263/analyze_quality263_content.py new file mode 100644 index 00000000..71c40841 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/analyze_quality263_content.py @@ -0,0 +1,50 @@ +"""Post-hoc diagnostic only: preserve frozen primary scores, separate wrapper/type failures.""" +import json,re,sys,itertools +from pathlib import Path + +def content(row): + text=row['text'].strip();fenced=False + match=re.fullmatch(r'```(?:json)?[ \t]*\n(.*?)\n```',text,flags=re.S) + if match:text=match[1];fenced=True + def unique(pairs): + if len(dict(pairs))!=len(pairs):raise ValueError("duplicate keys") + return dict(pairs) + try:obj=json.loads(text,object_pairs_hook=unique) + except (ValueError,TypeError):return None,fenced,False + if not isinstance(obj,dict) or set(obj)!={'answer'}:return None,fenced,False + answer=obj['answer'];numeric=type(answer) is int and row['case'].startswith('arithmetic-') + if numeric:answer=str(answer) + if not isinstance(answer,str):answer=None + return answer,fenced,numeric + +def check(): + def row(text,case='arithmetic-0-0'):return dict(text=text,case=case) + assert content(row('```json\n{"answer": 96}\n```'))==('96',True,True) + assert content(row('{"answer": true}'))[0] is None + assert content(row('{"answer": 96.0}'))[0] is None + assert content(row('{"answer": 96}','passkey-0-0'))[0] is None + assert content(row('prefix ```json\n{"answer":"96"}\n```'))[0] is None + assert content(row('{"answer":"096"}'))[0]=='096' + assert content(row('{"answer":null}'))[0] is None + assert content(row('{"answer":"wrong","answer":"96"}'))[0] is None + assert content(row('{"answer":"96","extra":1}'))[0] is None + +check() +if __name__=='__main__': + root=Path(sys.argv[1]);out={'scope':'Post-hoc exploratory diagnostic. Frozen primary scores unchanged; this is not a replacement acceptance criterion. Only an outer JSON/unlabelled fence and integer-to-string arithmetic answers are relaxed.','models':{}} + routes=('native-gpu','rngd-gpu','native-atom','rngd-atom') + for model in ('qwen06b','qwen4b','koni','gemma'): + rows={route:[json.loads(x) for x in (root/f'quality263-{model}-{route}.jsonl').read_text().splitlines()] for route in routes} + assert all(len(v)==36 and len({r['case'] for r in v})==36 for v in rows.values()) + stats={};parsed={} + for route,records in rows.items(): + parsed[route]={r['case']:content(r)[0] for r in records} + stats[route]=dict(cases=36,primary_correct=sum(r['answer_correct'] for r in records),content_correct=sum(content(r)[0]==r['expected'] for r in records),outer_fence=sum(content(r)[1] for r in records),arithmetic_integer=sum(content(r)[2] for r in records),content_errors=[dict(case=r['case'],expected=r['expected'],answer=content(r)[0],text=r['text']) for r in records if content(r)[0]!=r['expected']]) + pairs={} + for a,b in itertools.combinations(routes,2): + aa={r['case']:r for r in rows[a]};bb={r['case']:r for r in rows[b]} + assert aa.keys()==bb.keys() + assert all(aa[k]['input_sha256']==bb[k]['input_sha256'] and aa[k]['expected']==bb[k]['expected'] for k in aa) + pairs[a+' vs '+b]=[dict(case=k,left=parsed[a][k],right=parsed[b][k],expected=aa[k]['expected']) for k in aa if parsed[a][k]!=parsed[b][k]] + out['models'][model]=dict(routes=stats,content_differences=pairs) + print(json.dumps(out,ensure_ascii=False,indent=2)) diff --git a/opt_prime/examples/pd/http_matrix/evidence263/fixture-manifest.json b/opt_prime/examples/pd/http_matrix/evidence263/fixture-manifest.json new file mode 100644 index 00000000..28505930 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/fixture-manifest.json @@ -0,0 +1,74 @@ +{ + "quality263-gemma-fixtures.json": { + "sha256": "0c24b1e759148dd7de68e15fa435e03c77a85b947fd6e923eee46af8c87edf44", + "cases": 36, + "length_ranges": { + "0": [ + 38, + 68 + ], + "2048": [ + 2034, + 2048 + ], + "8192": [ + 8183, + 8187 + ] + } + }, + "quality263-koni-fixtures.json": { + "sha256": "22f6e5b117913530ef6d58ed5352bb47684d55409535157b75f0462d64797eaf", + "cases": 36, + "length_ranges": { + "0": [ + 150, + 182 + ], + "2048": [ + 2040, + 2045 + ], + "8192": [ + 8186, + 8191 + ] + } + }, + "quality263-qwen06b-fixtures.json": { + "sha256": "ed8adc8fe7bd7e6b5dff206ece812b74e398616a2b043092dcb3f5e26a20a27a", + "cases": 36, + "length_ranges": { + "0": [ + 40, + 77 + ], + "2048": [ + 2037, + 2046 + ], + "8192": [ + 8183, + 8192 + ] + } + }, + "quality263-qwen4b-fixtures.json": { + "sha256": "701be76546e6fa3c7410475bc026a50c21c677abb7725565fb650b514fc71c98", + "cases": 36, + "length_ranges": { + "0": [ + 40, + 77 + ], + "2048": [ + 2037, + 2046 + ], + "8192": [ + 8183, + 8192 + ] + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence263/hash_models.py b/opt_prime/examples/pd/http_matrix/evidence263/hash_models.py new file mode 100644 index 00000000..b521b493 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/hash_models.py @@ -0,0 +1,21 @@ +import hashlib,json,sys +from pathlib import Path +hub=Path.home()/'.cache/huggingface/hub' +models=[('qwen06b','Qwen--Qwen3-0.6B',None),('qwen4b','Qwen--Qwen3-4B','Qwen3-4B'),('koni','KISTI-KONI--KONI-Llama3.1-8B-Instruct-20241024','KONI-8B'),('gemma','unsloth--gemma-3-1b-it','Gemma3-1B')] +result={} +for label,repo,alias in models: + cache=hub/('models--'+repo) + revision=(cache/'refs/main').read_text().strip() if (cache/'refs/main').exists() else None + if sys.argv[1]=='local' and alias: folder=Path('/tmp/serving-matrix-models')/alias + else: folder=cache/'snapshots'/revision + files=sorted(set(folder.glob('*.safetensors')) | {folder/'config.json',folder/'generation_config.json',folder/'tokenizer.json'}) + row={} + for p in files: + if not p.exists():continue + h=hashlib.sha256() + with p.open('rb') as stream: + for chunk in iter(lambda:stream.read(8*1024*1024),b''): h.update(chunk) + row[p.name]={'bytes':p.stat().st_size,'sha256':h.hexdigest()} + assert any(name.endswith('.safetensors') for name in row) + result[label]={'revision_ref':revision,'files':row} +print(json.dumps(result,indent=2)) diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-atom-source-sha256.txt b/opt_prime/examples/pd/http_matrix/evidence263/quality263-atom-source-sha256.txt new file mode 100644 index 00000000..49d3e1b7 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-atom-source-sha256.txt @@ -0,0 +1,8 @@ +c6674a711e3c071bd71a99fe2f155922b9b884340d16c7d25a7b9fdd8ec5b7e8 atom_batching253.py +dbccc7a1d45671e46ea1ba413b11e4e4820ffc65e17566e0ad8b398e2fb37b95 pd_batching253.py +81fa8c78648064c38a8146f52f5207553cf91fb5de8d9048f2de3340f4a8b316 batched253.py +849165a74a2046f7bc201d346201bf1fa98ef4d7ec0fe4bd71498614d5ec47fd llmd253.py +7fe175b5c13fb8a3d913d5f42af906e7916de2d5cb1cba64905e5e2cea8ca935 worker_python253.sh +f497a2c9adeabdd46a0fe2a62553dd6c3e9be632c09b82df3eef441557af0639 quality263_atom.py +b4ab0f8c7810006bcbb6fb239a5d9f56d8a2ea9624d7174ec33c7344a57f4624 heldout_quality263.py +587908a00c72826df0cb0b4175e1b0e219962b5a42a72200d0e42c2b9b89f1f0 run-quality263-atom.sh diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-content-diagnostic.json b/opt_prime/examples/pd/http_matrix/evidence263/quality263-content-diagnostic.json new file mode 100644 index 00000000..95258fb4 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-content-diagnostic.json @@ -0,0 +1,609 @@ +{ + "scope": "Post-hoc exploratory diagnostic. Frozen primary scores unchanged; this is not a replacement acceptance criterion. Only an outer JSON/unlabelled fence and integer-to-string arithmetic answers are relaxed.", + "models": { + "qwen06b": { + "routes": { + "native-gpu": { + "cases": 36, + "primary_correct": 32, + "content_correct": 32, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [ + { + "case": "korean_table-1-0", + "expected": "목포", + "answer": "부산", + "text": "{\n \"answer\": \"부산\"\n}" + }, + { + "case": "passkey-3-0", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + }, + { + "case": "passkey-3-2048", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + }, + { + "case": "passkey-3-8192", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + } + ] + }, + "rngd-gpu": { + "cases": 36, + "primary_correct": 34, + "content_correct": 34, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [ + { + "case": "passkey-3-0", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + }, + { + "case": "passkey-3-8192", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + } + ] + }, + "native-atom": { + "cases": 36, + "primary_correct": 33, + "content_correct": 33, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [ + { + "case": "passkey-3-0", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + }, + { + "case": "passkey-3-2048", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + }, + { + "case": "passkey-3-8192", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + } + ] + }, + "rngd-atom": { + "cases": 36, + "primary_correct": 33, + "content_correct": 33, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [ + { + "case": "passkey-3-0", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + }, + { + "case": "passkey-3-2048", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + }, + { + "case": "passkey-3-8192", + "expected": "SBYZEGE7", + "answer": "SYBZEGE7", + "text": "{\n \"answer\": \"SYBZEGE7\"\n}" + } + ] + } + }, + "content_differences": { + "native-gpu vs rngd-gpu": [ + { + "case": "korean_table-1-0", + "left": "부산", + "right": "목포", + "expected": "목포" + }, + { + "case": "passkey-3-2048", + "left": "SYBZEGE7", + "right": "SBYZEGE7", + "expected": "SBYZEGE7" + } + ], + "native-gpu vs native-atom": [ + { + "case": "korean_table-1-0", + "left": "부산", + "right": "목포", + "expected": "목포" + } + ], + "native-gpu vs rngd-atom": [ + { + "case": "korean_table-1-0", + "left": "부산", + "right": "목포", + "expected": "목포" + } + ], + "rngd-gpu vs native-atom": [ + { + "case": "passkey-3-2048", + "left": "SBYZEGE7", + "right": "SYBZEGE7", + "expected": "SBYZEGE7" + } + ], + "rngd-gpu vs rngd-atom": [ + { + "case": "passkey-3-2048", + "left": "SBYZEGE7", + "right": "SYBZEGE7", + "expected": "SBYZEGE7" + } + ], + "native-atom vs rngd-atom": [] + } + }, + "qwen4b": { + "routes": { + "native-gpu": { + "cases": 36, + "primary_correct": 36, + "content_correct": 36, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [] + }, + "rngd-gpu": { + "cases": 36, + "primary_correct": 36, + "content_correct": 36, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [] + }, + "native-atom": { + "cases": 36, + "primary_correct": 36, + "content_correct": 36, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [] + }, + "rngd-atom": { + "cases": 36, + "primary_correct": 36, + "content_correct": 36, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [] + } + }, + "content_differences": { + "native-gpu vs rngd-gpu": [], + "native-gpu vs native-atom": [], + "native-gpu vs rngd-atom": [], + "rngd-gpu vs native-atom": [], + "rngd-gpu vs rngd-atom": [], + "native-atom vs rngd-atom": [] + } + }, + "koni": { + "routes": { + "native-gpu": { + "cases": 36, + "primary_correct": 31, + "content_correct": 31, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [ + { + "case": "korean_table-0-0", + "expected": "춘천", + "answer": "부산", + "text": "{\"answer\": \"부산\"}" + }, + { + "case": "arithmetic-1-0", + "expected": "83", + "answer": "48 + 35 = 83", + "text": "{\n \"answer\": \"48 + 35 = 83\"\n}" + }, + { + "case": "arithmetic-2-0", + "expected": "43", + "answer": "The sum of 32 and 11 is 43.", + "text": "{\n \"answer\": \"The sum of 32 and 11 is 43.\"\n}" + }, + { + "case": "passkey-0-8192", + "expected": "34HYL6W8", + "answer": null, + "text": "34HYL6W8" + }, + { + "case": "passkey-2-8192", + "expected": "WMLLRJUL", + "answer": null, + "text": "WMLLRJUL" + } + ] + }, + "rngd-gpu": { + "cases": 36, + "primary_correct": 31, + "content_correct": 31, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [ + { + "case": "korean_table-0-0", + "expected": "춘천", + "answer": "부산", + "text": "{\"answer\": \"부산\"}" + }, + { + "case": "arithmetic-1-0", + "expected": "83", + "answer": "48 + 35 = 83", + "text": "{\n \"answer\": \"48 + 35 = 83\"\n}" + }, + { + "case": "arithmetic-2-0", + "expected": "43", + "answer": "The sum of 32 and 11 is 43.", + "text": "{\n \"answer\": \"The sum of 32 and 11 is 43.\"\n}" + }, + { + "case": "passkey-0-8192", + "expected": "34HYL6W8", + "answer": null, + "text": "34HYL6W8" + }, + { + "case": "passkey-2-8192", + "expected": "WMLLRJUL", + "answer": null, + "text": "WMLLRJUL" + } + ] + }, + "native-atom": { + "cases": 36, + "primary_correct": 31, + "content_correct": 31, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [ + { + "case": "korean_table-0-0", + "expected": "춘천", + "answer": "부산", + "text": "{\"answer\": \"부산\"}" + }, + { + "case": "arithmetic-1-0", + "expected": "83", + "answer": "48 + 35 = 83", + "text": "{\n \"answer\": \"48 + 35 = 83\"\n}" + }, + { + "case": "arithmetic-2-0", + "expected": "43", + "answer": "The sum of 32 and 11 is 43.", + "text": "{\n \"answer\": \"The sum of 32 and 11 is 43.\"\n}" + }, + { + "case": "passkey-0-8192", + "expected": "34HYL6W8", + "answer": null, + "text": "34HYL6W8" + }, + { + "case": "passkey-2-8192", + "expected": "WMLLRJUL", + "answer": null, + "text": "WMLLRJUL" + } + ] + }, + "rngd-atom": { + "cases": 36, + "primary_correct": 31, + "content_correct": 31, + "outer_fence": 0, + "arithmetic_integer": 0, + "content_errors": [ + { + "case": "korean_table-0-0", + "expected": "춘천", + "answer": "부산", + "text": "{\"answer\": \"부산\"}" + }, + { + "case": "arithmetic-1-0", + "expected": "83", + "answer": "48 + 35 = 83", + "text": "{\n \"answer\": \"48 + 35 = 83\"\n}" + }, + { + "case": "arithmetic-2-0", + "expected": "43", + "answer": "The sum of 32 and 11 is 43.", + "text": "{\n \"answer\": \"The sum of 32 and 11 is 43.\"\n}" + }, + { + "case": "passkey-0-8192", + "expected": "34HYL6W8", + "answer": null, + "text": "34HYL6W8" + }, + { + "case": "passkey-2-8192", + "expected": "WMLLRJUL", + "answer": null, + "text": "WMLLRJUL" + } + ] + } + }, + "content_differences": { + "native-gpu vs rngd-gpu": [], + "native-gpu vs native-atom": [], + "native-gpu vs rngd-atom": [], + "rngd-gpu vs native-atom": [], + "rngd-gpu vs rngd-atom": [], + "native-atom vs rngd-atom": [] + } + }, + "gemma": { + "routes": { + "native-gpu": { + "cases": 36, + "primary_correct": 0, + "content_correct": 29, + "outer_fence": 36, + "arithmetic_integer": 8, + "content_errors": [ + { + "case": "arithmetic-2-0", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": 33\n}\n```\n" + }, + { + "case": "korean_table-3-0", + "expected": "속초", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + }, + { + "case": "korean_table-0-2048", + "expected": "춘천", + "answer": null, + "text": "```json\nnull\n```" + }, + { + "case": "arithmetic-2-2048", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": 33\n}\n```" + }, + { + "case": "korean_table-0-8192", + "expected": "춘천", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + }, + { + "case": "arithmetic-2-8192", + "expected": "43", + "answer": "33", + "text": "```json\n{\"answer\": \"33\"}\n```" + }, + { + "case": "korean_table-3-8192", + "expected": "속초", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + } + ] + }, + "rngd-gpu": { + "cases": 36, + "primary_correct": 0, + "content_correct": 30, + "outer_fence": 36, + "arithmetic_integer": 8, + "content_errors": [ + { + "case": "arithmetic-2-0", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": 33\n}\n```" + }, + { + "case": "korean_table-3-0", + "expected": "속초", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + }, + { + "case": "arithmetic-2-2048", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": 33\n}\n```" + }, + { + "case": "korean_table-0-8192", + "expected": "춘천", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + }, + { + "case": "arithmetic-2-8192", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": \"33\"\n}\n```" + }, + { + "case": "korean_table-3-8192", + "expected": "속초", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + } + ] + }, + "native-atom": { + "cases": 36, + "primary_correct": 0, + "content_correct": 29, + "outer_fence": 36, + "arithmetic_integer": 8, + "content_errors": [ + { + "case": "arithmetic-2-0", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": 33\n}\n```" + }, + { + "case": "korean_table-3-0", + "expected": "속초", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + }, + { + "case": "korean_table-0-2048", + "expected": "춘천", + "answer": null, + "text": "```json\nnull\n```" + }, + { + "case": "arithmetic-2-2048", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": 33\n}\n```" + }, + { + "case": "korean_table-0-8192", + "expected": "춘천", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + }, + { + "case": "arithmetic-2-8192", + "expected": "43", + "answer": "33", + "text": "```json\n{\"answer\":\"33\"}\n```" + }, + { + "case": "korean_table-3-8192", + "expected": "속초", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + } + ] + }, + "rngd-atom": { + "cases": 36, + "primary_correct": 0, + "content_correct": 29, + "outer_fence": 36, + "arithmetic_integer": 8, + "content_errors": [ + { + "case": "arithmetic-2-0", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": 33\n}\n```" + }, + { + "case": "korean_table-3-0", + "expected": "속초", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + }, + { + "case": "korean_table-0-2048", + "expected": "춘천", + "answer": null, + "text": "```json\nnull\n```" + }, + { + "case": "arithmetic-2-2048", + "expected": "43", + "answer": "33", + "text": "```json\n{\n \"answer\": 33\n}\n```" + }, + { + "case": "korean_table-0-8192", + "expected": "춘천", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + }, + { + "case": "arithmetic-2-8192", + "expected": "43", + "answer": "33", + "text": "```json\n{\"answer\":\"33\"}\n```" + }, + { + "case": "korean_table-3-8192", + "expected": "속초", + "answer": "부산", + "text": "```json\n{\n \"answer\": \"부산\"\n}\n```" + } + ] + } + }, + "content_differences": { + "native-gpu vs rngd-gpu": [ + { + "case": "korean_table-0-2048", + "left": null, + "right": "춘천", + "expected": "춘천" + } + ], + "native-gpu vs native-atom": [], + "native-gpu vs rngd-atom": [], + "rngd-gpu vs native-atom": [ + { + "case": "korean_table-0-2048", + "left": "춘천", + "right": null, + "expected": "춘천" + } + ], + "rngd-gpu vs rngd-atom": [ + { + "case": "korean_table-0-2048", + "left": "춘천", + "right": null, + "expected": "춘천" + } + ], + "native-atom vs rngd-atom": [] + } + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-full.log b/opt_prime/examples/pd/http_matrix/evidence263/quality263-full.log new file mode 100644 index 00000000..ce96f7de --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-full.log @@ -0,0 +1,46 @@ +............................................................................................s..... [ 17%] +.................................................................s.s.... [ 31%] +.............ss...........................s............................. [ 44%] +........................................................................ [ 57%] +.............s.s...........................................s............ [ 70%] +........................................................................ [ 84%] +....s.s.......................................................................... [ 98%] +.s.... [100%] +=============================== warnings summary =============================== +examples/pd/native_kv/test_native_pack.py::NativePackTest::test_exact_bytes_and_independent_lifetime +examples/pd/native_kv/test_native_pack.py::NativePackTest::test_exact_bytes_and_independent_lifetime +examples/pd/native_kv/test_native_pack.py::NativePackTest::test_exact_bytes_and_independent_lifetime + /home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/native_kv/test_native_pack.py:24: DeprecationWarning: The argument 'device' of Tensor.is_pinned() is deprecated. Please do not pass this argument. (Triggered internally at /__w/pytorch/pytorch/aten/src/ATen/native/Memory.cpp:31.) + self.assertTrue(actual.is_pinned('cuda')) + +examples/pd/native_kv/test_packed_lane.py::PackedLaneTest::test_owned_packet_and_other_lane_isolation +examples/pd/native_kv/test_packed_lane.py::PackedLaneTest::test_owned_packet_and_other_lane_isolation +examples/pd/native_kv/test_packed_lane.py::PackedLaneTest::test_owned_packet_and_other_lane_isolation + /home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/native_kv/gpu_batch_decoder.py:93: DeprecationWarning: The argument 'device' of Tensor.is_pinned() is deprecated. Please do not pass this argument. (Triggered internally at /__w/pytorch/pytorch/aten/src/ATen/native/Memory.cpp:31.) + or not host.is_pinned('cuda') or not host.is_contiguous()): + +examples/pd/native_kv/test_pinned_receiver.py::PinnedReceiveTest::test_eight_slot_tcp_frames_exact_at_capacity +examples/pd/native_kv/test_pinned_receiver.py::PinnedReceiveTest::test_eight_slot_tcp_frames_exact_at_capacity +examples/pd/native_kv/test_pinned_receiver.py::PinnedReceiveTest::test_eight_slot_tcp_frames_exact_at_capacity + /home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/native_kv/test_pinned_receiver.py:75: DeprecationWarning: The argument 'device' of Tensor.is_pinned() is deprecated. Please do not pass this argument. (Triggered internally at /__w/pytorch/pytorch/aten/src/ATen/native/Memory.cpp:31.) + self.assertTrue(direct.data.host.is_pinned('cuda')) + +examples/pd/native_kv/test_pinned_receiver.py::PinnedReceiveTest::test_real_multipart_exact_and_owned +examples/pd/native_kv/test_pinned_receiver.py::PinnedReceiveTest::test_real_multipart_exact_and_owned +examples/pd/native_kv/test_pinned_receiver.py::PinnedReceiveTest::test_real_multipart_exact_and_owned + /home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/native_kv/test_pinned_receiver.py:25: DeprecationWarning: The argument 'device' of Tensor.is_pinned() is deprecated. Please do not pass this argument. (Triggered internally at /__w/pytorch/pytorch/aten/src/ATen/native/Memory.cpp:31.) + self.assertTrue(direct.data.host.is_pinned('cuda')) + +tests/test_kvstore.py::test_second_engine_reuses_chunks_from_the_store[lmcache] + /home/gspark/venvs/pt213/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:2481: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel. + + SOLUTION: Use torch.compile(flex_attention)(...) + + If you want to debug your score_mod/mask_mod, you can set: + torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True + + This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results. + _warn_once( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +533 passed, 15 skipped, 13 warnings, 253 subtests passed in 160.17s (0:02:40) diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-fixtures.json.gz b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-fixtures.json.gz new file mode 100644 index 00000000..4cb6f6c0 Binary files /dev/null and b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-fixtures.json.gz differ diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.jsonl b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.jsonl new file mode 100644 index 00000000..b5b1ed87 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.jsonl @@ -0,0 +1,36 @@ +{"case": "arithmetic-0-0", "route": "native-atom", "input_sha256": "ca6bce506a59e3ff2ea456d58be66d889b684a8d3c0d8264ea1adf319eba3215", "input_tokens": 38, "expected": "96", "text": "```json\n{\n \"answer\": 96\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236819, 236825, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-0-0", "object": "text_completion", "created": 1792789514, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": 96\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236819, 236825, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 38, "completion_tokens": 17, "total_tokens": 55}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.20122857205569744} +{"case": "passkey-0-0", "route": "native-atom", "input_sha256": "e6af0e26e172ab6739b897c7e5886bbd244a67d68e688c29251f38f0b623c516", "input_tokens": 57, "expected": "34HYL6W8", "text": "```json\n{\n \"answer\": \"34HYL6W8\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236800, 236812, 32207, 236798, 236825, 236824, 236828, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-0-0", "object": "text_completion", "created": 1792789514, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"34HYL6W8\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236800, 236812, 32207, 236798, 236825, 236824, 236828, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 57, "completion_tokens": 23, "total_tokens": 80}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.2617191660683602} +{"case": "korean_table-0-0", "route": "native-atom", "input_sha256": "ea4ffcca39f8322f4ad91bec1ebc6bd77d5e11cfa2c10de1e5e0491a0c771c04", "input_tokens": 68, "expected": "춘천", "text": "```json\n{\n \"answer\": \"춘천\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 242349, 239360, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-0-0", "object": "text_completion", "created": 1792789514, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"춘천\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 242349, 239360, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 68, "completion_tokens": 18, "total_tokens": 86}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.2078818540321663} +{"case": "arithmetic-1-0", "route": "native-atom", "input_sha256": "9f6534dcba2389afa359614d713c5ccd0c632c8a3d737e948f5744236216751c", "input_tokens": 38, "expected": "83", "text": "```json\n{\n \"answer\": 83\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236828, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-1-0", "object": "text_completion", "created": 1792789514, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": 83\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236828, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 38, "completion_tokens": 17, "total_tokens": 55}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.19829980400390923} +{"case": "passkey-1-0", "route": "native-atom", "input_sha256": "8110d36233ff6302384e08812339b32a6505ad4f29827c8fcbc3cc5e6f7f8143", "input_tokens": 54, "expected": "CNBKEFR7", "text": "```json\n{\n \"answer\": \"CNBKEFR7\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 9353, 236799, 11998, 11869, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-1-0", "object": "text_completion", "created": 1792789515, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"CNBKEFR7\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 9353, 236799, 11998, 11869, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 54, "completion_tokens": 21, "total_tokens": 75}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.2414139130851254} +{"case": "korean_table-1-0", "route": "native-atom", "input_sha256": "e4ca0aa029d8e1569bb761ba0f1d8945c6ff89c30fc5676f2f5a70ebebebded6", "input_tokens": 68, "expected": "목포", "text": "```json\n{\n \"answer\": \"목포\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 239136, 238709, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-1-0", "object": "text_completion", "created": 1792789515, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"목포\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 239136, 238709, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 68, "completion_tokens": 18, "total_tokens": 86}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.20915404707193375} +{"case": "arithmetic-2-0", "route": "native-atom", "input_sha256": "86c0f5977a407fb0b704b381d799c8ec10e3c96c8c943eb9d09f69f90bcb7337", "input_tokens": 38, "expected": "43", "text": "```json\n{\n \"answer\": 33\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236800, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-2-0", "object": "text_completion", "created": 1792789515, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": 33\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236800, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 38, "completion_tokens": 17, "total_tokens": 55}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.19815890293102711} +{"case": "passkey-2-0", "route": "native-atom", "input_sha256": "beaa3fab948d71c3e9b2793b3fe179ce4cf2eeee17923a2f36cac0da494b4e0e", "input_tokens": 53, "expected": "WMLLRJUL", "text": "```json\n{\n \"answer\": \"WMLLRJUL\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 15378, 2182, 136559, 4057, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-2-0", "object": "text_completion", "created": 1792789515, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"WMLLRJUL\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 15378, 2182, 136559, 4057, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 53, "completion_tokens": 20, "total_tokens": 73}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.23161550099030137} +{"case": "korean_table-2-0", "route": "native-atom", "input_sha256": "6c40dd52b87165747a3f02db798f2ae066b85043ed5d362ce32ca37107517bc5", "input_tokens": 68, "expected": "전주", "text": "```json\n{\n \"answer\": \"전주\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 237773, 237754, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-2-0", "object": "text_completion", "created": 1792789515, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"전주\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 237773, 237754, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 68, "completion_tokens": 18, "total_tokens": 86}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.20851796097122133} +{"case": "arithmetic-3-0", "route": "native-atom", "input_sha256": "fef18b91d4de61b1ee4714defa961ff75523d237e32fd6d1a9aa05105b64a23e", "input_tokens": 38, "expected": "63", "text": "```json\n{\n \"answer\": 63\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236825, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-3-0", "object": "text_completion", "created": 1792789516, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": 63\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236825, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 38, "completion_tokens": 17, "total_tokens": 55}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.1981379520148039} +{"case": "passkey-3-0", "route": "native-atom", "input_sha256": "9d26ce509899d1c3ee7d654c2a5b9eef427fe3f062445022f1c12dc1c7a5d433", "input_tokens": 54, "expected": "SBYZEGE7", "text": "```json\n{\n \"answer\": \"SBYZEGE7\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236773, 19470, 7942, 8809, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-3-0", "object": "text_completion", "created": 1792789516, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"SBYZEGE7\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236773, 19470, 7942, 8809, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 54, "completion_tokens": 21, "total_tokens": 75}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.24013753200415522} +{"case": "korean_table-3-0", "route": "native-atom", "input_sha256": "80e75874d5f77f73b151cd532d78d8e353da8fa375ce6aa622fb4edaa66b2c11", "input_tokens": 68, "expected": "속초", "text": "```json\n{\n \"answer\": \"부산\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 184764, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-3-0", "object": "text_completion", "created": 1792789516, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"부산\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 184764, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 68, "completion_tokens": 17, "total_tokens": 85}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.19819610309787095} +{"case": "arithmetic-0-2048", "route": "native-atom", "input_sha256": "af4879aef2e6d64f89abce61ba827a7147ce7aa68f131b5059730f6daedfbf89", "input_tokens": 2048, "expected": "96", "text": "```json\n{\n \"answer\": 96\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236819, 236825, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-0-2048", "object": "text_completion", "created": 1792789516, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": 96\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236819, 236825, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 17, "total_tokens": 2065}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.38186334806960076} +{"case": "passkey-0-2048", "route": "native-atom", "input_sha256": "ec3792ad17aed14871774cb4bfb238a098a5fbfab84f26321dbaffef4b4bc183", "input_tokens": 2037, "expected": "34HYL6W8", "text": "```json\n{\n \"answer\": \"34HYL6W8\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236800, 236812, 32207, 236798, 236825, 236824, 236828, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-0-2048", "object": "text_completion", "created": 1792789517, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"34HYL6W8\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236800, 236812, 32207, 236798, 236825, 236824, 236828, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2037, "completion_tokens": 23, "total_tokens": 2060}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.4327426420059055} +{"case": "korean_table-0-2048", "route": "native-atom", "input_sha256": "3f9e6fea06717095d1a90b92ac31397cb680a716891bcdd1d20b251fb5511eb5", "input_tokens": 2048, "expected": "춘천", "text": "```json\nnull\n```", "tokens": [2717, 3723, 107, 6455, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-0-2048", "object": "text_completion", "created": 1792789517, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\nnull\n```", "token_ids": [2717, 3723, 107, 6455, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 7, "total_tokens": 2055}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.2619548300281167} +{"case": "arithmetic-1-2048", "route": "native-atom", "input_sha256": "c705989bcfc419d3aa14ab0b042b6e1ae5353866cd792b3c7891b76451e8ee5f", "input_tokens": 2048, "expected": "83", "text": "```json\n{\n \"answer\": 83\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236828, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-1-2048", "object": "text_completion", "created": 1792789517, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": 83\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236828, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 17, "total_tokens": 2065}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.36822352989111096} +{"case": "passkey-1-2048", "route": "native-atom", "input_sha256": "e1fbe20650a58df3fd16efb5b61c1e1d2a45ea5e677d760cea1ad435789ccec6", "input_tokens": 2034, "expected": "CNBKEFR7", "text": "```json\n{\n \"answer\": \"CNBKEFR7\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 9353, 236799, 11998, 11869, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-1-2048", "object": "text_completion", "created": 1792789518, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"CNBKEFR7\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 9353, 236799, 11998, 11869, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2034, "completion_tokens": 21, "total_tokens": 2055}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.41048803203739226} +{"case": "korean_table-1-2048", "route": "native-atom", "input_sha256": "d5a80b27cd41a44c089a38a4654dc064496f7e833b906493f407cf49c12879f5", "input_tokens": 2048, "expected": "목포", "text": "```json\n{\n \"answer\": \"목포\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 239136, 238709, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-1-2048", "object": "text_completion", "created": 1792789518, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"목포\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 239136, 238709, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 18, "total_tokens": 2066}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.38205373100936413} +{"case": "arithmetic-2-2048", "route": "native-atom", "input_sha256": "3718106299b7ba6fbcead43904b7e934e114eeb3f6164df3c7fe1db11599a006", "input_tokens": 2048, "expected": "43", "text": "```json\n{\n \"answer\": 33\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236800, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-2-2048", "object": "text_completion", "created": 1792789519, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": 33\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236800, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 17, "total_tokens": 2065}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.37025855004321784} +{"case": "passkey-2-2048", "route": "native-atom", "input_sha256": "31115153f401db2cb1466d72cf519d69168b22f449778893b0c3acca34cea0fb", "input_tokens": 2048, "expected": "WMLLRJUL", "text": "```json\n{\n \"answer\": \"WMLLRJUL\"\n}\n```\n", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 15378, 2182, 136559, 4057, 236775, 107, 236783, 107, 2717, 107, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-2-2048", "object": "text_completion", "created": 1792789519, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"WMLLRJUL\"\n}\n```\n", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 15378, 2182, 136559, 4057, 236775, 107, 236783, 107, 2717, 107, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 21, "total_tokens": 2069}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.41535092901904136} +{"case": "korean_table-2-2048", "route": "native-atom", "input_sha256": "9599db7b28431faf2421bf2180833e5a4aef98483a7254721f1ec83c92fd46f7", "input_tokens": 2048, "expected": "전주", "text": "```json\n{\n \"answer\": \"전주\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 237773, 237754, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-2-2048", "object": "text_completion", "created": 1792789519, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"전주\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 237773, 237754, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 18, "total_tokens": 2066}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.37819920596666634} +{"case": "arithmetic-3-2048", "route": "native-atom", "input_sha256": "251532c0d3e1841162c6c7e2dcc8bd256cc44a258af9daae4aa3d23024810d7c", "input_tokens": 2048, "expected": "63", "text": "```json\n{\n \"answer\": 63\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236825, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-3-2048", "object": "text_completion", "created": 1792789520, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": 63\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 236743, 236825, 236800, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 17, "total_tokens": 2065}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.37116554903332144} +{"case": "passkey-3-2048", "route": "native-atom", "input_sha256": "1bfc4b6311f887ae68cf363ab09f2e0115e9a5bce4dee802a84068ee11ea73ee", "input_tokens": 2034, "expected": "SBYZEGE7", "text": "```json\n{\n \"answer\": \"SBYZEGE7\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236773, 19470, 7942, 8809, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-3-2048", "object": "text_completion", "created": 1792789520, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"SBYZEGE7\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236773, 19470, 7942, 8809, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2034, "completion_tokens": 21, "total_tokens": 2055}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.41558859206270427} +{"case": "korean_table-3-2048", "route": "native-atom", "input_sha256": "3f1148302c3113bb35b82b8c90495e402f0fd8a6ee32cc13ab54f2e88a5bc569", "input_tokens": 2048, "expected": "속초", "text": "```json\n{\n \"answer\": \"속초\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 238701, 239378, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-3-2048", "object": "text_completion", "created": 1792789521, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"속초\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 238701, 239378, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2048, "completion_tokens": 18, "total_tokens": 2066}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.3807854199549183} +{"case": "arithmetic-0-8192", "route": "native-atom", "input_sha256": "8f09d7d4d70782f016ec9530e14fd8992c798993340fd89dc91a1c1dd8f98686", "input_tokens": 8183, "expected": "96", "text": "```json\n{\n \"answer\": \"96\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236819, 236825, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-0-8192", "object": "text_completion", "created": 1792789522, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"96\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236819, 236825, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 18, "total_tokens": 8201}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 1.006193727022037} +{"case": "passkey-0-8192", "route": "native-atom", "input_sha256": "04b383a0a4fbcb099f7b148e3e949fde72c21f6231bbec3dce68f53027894edf", "input_tokens": 8187, "expected": "34HYL6W8", "text": "```json\n{\n \"answer\": \"34HYL6W8\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236800, 236812, 32207, 236798, 236825, 236824, 236828, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-0-8192", "object": "text_completion", "created": 1792789523, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"34HYL6W8\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236800, 236812, 32207, 236798, 236825, 236824, 236828, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8187, "completion_tokens": 23, "total_tokens": 8210}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 1.0519393640570343} +{"case": "korean_table-0-8192", "route": "native-atom", "input_sha256": "e770191b7695f0d43ce4a12097d1d2ef0ae30f8c39118e7104e4d21047a4d5c4", "input_tokens": 8183, "expected": "춘천", "text": "```json\n{\n \"answer\": \"부산\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 184764, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-0-8192", "object": "text_completion", "created": 1792789524, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"부산\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 184764, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 17, "total_tokens": 8200}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.9891978190280497} +{"case": "arithmetic-1-8192", "route": "native-atom", "input_sha256": "ae1469d2d3a14cf7da2fa13c1f8128325479c8db9c7bb8f6123fbecd93e3e4cf", "input_tokens": 8183, "expected": "83", "text": "```json\n{\"answer\": \"83\"}\n```", "tokens": [2717, 3723, 107, 14937, 14433, 1083, 623, 236828, 236800, 25938, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-1-8192", "object": "text_completion", "created": 1792789525, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\"answer\": \"83\"}\n```", "token_ids": [2717, 3723, 107, 14937, 14433, 1083, 623, 236828, 236800, 25938, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 13, "total_tokens": 8196}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.9439002689905465} +{"case": "passkey-1-8192", "route": "native-atom", "input_sha256": "e59b6d8def078023ad5694672bb2bbec6994e459bc39c7cfab95eb2a0418008a", "input_tokens": 8184, "expected": "CNBKEFR7", "text": "```json\n{\n \"answer\": \"CNBKEFR7\"\n}\n```\n", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 9353, 236799, 11998, 11869, 236832, 236775, 107, 236783, 107, 2717, 107, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-1-8192", "object": "text_completion", "created": 1792789526, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"CNBKEFR7\"\n}\n```\n", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 9353, 236799, 11998, 11869, 236832, 236775, 107, 236783, 107, 2717, 107, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8184, "completion_tokens": 22, "total_tokens": 8206}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 1.0462756140623242} +{"case": "korean_table-1-8192", "route": "native-atom", "input_sha256": "9192491b17b6fa3a60efa8f084df443ffe2522a806a7eb6abbf9904ad6a5a50e", "input_tokens": 8183, "expected": "목포", "text": "```json\n{\n \"answer\": \"목포\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 239136, 238709, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-1-8192", "object": "text_completion", "created": 1792789527, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"목포\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 239136, 238709, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 18, "total_tokens": 8201}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 1.0007178611122072} +{"case": "arithmetic-2-8192", "route": "native-atom", "input_sha256": "04e5f9a4c4bdf5b4da2accc13cc755f3c422b8ff16dc60383234ed0ce0c1d744", "input_tokens": 8183, "expected": "43", "text": "```json\n{\"answer\":\"33\"}\n```", "tokens": [2717, 3723, 107, 14937, 14433, 12375, 236800, 236800, 25938, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-2-8192", "object": "text_completion", "created": 1792789528, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\"answer\":\"33\"}\n```", "token_ids": [2717, 3723, 107, 14937, 14433, 12375, 236800, 236800, 25938, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 12, "total_tokens": 8195}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.9364523710682988} +{"case": "passkey-2-8192", "route": "native-atom", "input_sha256": "bf1531f20519b1d1162a4230d9835ed8e395f12c9c94444744271828d7dea36a", "input_tokens": 8183, "expected": "WMLLRJUL", "text": "```json\n{\n \"answer\": \"WMLLRJUL\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 15378, 2182, 136559, 4057, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-2-8192", "object": "text_completion", "created": 1792789529, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"WMLLRJUL\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 15378, 2182, 136559, 4057, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 20, "total_tokens": 8203}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 1.0230182459345087} +{"case": "korean_table-2-8192", "route": "native-atom", "input_sha256": "bc8f317a6ec4db698ed1f794e9d056bc62a289ce6f49ff08b918454cf800456e", "input_tokens": 8183, "expected": "전주", "text": "```json\n{\n \"answer\": \"전주\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 237773, 237754, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-2-8192", "object": "text_completion", "created": 1792789530, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"전주\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 237773, 237754, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 18, "total_tokens": 8201}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.9982714019715786} +{"case": "arithmetic-3-8192", "route": "native-atom", "input_sha256": "a1fb40e56504a08b4fea4da371f0fce7183ca271335fa8039921ecf9f752868c", "input_tokens": 8183, "expected": "63", "text": "```json\n{\"answer\": \"63\"}\n```", "tokens": [2717, 3723, 107, 14937, 14433, 1083, 623, 236825, 236800, 25938, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-3-8192", "object": "text_completion", "created": 1792789531, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\"answer\": \"63\"}\n```", "token_ids": [2717, 3723, 107, 14937, 14433, 1083, 623, 236825, 236800, 25938, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 13, "total_tokens": 8196}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.9480245650047436} +{"case": "passkey-3-8192", "route": "native-atom", "input_sha256": "f3ba91ce9a3a993574e925264bc3207381e9bfb0710fbf6a2fa3f424774268b1", "input_tokens": 8184, "expected": "SBYZEGE7", "text": "```json\n{\n \"answer\": \"SBYZEGE7\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236773, 19470, 7942, 8809, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-passkey-3-8192", "object": "text_completion", "created": 1792789532, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"SBYZEGE7\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 236773, 19470, 7942, 8809, 236832, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8184, "completion_tokens": 21, "total_tokens": 8205}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 1.027090738993138} +{"case": "korean_table-3-8192", "route": "native-atom", "input_sha256": "4ef06a4874f17e50cec21d5f2611a9ff97e8c19b35a1446fa0cafd1f57c3fb40", "input_tokens": 8183, "expected": "속초", "text": "```json\n{\n \"answer\": \"부산\"\n}\n```", "tokens": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 184764, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop", "response": {"id": "quality263-korean_table-3-8192", "object": "text_completion", "created": 1792789533, "model": "/tmp/serving-matrix-models/Gemma3-1B", "choices": [{"index": 0, "text": "```json\n{\n \"answer\": \"부산\"\n}\n```", "token_ids": [2717, 3723, 107, 236782, 107, 138, 236775, 14433, 1083, 623, 184764, 236775, 107, 236783, 107, 2717, 106], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8183, "completion_tokens": 17, "total_tokens": 8200}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 0.9868242860538885} diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.log b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.log new file mode 100644 index 00000000..e287ebe7 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.log @@ -0,0 +1,37 @@ +ATOM_BINDING {"visible_count": 1, "physical": 1, "opened": ["/dev/rbln1"]} +arithmetic-0-0 False False +passkey-0-0 False False +korean_table-0-0 False False +arithmetic-1-0 False False +passkey-1-0 False False +korean_table-1-0 False False +arithmetic-2-0 False False +passkey-2-0 False False +korean_table-2-0 False False +arithmetic-3-0 False False +passkey-3-0 False False +korean_table-3-0 False False +arithmetic-0-2048 False False +passkey-0-2048 False False +korean_table-0-2048 False False +arithmetic-1-2048 False False +passkey-1-2048 False False +korean_table-1-2048 False False +arithmetic-2-2048 False False +passkey-2-2048 False False +korean_table-2-2048 False False +arithmetic-3-2048 False False +passkey-3-2048 False False +korean_table-3-2048 False False +arithmetic-0-8192 False False +passkey-0-8192 False False +korean_table-0-8192 False False +arithmetic-1-8192 False False +passkey-1-8192 False False +korean_table-1-8192 False False +arithmetic-2-8192 False False +passkey-2-8192 False False +korean_table-2-8192 False False +arithmetic-3-8192 False False +passkey-3-8192 False False +korean_table-3-8192 False False diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.run.json b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.run.json new file mode 100644 index 00000000..f5f041c3 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-atom.run.json @@ -0,0 +1,16 @@ +{ + "stage": "complete", + "route": "native-atom", + "model": "unsloth/gemma-3-1b-it", + "runtime_commit": "3e16c33", + "torch": "2.11.0+cpu", + "tokenizer_inputs_equal": true, + "cases": 36, + "context": 9216, + "batch_size": 1, + "kv_blocks": 3, + "cache": false, + "producer_width": 8192, + "answer_correct": 0, + "format_correct": 0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-gpu-run.json b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-gpu-run.json new file mode 100644 index 00000000..1991387e --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-gpu-run.json @@ -0,0 +1,43 @@ +{ + "label": "gemma", + "route": "native-gpu", + "argv": [ + "/home/gspark/venvs/pt213/bin/python", + "-c", + "import sys,importlib.util,runpy\ns=importlib.util.spec_from_file_location('opt_prime.accel.serve.llmd','/tmp/llmd240.py')\nm=importlib.util.module_from_spec(s);sys.modules[s.name]=m;s.loader.exec_module(m)\nrunpy.run_path('/home/gspark/workspace/aicomp-dense-hybrid-258/opt_prime/examples/pd/serve_batched.py',run_name='__main__')\n", + "--model", + "/tmp/serving-matrix-models/Gemma3-1B", + "--device", + "cuda:0", + "--dtype", + "bfloat16", + "--slots", + "4", + "--max-context", + "9216", + "--max-output", + "128", + "--kv", + "paged", + "--page-size", + "64", + "--chunk", + "256", + "--port", + "40861" + ], + "env": { + "OMP_NUM_THREADS": "4", + "MKL_NUM_THREADS": "4", + "TORCH_DEVICE_BACKEND_AUTOLOAD": "0", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "PYTHONPATH": "/tmp/rngd-gpu201-src:/home/gspark/workspace/aicomp-live-pd-234/opt_prime/examples/pd/serving_matrix:/home/gspark/workspace/aicomp-dense-hybrid-258/opt_prime" + }, + "runtime_commit": "32ce5d5", + "stage": "complete", + "cases": 36, + "answer_correct": 0, + "format_correct": 0, + "server_exit": -15 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-gpu-server.log b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-gpu-server.log new file mode 100644 index 00000000..e9629633 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-gemma-native-gpu-server.log @@ -0,0 +1,3 @@ + +Loading weights: 0%| | 0/340 [00:00 + + def test_http_runner_supplies_required_request_id_and_frozen_tokens(tmp_path, monkeypatch): + import json + import sys + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + seen = [] + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + seen.append((self.headers.get('x-request-id'), body)) + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps({'choices': [{'text': '{"answer":"73"}', + 'token_ids': [9], 'finish_reason': 'stop'}]}).encode()) + def log_message(self, *args): + pass + server = HTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + fixture = tmp_path / 'input.json' + output = tmp_path / 'output.jsonl' + fixture.write_text(json.dumps({'model':'model', 'max_tokens':128, + 'cases':[{'case':'a','ids':[4,5], 'input_sha256':'hash','expected':'73'}]})) + monkeypatch.setattr(sys, 'argv', ['heldout_quality', 'run', '--fixtures',str(fixture), + '--endpoint',f'http://127.0.0.1:{server.server_port}', '--route','test', '--output',str(output)]) + try: + module().main() + finally: + server.shutdown() + thread.join() + server.server_close() +> assert seen[0][0], 'stage HTTP API requires a nonempty request ID' +E AssertionError: stage HTTP API requires a nonempty request ID +E assert None + +opt_prime/tests/test_heldout_quality.py:87: AssertionError +----------------------------- Captured stdout call ----------------------------- +a True True +=========================== short test summary info ============================ +FAILED opt_prime/tests/test_heldout_quality.py::test_http_runner_supplies_required_request_id_and_frozen_tokens +1 failed, 9 passed in 0.97s diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-fixtures.json.gz b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-fixtures.json.gz new file mode 100644 index 00000000..2b2316a7 Binary files /dev/null and b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-fixtures.json.gz differ diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.jsonl b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.jsonl new file mode 100644 index 00000000..f5d2e9c3 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.jsonl @@ -0,0 +1,36 @@ +{"case": "arithmetic-0-0", "route": "native-atom", "input_sha256": "0d5cbdfc04d8d8d4abc3243f4d3df37dac0664f1c9a910f6789cce0e8c022d6f", "input_tokens": 150, "expected": "96", "text": "{\n \"answer\": \"96\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 4161, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-0-0", "object": "text_completion", "created": 1792788772, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"96\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 4161, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 150, "completion_tokens": 10, "total_tokens": 160}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 0.8063394429627806} +{"case": "passkey-0-0", "route": "native-atom", "input_sha256": "148a205e1ed4cf9dea9bf1a236ed8d55273022522455de9786990af33647db4f", "input_tokens": 169, "expected": "34HYL6W8", "text": "{\n \"answer\": \"34HYL6W8\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 1958, 49952, 43, 21, 54, 23, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-0-0", "object": "text_completion", "created": 1792788773, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"34HYL6W8\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 1958, 49952, 43, 21, 54, 23, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 169, "completion_tokens": 15, "total_tokens": 184}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 1.1421866780146956} +{"case": "korean_table-0-0", "route": "native-atom", "input_sha256": "f7a96c6c61f14373ba3ecd21a228d373702ffd016002074ea5c76b914e787d9b", "input_tokens": 182, "expected": "춘천", "text": "{\"answer\": \"부산\"}", "tokens": [5018, 9399, 794, 330, 64189, 86157, 9388, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-0-0", "object": "text_completion", "created": 1792788773, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\"answer\": \"부산\"}", "token_ids": [5018, 9399, 794, 330, 64189, 86157, 9388, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 182, "completion_tokens": 8, "total_tokens": 190}}, "answer_correct": false, "format_correct": true, "elapsed_seconds": 0.6708941559772938} +{"case": "arithmetic-1-0", "route": "native-atom", "input_sha256": "de07f2ba6e0e0c54d00299bec55bfea0bca3115535a319781b982243ed30bc72", "input_tokens": 150, "expected": "83", "text": "{\n \"answer\": \"48 + 35 = 83\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 2166, 489, 220, 1758, 284, 220, 6069, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-1-0", "object": "text_completion", "created": 1792788775, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"48 + 35 = 83\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 2166, 489, 220, 1758, 284, 220, 6069, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 150, "completion_tokens": 16, "total_tokens": 166}}, "answer_correct": false, "format_correct": true, "elapsed_seconds": 1.2093586371047422} +{"case": "passkey-1-0", "route": "native-atom", "input_sha256": "a19a5cd366d8c5db6307c579eacddc7c523b61a88765a4b496acf7421c5f0977", "input_tokens": 167, "expected": "CNBKEFR7", "text": "{\n \"answer\": \"CNBKEFR7\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 29768, 33, 3472, 10725, 22, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-1-0", "object": "text_completion", "created": 1792788776, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"CNBKEFR7\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 29768, 33, 3472, 10725, 22, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 167, "completion_tokens": 14, "total_tokens": 181}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 1.074304271955043} +{"case": "korean_table-1-0", "route": "native-atom", "input_sha256": "46904506fbff1f58eb3caa030a441988b3fd92df83c12ebddf6ed596cd07c648", "input_tokens": 182, "expected": "목포", "text": "{\"answer\": \"목포\"}", "tokens": [5018, 9399, 794, 330, 88708, 101796, 9388, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-1-0", "object": "text_completion", "created": 1792788776, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\"answer\": \"목포\"}", "token_ids": [5018, 9399, 794, 330, 88708, 101796, 9388, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 182, "completion_tokens": 8, "total_tokens": 190}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 0.6741422610357404} +{"case": "arithmetic-2-0", "route": "native-atom", "input_sha256": "5d57407e124bd1b645bb44e6156a7e8b8774dbf31049daf0aefdf36aba6ffabf", "input_tokens": 150, "expected": "43", "text": "{\n \"answer\": \"The sum of 32 and 11 is 43.\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 791, 2694, 315, 220, 843, 323, 220, 806, 374, 220, 3391, 10246, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-2-0", "object": "text_completion", "created": 1792788778, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"The sum of 32 and 11 is 43.\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 791, 2694, 315, 220, 843, 323, 220, 806, 374, 220, 3391, 10246, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 150, "completion_tokens": 20, "total_tokens": 170}}, "answer_correct": false, "format_correct": true, "elapsed_seconds": 1.4772264510393143} +{"case": "passkey-2-0", "route": "native-atom", "input_sha256": "67df48736d5f8eed8ed87cb21f9bbf14515b4a30c7002f04e8a029e188dea2fd", "input_tokens": 167, "expected": "WMLLRJUL", "text": "{\n \"answer\": \"WMLLRJUL\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 54, 2735, 20721, 41, 1112, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-2-0", "object": "text_completion", "created": 1792788779, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"WMLLRJUL\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 54, 2735, 20721, 41, 1112, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 167, "completion_tokens": 14, "total_tokens": 181}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 1.074556282022968} +{"case": "korean_table-2-0", "route": "native-atom", "input_sha256": "603c9dece7072b82f4958eaffa237fe469ca506451e0a250bee5530a652beb51", "input_tokens": 182, "expected": "전주", "text": "{\"answer\": \"전주\"}", "tokens": [5018, 9399, 794, 330, 66965, 55430, 9388, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-2-0", "object": "text_completion", "created": 1792788780, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\"answer\": \"전주\"}", "token_ids": [5018, 9399, 794, 330, 66965, 55430, 9388, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 182, "completion_tokens": 8, "total_tokens": 190}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 0.6719808459747583} +{"case": "arithmetic-3-0", "route": "native-atom", "input_sha256": "163a31cf837d8a76b757bcfd787ed0639cca12c434e8528993bddd4c7966338f", "input_tokens": 150, "expected": "63", "text": "{\n \"answer\": \"63\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 5495, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-3-0", "object": "text_completion", "created": 1792788780, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"63\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 5495, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 150, "completion_tokens": 10, "total_tokens": 160}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 0.8066368539584801} +{"case": "passkey-3-0", "route": "native-atom", "input_sha256": "99e6bcbba020691ee22c29fbdc8eb6c89359c3e8647f833586a2805ba089cc9a", "input_tokens": 168, "expected": "SBYZEGE7", "text": "{\n \"answer\": \"SBYZEGE7\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 17094, 41309, 9560, 36, 22, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-3-0", "object": "text_completion", "created": 1792788781, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"SBYZEGE7\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 17094, 41309, 9560, 36, 22, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 168, "completion_tokens": 14, "total_tokens": 182}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 1.0747308069840074} +{"case": "korean_table-3-0", "route": "native-atom", "input_sha256": "80c3a0a96f8fc932afa2501e8fc3bd7d7c0a57b3d22c68a0fd8b20c7a1434527", "input_tokens": 182, "expected": "속초", "text": "{\"answer\": \"속초\"}", "tokens": [5018, 9399, 794, 330, 102130, 103719, 9388, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-3-0", "object": "text_completion", "created": 1792788782, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\"answer\": \"속초\"}", "token_ids": [5018, 9399, 794, 330, 102130, 103719, 9388, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 182, "completion_tokens": 8, "total_tokens": 190}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 0.6722363679436967} +{"case": "arithmetic-0-2048", "route": "native-atom", "input_sha256": "d5e3fc2d079f487aa54bf12711c278e030ae54da55f6cb57570ce489730d1260", "input_tokens": 2040, "expected": "96", "text": "{\n \"answer\": \"96\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 4161, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-0-2048", "object": "text_completion", "created": 1792788784, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"96\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 4161, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2040, "completion_tokens": 10, "total_tokens": 2050}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.243248249986209} +{"case": "passkey-0-2048", "route": "native-atom", "input_sha256": "6cb719c60a8d55b8ad05fa80978bbf003d37f93c9564b7425493cd5c951facfc", "input_tokens": 2045, "expected": "34HYL6W8", "text": "{\n \"answer\": \"34HYL6W8\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 1958, 49952, 43, 21, 54, 23, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-0-2048", "object": "text_completion", "created": 1792788787, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"34HYL6W8\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 1958, 49952, 43, 21, 54, 23, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2045, "completion_tokens": 15, "total_tokens": 2060}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.5805677130119875} +{"case": "korean_table-0-2048", "route": "native-atom", "input_sha256": "2de6ba8e4aab29c2b729c718bbee240e5b5b5f704d658667fb633ccff3950cdc", "input_tokens": 2044, "expected": "춘천", "text": "{\n \"answer\": \"춘천\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 122432, 101584, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-0-2048", "object": "text_completion", "created": 1792788789, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"춘천\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 122432, 101584, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2044, "completion_tokens": 11, "total_tokens": 2055}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.3120936829363927} +{"case": "arithmetic-1-2048", "route": "native-atom", "input_sha256": "ad0b2916c8ea5d6bbba7adeb75c78be114d5961607cf11fd4876214fe76f2cb8", "input_tokens": 2040, "expected": "83", "text": "{\n \"answer\": \"83\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 6069, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-1-2048", "object": "text_completion", "created": 1792788791, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"83\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 6069, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2040, "completion_tokens": 10, "total_tokens": 2050}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.2428762119961902} +{"case": "passkey-1-2048", "route": "native-atom", "input_sha256": "c6f1f8c084a0f7adf16c51f2b1914dfb4da2d68c8c3a245849e38b9837183118", "input_tokens": 2043, "expected": "CNBKEFR7", "text": "{\n \"answer\": \"CNBKEFR7\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 29768, 33, 3472, 10725, 22, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-1-2048", "object": "text_completion", "created": 1792788794, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"CNBKEFR7\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 29768, 33, 3472, 10725, 22, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2043, "completion_tokens": 14, "total_tokens": 2057}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.516404736088589} +{"case": "korean_table-1-2048", "route": "native-atom", "input_sha256": "a60cd701a8ef4007962836a35beef128c02ee2985b13b7fcbe1d21e7849ea514", "input_tokens": 2044, "expected": "목포", "text": "{\n \"answer\": \"목포\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 88708, 101796, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-1-2048", "object": "text_completion", "created": 1792788796, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"목포\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 88708, 101796, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2044, "completion_tokens": 11, "total_tokens": 2055}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.3110244090203196} +{"case": "arithmetic-2-2048", "route": "native-atom", "input_sha256": "5f0f4897424086c68ec9f858d7c2179b0335fded13c090f716857d5f9ba0bc41", "input_tokens": 2040, "expected": "43", "text": "{\n \"answer\": \"43\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 3391, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-2-2048", "object": "text_completion", "created": 1792788799, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"43\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 3391, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2040, "completion_tokens": 10, "total_tokens": 2050}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.2431005319813266} +{"case": "passkey-2-2048", "route": "native-atom", "input_sha256": "793cc606a21f2d953fec07a91e099d20f3624409040251105573dab8236dad2e", "input_tokens": 2043, "expected": "WMLLRJUL", "text": "{\n \"answer\": \"WMLLRJUL\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 54, 2735, 20721, 41, 1112, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-2-2048", "object": "text_completion", "created": 1792788801, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"WMLLRJUL\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 54, 2735, 20721, 41, 1112, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2043, "completion_tokens": 14, "total_tokens": 2057}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.5119675419991836} +{"case": "korean_table-2-2048", "route": "native-atom", "input_sha256": "adec171bcbf4600c8d59467a432af029bc475ce9575c1f4817eece553e37a027", "input_tokens": 2044, "expected": "전주", "text": "{\n \"answer\": \"전주\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 66965, 55430, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-2-2048", "object": "text_completion", "created": 1792788803, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"전주\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 66965, 55430, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2044, "completion_tokens": 11, "total_tokens": 2055}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.3103487459011376} +{"case": "arithmetic-3-2048", "route": "native-atom", "input_sha256": "679e19177db7acb1a7dd097a45e30f3e005511311577f7e9e9add8b11c6e9933", "input_tokens": 2040, "expected": "63", "text": "{\n \"answer\": \"63\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 5495, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-3-2048", "object": "text_completion", "created": 1792788806, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"63\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 5495, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2040, "completion_tokens": 10, "total_tokens": 2050}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.2475822180276737} +{"case": "passkey-3-2048", "route": "native-atom", "input_sha256": "f4347eded9d6462727022c7ab97a9f961d43f4c44828ff7cf901fcd2a7e05510", "input_tokens": 2044, "expected": "SBYZEGE7", "text": "{\n \"answer\": \"SBYZEGE7\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 17094, 41309, 9560, 36, 22, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-3-2048", "object": "text_completion", "created": 1792788808, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"SBYZEGE7\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 17094, 41309, 9560, 36, 22, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2044, "completion_tokens": 14, "total_tokens": 2058}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.512888388009742} +{"case": "korean_table-3-2048", "route": "native-atom", "input_sha256": "6d84fc95ed8b4b55fda6af31a18bb7c32b6f19ab02d745fddd56f5ae5d74efb4", "input_tokens": 2044, "expected": "속초", "text": "{\n \"answer\": \"속초\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 102130, 103719, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-3-2048", "object": "text_completion", "created": 1792788810, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"속초\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 102130, 103719, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 2044, "completion_tokens": 11, "total_tokens": 2055}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 2.3106573990080506} +{"case": "arithmetic-0-8192", "route": "native-atom", "input_sha256": "b5ac979ed58c0020fb09bed6e865da33798778a4e3f94b9bbd5608832c418e7b", "input_tokens": 8186, "expected": "96", "text": "{\n \"answer\": \"96\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 4161, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-0-8192", "object": "text_completion", "created": 1792788818, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"96\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 4161, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8186, "completion_tokens": 10, "total_tokens": 8196}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 7.9506545721087605} +{"case": "passkey-0-8192", "route": "native-atom", "input_sha256": "d5cd6c30057ebdf9e65495c4a7bd757f811bf4e64860188c7357a67414f94222", "input_tokens": 8191, "expected": "34HYL6W8", "text": "34HYL6W8", "tokens": [1958, 49952, 43, 21, 54, 23, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-0-8192", "object": "text_completion", "created": 1792788826, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "34HYL6W8", "token_ids": [1958, 49952, 43, 21, 54, 23, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8191, "completion_tokens": 7, "total_tokens": 8198}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 7.733989814994857} +{"case": "korean_table-0-8192", "route": "native-atom", "input_sha256": "cb0f1c7ae26517f1e9d859e6d0d24aed6e4dc2cd4d3db9fb4983c5a01d71f745", "input_tokens": 8190, "expected": "춘천", "text": "{\n \"answer\": \"춘천\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 122432, 101584, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-0-8192", "object": "text_completion", "created": 1792788834, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"춘천\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 122432, 101584, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8190, "completion_tokens": 11, "total_tokens": 8201}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 8.020284185069613} +{"case": "arithmetic-1-8192", "route": "native-atom", "input_sha256": "41e87fb89c71cd493cec4801e39fccc4f625e205a485f48b1310880c5e72173f", "input_tokens": 8186, "expected": "83", "text": "{\n \"answer\": \"83\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 6069, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-1-8192", "object": "text_completion", "created": 1792788842, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"83\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 6069, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8186, "completion_tokens": 10, "total_tokens": 8196}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 7.948859789059497} +{"case": "passkey-1-8192", "route": "native-atom", "input_sha256": "6aec4ccde3c8e068a525f3141930fd0dd89b0066fb57ae44bf1f3695d03a9d38", "input_tokens": 8189, "expected": "CNBKEFR7", "text": "{\n \"answer\": \"CNBKEFR7\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 29768, 33, 3472, 10725, 22, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-1-8192", "object": "text_completion", "created": 1792788850, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"CNBKEFR7\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 29768, 33, 3472, 10725, 22, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8189, "completion_tokens": 14, "total_tokens": 8203}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 8.235838730004616} +{"case": "korean_table-1-8192", "route": "native-atom", "input_sha256": "a45e928bc070aa02a52f8f8c09fe63eb2c240804cb082b8d38395f45f19e2ac7", "input_tokens": 8190, "expected": "목포", "text": "{\n \"answer\": \"목포\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 88708, 101796, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-1-8192", "object": "text_completion", "created": 1792788858, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"목포\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 88708, 101796, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8190, "completion_tokens": 11, "total_tokens": 8201}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 8.021923388005234} +{"case": "arithmetic-2-8192", "route": "native-atom", "input_sha256": "3b2113bbe36b0528ec715ac215ad332254fdcc741b55bba2fad14beb172ce03c", "input_tokens": 8186, "expected": "43", "text": "{\n \"answer\": \"43\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 3391, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-2-8192", "object": "text_completion", "created": 1792788866, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"43\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 3391, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8186, "completion_tokens": 10, "total_tokens": 8196}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 7.9510668410221115} +{"case": "passkey-2-8192", "route": "native-atom", "input_sha256": "e8784fdb10cbfc3bc05540d0a7c9cad53c915638b3887a205ea7df911b53db53", "input_tokens": 8189, "expected": "WMLLRJUL", "text": "WMLLRJUL", "tokens": [54, 2735, 20721, 41, 1112, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-2-8192", "object": "text_completion", "created": 1792788874, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "WMLLRJUL", "token_ids": [54, 2735, 20721, 41, 1112, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8189, "completion_tokens": 6, "total_tokens": 8195}}, "answer_correct": false, "format_correct": false, "elapsed_seconds": 7.665244259987958} +{"case": "korean_table-2-8192", "route": "native-atom", "input_sha256": "e2a1f3ecbfbde65119093f3e14d982ffe15fc04ff0f46e6dfb0a73aafb6bc301", "input_tokens": 8190, "expected": "전주", "text": "{\n \"answer\": \"전주\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 66965, 55430, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-2-8192", "object": "text_completion", "created": 1792788882, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"전주\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 66965, 55430, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8190, "completion_tokens": 11, "total_tokens": 8201}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 8.025514162960462} +{"case": "arithmetic-3-8192", "route": "native-atom", "input_sha256": "9649667fd3e0e3f487ddf07287c5f43d01c9d0a4545f6fb163c215464c0ba727", "input_tokens": 8186, "expected": "63", "text": "{\n \"answer\": \"63\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 5495, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-arithmetic-3-8192", "object": "text_completion", "created": 1792788890, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"63\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 5495, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8186, "completion_tokens": 10, "total_tokens": 8196}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 7.951001578941941} +{"case": "passkey-3-8192", "route": "native-atom", "input_sha256": "e2301ac0d434cc1b0872c0ae4240fa21c7a89cda9a5727ff7adfbb66a6757fb3", "input_tokens": 8190, "expected": "SBYZEGE7", "text": "{\n \"answer\": \"SBYZEGE7\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 17094, 41309, 9560, 36, 22, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-passkey-3-8192", "object": "text_completion", "created": 1792788898, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"SBYZEGE7\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 17094, 41309, 9560, 36, 22, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8190, "completion_tokens": 14, "total_tokens": 8204}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 8.236702271969989} +{"case": "korean_table-3-8192", "route": "native-atom", "input_sha256": "ddf60d0902dc767a5b12fa8c67ed1c79a410df9d76a63d7fcccd2f21b5807ce7", "input_tokens": 8190, "expected": "속초", "text": "{\n \"answer\": \"속초\"\n}", "tokens": [517, 220, 330, 9399, 794, 330, 102130, 103719, 702, 92, 128009], "finish_reason": "stop", "response": {"id": "quality263-korean_table-3-8192", "object": "text_completion", "created": 1792788906, "model": "/tmp/serving-matrix-models/KONI-8B", "choices": [{"index": 0, "text": "{\n \"answer\": \"속초\"\n}", "token_ids": [517, 220, 330, 9399, 794, 330, 102130, 103719, 702, 92, 128009], "finish_reason": "stop"}], "usage": {"prompt_tokens": 8190, "completion_tokens": 11, "total_tokens": 8201}}, "answer_correct": true, "format_correct": true, "elapsed_seconds": 8.019978774012998} diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.log b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.log new file mode 100644 index 00000000..71c15b62 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.log @@ -0,0 +1,38 @@ +ATOM_BINDING {"visible_count": 1, "physical": 1, "opened": ["/dev/rbln1"]} +[transformers] Ignoring clean_up_tokenization_spaces=True for BPE tokenizer TokenizersBackend. The clean_up_tokenization post-processing step is designed for WordPiece tokenizers and is destructive for BPE (it strips spaces before punctuation). Set clean_up_tokenization_spaces=False to suppress this warning, or set clean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_output=True to force cleanup anyway. +arithmetic-0-0 True True +passkey-0-0 True True +korean_table-0-0 False True +arithmetic-1-0 False True +passkey-1-0 True True +korean_table-1-0 True True +arithmetic-2-0 False True +passkey-2-0 True True +korean_table-2-0 True True +arithmetic-3-0 True True +passkey-3-0 True True +korean_table-3-0 True True +arithmetic-0-2048 True True +passkey-0-2048 True True +korean_table-0-2048 True True +arithmetic-1-2048 True True +passkey-1-2048 True True +korean_table-1-2048 True True +arithmetic-2-2048 True True +passkey-2-2048 True True +korean_table-2-2048 True True +arithmetic-3-2048 True True +passkey-3-2048 True True +korean_table-3-2048 True True +arithmetic-0-8192 True True +passkey-0-8192 False False +korean_table-0-8192 True True +arithmetic-1-8192 True True +passkey-1-8192 True True +korean_table-1-8192 True True +arithmetic-2-8192 True True +passkey-2-8192 False False +korean_table-2-8192 True True +arithmetic-3-8192 True True +passkey-3-8192 True True +korean_table-3-8192 True True diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.run.json b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.run.json new file mode 100644 index 00000000..cf276b88 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-atom.run.json @@ -0,0 +1,16 @@ +{ + "stage": "complete", + "route": "native-atom", + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "runtime_commit": "3e16c33", + "torch": "2.11.0+cpu", + "tokenizer_inputs_equal": true, + "cases": 36, + "context": 9216, + "batch_size": 1, + "kv_blocks": 1, + "cache": false, + "producer_width": 8192, + "answer_correct": 31, + "format_correct": 34 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-gpu-run.json b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-gpu-run.json new file mode 100644 index 00000000..34e8441c --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-gpu-run.json @@ -0,0 +1,43 @@ +{ + "label": "koni", + "route": "native-gpu", + "argv": [ + "/home/gspark/venvs/pt213/bin/python", + "-c", + "import sys,importlib.util,runpy\ns=importlib.util.spec_from_file_location('opt_prime.accel.serve.llmd','/tmp/llmd240.py')\nm=importlib.util.module_from_spec(s);sys.modules[s.name]=m;s.loader.exec_module(m)\nrunpy.run_path('/home/gspark/workspace/aicomp-atom-prefetch-253/opt_prime/examples/pd/serve_batched.py',run_name='__main__')\n", + "--model", + "/tmp/serving-matrix-models/KONI-8B", + "--device", + "cuda:0", + "--dtype", + "bfloat16", + "--slots", + "4", + "--max-context", + "9216", + "--max-output", + "128", + "--kv", + "paged", + "--page-size", + "64", + "--chunk", + "256", + "--port", + "41385" + ], + "env": { + "OMP_NUM_THREADS": "4", + "MKL_NUM_THREADS": "4", + "TORCH_DEVICE_BACKEND_AUTOLOAD": "0", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "PYTHONPATH": "/tmp/rngd-gpu201-src:/home/gspark/workspace/aicomp-live-pd-234/opt_prime/examples/pd/serving_matrix:/home/gspark/workspace/aicomp-dense-hybrid-258/opt_prime" + }, + "runtime_commit": "32ce5d5", + "stage": "complete", + "cases": 36, + "answer_correct": 31, + "format_correct": 34, + "server_exit": -15 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-gpu-server.log b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-gpu-server.log new file mode 100644 index 00000000..890b0319 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/quality263-koni-native-gpu-server.log @@ -0,0 +1,4 @@ + +Loading weights: 0%| | 0/291 [00:00deadline: raise TimeoutError('owner startup') + time.sleep(.01) + result['stage']='running';save() + subprocess.run([sys.executable,'heldout_quality263.py','run','--fixtures',a.fixture, + '--endpoint',f'http://127.0.0.1:{server.server_port}','--route',a.route,'--output',a.output],check=True) + rows=[json.loads(line) for line in Path(a.output).read_text().splitlines()] + assert len(rows)==36 and all('error' not in row for row in rows) + deadline=time.monotonic()+30 + while not b.idle and time.monotonic()deadline: + raise TimeoutError('readiness timeout') + time.sleep(2) + subprocess.run([sys.executable,script,'run','--fixtures',str(fixture), + '--endpoint',endpoint,'--route',route,'--output',str(output)],env=env,check=True) + rows=[json.loads(line) for line in output.read_text().splitlines()] + assert len(rows)==36 and all('error' not in row for row in rows) + metadata.update(stage='complete',cases=36,answer_correct=sum(x['answer_correct'] for x in rows),format_correct=sum(x['format_correct'] for x in rows)) + except BaseException as error: + metadata.update(stage='failed',error=repr(error)) + raise + finally: + if proc.poll() is None: + os.killpg(proc.pid,signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(proc.pid,signal.SIGKILL) + proc.wait() + metadata['server_exit']=proc.returncode + meta.write_text(json.dumps(metadata,indent=2)+'\n') + print('END',label,route,metadata['answer_correct'],metadata['format_correct'],flush=True) + time.sleep(3) diff --git a/opt_prime/examples/pd/http_matrix/evidence263/summarize_quality263.py b/opt_prime/examples/pd/http_matrix/evidence263/summarize_quality263.py new file mode 100644 index 00000000..a21dbcbf --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence263/summarize_quality263.py @@ -0,0 +1,64 @@ +"""Audit complete runs against frozen fixtures; preserve pending/failed routes explicitly.""" +from itertools import combinations +import hashlib +import importlib.util +import json +from pathlib import Path + +root=Path(__file__).resolve().parent +spec=importlib.util.spec_from_file_location('quality263',Path(__file__).parents[1]/'heldout_quality.py') +quality=importlib.util.module_from_spec(spec);spec.loader.exec_module(quality) +manifest=json.loads((root/'fixture-manifest.json').read_text()) +report={'models':{},'limits':'36 synthetic structured-answer cases/model; no general quality equivalence or performance claim. Strict historical token failures remain.'} +for model in ['qwen06b','qwen4b','koni','gemma']: + fixture=root/f'quality263-{model}-fixtures.json' + assert hashlib.sha256(fixture.read_bytes()).hexdigest()==manifest[fixture.name]['sha256'] + cases={c['case']:c for c in json.loads(fixture.read_text())['cases']} + assert len(cases)==36 + model_result={'routes':{},'pairs':{}} + completed={} + for route in ['native-gpu','rngd-gpu','native-atom','rngd-atom']: + stem=f'quality263-{model}-{route}' + meta=root/(stem+('-run.json' if 'gpu' in route else '.run.json')) + if not meta.exists(): + model_result['routes'][route]={'stage':'not_collected'};continue + metadata=json.loads(meta.read_text()) + if metadata['stage']!='complete': + model_result['routes'][route]={'stage':metadata['stage'],'error':metadata.get('error')};continue + rows=[json.loads(x) for x in (root/(stem+'.jsonl')).read_text().splitlines()] + assert len(rows)==36 and {r['case'] for r in rows}==cases.keys() + for row in rows: + case=cases[row['case']] + assert 'error' not in row + assert row['input_sha256']==case['input_sha256'] + assert row['input_tokens']==len(case['ids']) and row['expected']==case['expected'] + assert isinstance(row['tokens'],list) and all(type(t) is int for t in row['tokens']) + assert 0 got = module().score(text, '73') + +opt_prime/tests/test_heldout_quality.py:25: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +______________ test_score[{"answer":"73","note":"ok"}-True-False] ______________ + +text = '{"answer":"73","note":"ok"}', correct = True, formatted = False + + @pytest.mark.parametrize('text,correct,formatted', [ + ('{"answer":"73"}', True, True), + ('{"answer":"73","note":"ok"}', True, False), + ('{"answer":73}', False, False), + ('The answer is 73', False, False), + ('{"answer":"173"}', False, True), + ('{"answer":"wrong","answer":"73"}', False, False), + ('{"answer":"73"} trailing', False, False), + ]) + def test_score(text, correct, formatted): +> got = module().score(text, '73') + +opt_prime/tests/test_heldout_quality.py:25: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +____________________ test_score[{"answer":73}-False-False] _____________________ + +text = '{"answer":73}', correct = False, formatted = False + + @pytest.mark.parametrize('text,correct,formatted', [ + ('{"answer":"73"}', True, True), + ('{"answer":"73","note":"ok"}', True, False), + ('{"answer":73}', False, False), + ('The answer is 73', False, False), + ('{"answer":"173"}', False, True), + ('{"answer":"wrong","answer":"73"}', False, False), + ('{"answer":"73"} trailing', False, False), + ]) + def test_score(text, correct, formatted): +> got = module().score(text, '73') + +opt_prime/tests/test_heldout_quality.py:25: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +___________________ test_score[The answer is 73-False-False] ___________________ + +text = 'The answer is 73', correct = False, formatted = False + + @pytest.mark.parametrize('text,correct,formatted', [ + ('{"answer":"73"}', True, True), + ('{"answer":"73","note":"ok"}', True, False), + ('{"answer":73}', False, False), + ('The answer is 73', False, False), + ('{"answer":"173"}', False, True), + ('{"answer":"wrong","answer":"73"}', False, False), + ('{"answer":"73"} trailing', False, False), + ]) + def test_score(text, correct, formatted): +> got = module().score(text, '73') + +opt_prime/tests/test_heldout_quality.py:25: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +___________________ test_score[{"answer":"173"}-False-True] ____________________ + +text = '{"answer":"173"}', correct = False, formatted = True + + @pytest.mark.parametrize('text,correct,formatted', [ + ('{"answer":"73"}', True, True), + ('{"answer":"73","note":"ok"}', True, False), + ('{"answer":73}', False, False), + ('The answer is 73', False, False), + ('{"answer":"173"}', False, True), + ('{"answer":"wrong","answer":"73"}', False, False), + ('{"answer":"73"} trailing', False, False), + ]) + def test_score(text, correct, formatted): +> got = module().score(text, '73') + +opt_prime/tests/test_heldout_quality.py:25: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +___________ test_score[{"answer":"wrong","answer":"73"}-False-False] ___________ + +text = '{"answer":"wrong","answer":"73"}', correct = False, formatted = False + + @pytest.mark.parametrize('text,correct,formatted', [ + ('{"answer":"73"}', True, True), + ('{"answer":"73","note":"ok"}', True, False), + ('{"answer":73}', False, False), + ('The answer is 73', False, False), + ('{"answer":"173"}', False, True), + ('{"answer":"wrong","answer":"73"}', False, False), + ('{"answer":"73"} trailing', False, False), + ]) + def test_score(text, correct, formatted): +> got = module().score(text, '73') + +opt_prime/tests/test_heldout_quality.py:25: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +_______________ test_score[{"answer":"73"} trailing-False-False] _______________ + +text = '{"answer":"73"} trailing', correct = False, formatted = False + + @pytest.mark.parametrize('text,correct,formatted', [ + ('{"answer":"73"}', True, True), + ('{"answer":"73","note":"ok"}', True, False), + ('{"answer":73}', False, False), + ('The answer is 73', False, False), + ('{"answer":"173"}', False, True), + ('{"answer":"wrong","answer":"73"}', False, False), + ('{"answer":"73"} trailing', False, False), + ]) + def test_score(text, correct, formatted): +> got = module().score(text, '73') + +opt_prime/tests/test_heldout_quality.py:25: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +__________ test_paired_regressions_are_not_hidden_by_equal_aggregate ___________ + + def test_paired_regressions_are_not_hidden_by_equal_aggregate(): +> m = module() + +opt_prime/tests/test_heldout_quality.py:29: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +___________ test_context_builder_keeps_retrieval_fact_inside_budget ____________ + + def test_context_builder_keeps_retrieval_fact_inside_budget(): + class CharacterTokenizer: + def apply_chat_template(self, messages, **kwargs): + return list(messages[0]['content'].encode()) +> m = module() + +opt_prime/tests/test_heldout_quality.py:46: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def module(): +> assert path.exists(), 'held-out evaluator is not implemented' +E AssertionError: held-out evaluator is not implemented +E assert False +E + where False = exists() +E + where exists = PosixPath('/home/gspark/workspace/aicomp-heldout-quality-263/opt_prime/examples/pd/http_matrix/heldout_quality.py').exists + +opt_prime/tests/test_heldout_quality.py:9: AssertionError +=========================== short test summary info ============================ +FAILED opt_prime/tests/test_heldout_quality.py::test_score[{"answer":"73"}-True-True] +FAILED opt_prime/tests/test_heldout_quality.py::test_score[{"answer":"73","note":"ok"}-True-False] +FAILED opt_prime/tests/test_heldout_quality.py::test_score[{"answer":73}-False-False] +FAILED opt_prime/tests/test_heldout_quality.py::test_score[The answer is 73-False-False] +FAILED opt_prime/tests/test_heldout_quality.py::test_score[{"answer":"173"}-False-True] +FAILED opt_prime/tests/test_heldout_quality.py::test_score[{"answer":"wrong","answer":"73"}-False-False] +FAILED opt_prime/tests/test_heldout_quality.py::test_score[{"answer":"73"} trailing-False-False] +FAILED opt_prime/tests/test_heldout_quality.py::test_paired_regressions_are_not_hidden_by_equal_aggregate +FAILED opt_prime/tests/test_heldout_quality.py::test_context_builder_keeps_retrieval_fact_inside_budget +9 failed in 0.04s diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-24-heldout-quality263.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-24-heldout-quality263.md new file mode 100644 index 00000000..7fd75655 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-24-heldout-quality263.md @@ -0,0 +1,107 @@ +# Held-out structured answers across four model families + +- **Date**: 2026-10-24 +- **Task**: Structured answer compatibility (aicomp #263) +- **Host / device**: GPU and RNGD+ATOM roles; hardware results pending +- **Commit**: evaluator 7b2fd19, draft PR264 (base f9ce6e8); production runtime 32ce5d5 + +## Question + +Do token differences change structured-answer correctness across native GPU, RNGD→GPU, native ATOM, and RNGD→ATOM? Models: Qwen3-0.6B, Qwen3-4B, KONI-8B, Gemma3-1B. Qwen2.5 excluded at user request. + +## Setup + +Preregistered before new inference: seed 26320261024; three tasks (addition, passkey retrieval, Korean table lookup), four variants, short/~2048/~8192 input tokens = 36 cases per model. Same serialized chat-template input IDs for every route, thinking disabled, greedy, normal EOS, output cap 128. Retrieval fact is in the middle of deterministic filler. Context sizes include the chat template. No answer-conditioned prompt edits. + +Exact string answer in a parsed JSON object is scored separately from the one-key JSON format requirement. Duplicate JSON keys, non-string answers, trailing prose, and malformed JSON fail. This is structured-answer adherence, not a universal semantic scorer. Retain complete responses and token IDs; compare paired correct→wrong, wrong→correct, and exact tokens independently. Any transport or inference failure is retained and stops the route; do not silently exclude it. Fixture hashes identify paired inputs. + +## Command + +```bash +# In the isolated aicomp worktree: +python -m pytest opt_prime/tests/test_heldout_quality.py -q +# Create each fixture file once, before inference: +python opt_prime/examples/pd/http_matrix/heldout_quality.py fixtures --model "$MODEL" --output "$FIXTURE" +python opt_prime/examples/pd/http_matrix/heldout_quality.py run --fixtures "$FIXTURE" --endpoint "$ENDPOINT" --route "$ROUTE" --output "$RESULT" +``` + +Local Python: /home/gspark/venvs/pt213/bin/python. Full-suite environment: OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 TORCH_DEVICE_BACKEND_AUTOLOAD=0. + +## Results + +- Test-first: 9 failures for missing evaluator, then 9 passed in 0.43 seconds. +- HTTP boundary test caught missing request ID: 1 failed/9 passed before fix, then 10 passed. +- Full suite collected before that last test: 533 passed, 15 skipped, 253 subtests in 160.17 s; 13 warnings from existing pinned-memory deprecations and unfused flex attention. +- All 144 input cases frozen and SHA256 manifest saved before inference. GPU paired runs started; no new model accuracy or performance claim. +- Existing Qwen3-4B/KONI exact-token failures remain open (#256/#261). + +Raw logs: /tmp/quality263-red.log, /tmp/quality263-full.log; fixtures/results under analysis/serving-matrix-201/quality263-*. + +## Conclusion + +This small synthetic set can expose paired regressions but cannot establish general model quality or statistical equivalence. The compatibility preference question remains unanswered; exact token failures are not waived. + +## First completed paired run + +Qwen3-0.6B, 36 paired cases: nativeGPU32correct/36validformat; RNGD→GPU34correct/36validformat. Correct→wrong0, wrong→correct2; exacttoken34/36. This is a structured synthetic task result, not a general model quality advantage. Raw: quality263-qwen06b-{native-gpu,rngd-gpu}.jsonl and quality263-qwen06b-gpu-summary.json under analysis/serving-matrix-201. Other models remain in the active sequential queue. + +## ATOM execution + +Owned quality263 Pod allocated one ATOM and one RNGD through device plugins; no existing Pod modified. All36 stored prompts are re-tokenized with the remote model tokenizer and asserted equal to frozen input IDs before inference. B1, context9216, cacheoff, normalEOS, max128output; nativeATOM and liveRNGD→ATOM run sequentially with existing compiled artifacts. Runtime3e16c33; kv_blocks3 except KONI1; Gemma global225 artifacts. Queue deletes only its own Pod in finally. + +Local exact orchestrator commands: +```bash +/home/gspark/venvs/pt213/bin/python analysis/serving-matrix-201/run_quality263_gpu.py > analysis/serving-matrix-201/quality263-gpu-queue.log 2>&1 +/home/gspark/venvs/pt213/bin/python analysis/serving-matrix-201/run_quality263_atom_queue.py > analysis/serving-matrix-201/quality263-atom-queue.log 2>&1 +``` +Per-route argv/environment and raw HTTP outputs are retained; first-stage ATOM tokenizer equality passed. No final ATOM scores yet. + +Qwen3-4B GPU pair completed:36/36 answers and36/36 JSON format on both routes; exacttoken36/36; correct→wrong0 and wrong→correct0. Prior long-free-generation mismatch in#256 remains open. + +## GPU pair completion and provenance + +All288 GPU-route responses completed, no HTTP/inference failures; owned local GPU/RNGD processes exited and devices idle. + +| Model | NativeGPU structured score | RNGD→GPU score | Exact token pairs | Correct→wrong / wrong→correct | +|---|---:|---:|---:|---:| +| Qwen3-0.6B |32/36|34/36|34/36|0 / 2| +| Qwen3-4B |36/36|36/36|36/36|0 / 0| +| KONI-8B |31/36|31/36|35/36|0 / 0| +| Gemma3-1B |0/36|0/36|32/36|0 / 0| + +Gemma's zero score measures strict requested JSON/string-answer adherence, not zero content accuracy: code fences are emitted by both routes. Manual comparison of all four differing pairs: three differ only in whitespace; one Korean lookup changes native`null` to correct`춘천` on RNGD→GPU. The frozen primary scorer is unchanged. KONI's one changed pair differs only in JSON whitespace. Historical long-output strict-token failures remain open. + +ATOM partial completion: Qwen0.6 native33/36 and RNGD33/36 structured answers,36/36 exact token pairs; Qwen4 native36/36 and identical to its GPU outputs. Qwen4 RNGD→ATOM and both KONI/Gemma ATOM routes remain pending. + +Both hosts' complete source weight shards, config.json, generation_config.json and tokenizer.json were hashed: all21 files (~27.67GB per host) SHA256-identical across four models. Source revisions and individual hashes are retained in quality263-{local,remote}-model-hashes.json and quality263-model-identity.json. This checks checkpoint source identity, not compiled-artifact internals. + +Auditor summarize_quality263.py verifies fixture-file hashes, case/input-hash correspondence, response fields, scores recomputed from raw text, and paired transitions. Pending routes remain explicitly not_collected/starting; they are not treated as passes. + +Qwen3-4B allfour routes completed: each36/36 structured answers and36/36 format, every paired token sequence identical acrossGPU/RNGD/ATOM. Remaining ATOM routes: KONI and Gemma. This does not erase the prior different long-free-generation fixture failure#256. + +## Full pairwise audit (primary prompts/scorer unchanged) + +Compare all six unordered route pairs, not only pairs anchored on nativeGPU. This exposes a Qwen0.6 counterexample: passkey-3-2048 is correct`SBYZEGE7` on RNGD→GPU, but nativeATOM and RNGD→ATOM emit`SYBZEGE7`. Thus each ATOM route has one correct→wrong transition relative to RNGD→GPU, despite no such transition relative to nativeGPU (which also answered this case incorrectly). Do not claim universal cross-route quality equivalence or infer the failing operator without matched-KV controls. + +KONI allfour routes completed:31/36 structured answers each; nativeGPU/nativeATOM36/36 exacttokens; RNGD→GPU/RNGD→ATOM36/36 exacttokens. Native vsRNGD pair35/36; the differing text only changes JSON whitespace. Gemma nativeATOM completed with code-fenced responses (strictscore0); RNGD→ATOM remains running. + +## Final completion + +All576 responses (4models×4routes×36cases) completed with normal EOS and no length-capped outputs or HTTP/inference failures. Frozen fixture hashes and all returned case/input hashes verified; exact parsed-answer scores recomputed from raw text. OwnedPod quality263 deletion independently confirmed absent. + +| Model | nativeGPU | RNGD→GPU | nativeATOM | RNGD→ATOM | +|---|---:|---:|---:|---:| +| Qwen3-0.6B |32/36|34/36|33/36|33/36| +| Qwen3-4B |36/36|36/36|36/36|36/36| +| KONI-8B |31/36|31/36|31/36|31/36| +| Gemma3-1B |0/36|0/36|0/36|0/36| + +These are strict structured-answer adherence counts, not unrestricted semantic accuracy. Gemma emits Markdown code fences on every route. Its final ATOM native/PD pair is token-exact36/36; GPU→ATOM comparison still changes content: korean_table-0-2048 yields correct`춘천` on RNGD→GPU but`null` on RNGD→ATOM and nativeATOM. This is in addition to the Qwen0.6 passkey counterexample. Neither counterexample can be blamed on a particular operator or transport without matched-KV controls. + +NativeATOM/RNGD→ATOM exact pairs: Q06 36/36, Q4 36/36, KONI35/36 (JSON whitespace only), Gemma36/36. All six route-pair comparisons, per-context counts and changed raw texts are in quality263-summary.json. No universal quality-equivalence or performance-speedup conclusion. Numerical issues#233/#256/#261 and actual within-request streaming#265 remain separate work. + +## Post-hoc content diagnostic (primary criteria unchanged) + +No new inference. All16 input JSONL files are byte-identical to published PR264 evidence atc28d056. `python3 analysis/serving-matrix-201/analyze_quality263_content.py analysis/serving-matrix-201 > analysis/serving-matrix-201/quality263-content-diagnostic.json` checks all576 frozen records, case/input hashes/expected answers across all six route pairs. Separate exploratory metric removes only a whole outer JSON/unlabelled code fence and accepts integer arithmetic answers as their exact decimal string. No substring matching, float/bool/null coercion, case folding or extra-key acceptance; duplicate JSON keys rejected. Embedded self-checks accept the intended relaxation and reject malformed/ambiguous/over-permissive cases. Rerun output is byte-identical. These rules were selected after seeing outputs and must not replace the frozen primary scores or waive earlier exactness failures. + +Route order: nativeGPU / RNGD-GPU / nativeATOM / RNGD-ATOM. Exploratory content scores outof36: Qwen0.6=32/34/33/33; Qwen4=36/36/36/36; KONI=31/31/31/31; Gemma=29/30/29/29. Gemma primary remains0/0/0/0. All36 Gemma responses per route have outer fences;8 per route additionally use integer arithmetic answers. Thus formatting masks substantial content correctness but also genuine6–7 content errors. Exactly one Gemma case differs in normalized answer between RNGD-GPU and the other routes, the already-retained Korean city lookup; Qwen0.6 retains its prior content differences. KONI/Qwen4 normalized answers match across allsix pairs on these fixtures only. No operator or transport root-cause claim. diff --git a/opt_prime/examples/pd/http_matrix/heldout_quality.py b/opt_prime/examples/pd/http_matrix/heldout_quality.py new file mode 100644 index 00000000..af1ab56e --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/heldout_quality.py @@ -0,0 +1,133 @@ +"""Preregistered synthetic structured-answer checks; not a general quality benchmark.""" +import argparse +import hashlib +import json +import random +import time +import urllib.request +from pathlib import Path + + +def score(text, expected): + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate JSON key') + result[key] = value + return result + try: + value = json.loads(text, object_pairs_hook=unique) + except (ValueError, TypeError): + value = None + valid = isinstance(value, dict) and isinstance(value.get('answer'), str) + return dict(answer_correct=bool(valid and value['answer'] == expected), + format_correct=bool(valid and set(value) == {'answer'})) + + +def compare(baseline, candidate): + left = {x['case']: x for x in baseline} + right = {x['case']: x for x in candidate} + if len(left) != len(baseline) or len(right) != len(candidate) or left.keys() != right.keys(): + raise ValueError('duplicate or missing paired cases') + result = dict(cases=len(left), correct_to_wrong=0, wrong_to_correct=0, token_equal=0) + for key, a in left.items(): + b = right[key] + if a['input_sha256'] != b['input_sha256']: + raise ValueError('paired input IDs differ') + result['correct_to_wrong'] += bool(a['answer_correct'] and not b['answer_correct']) + result['wrong_to_correct'] += bool(not a['answer_correct'] and b['answer_correct']) + result['token_equal'] += a['tokens'] == b['tokens'] + return result + + +def fixtures(tokenizer): + rng = random.Random(26320261024) + tasks = [] + for variant in range(4): + a, b = rng.randrange(21, 80), rng.randrange(11, 60) + tasks.append(('arithmetic', variant, '', f'Compute {a} + {b}.', str(a + b))) + secret = ''.join(rng.choice('ABCDEFGHJKLMNPQRSTUVWXYZ23456789') for _ in range(8)) + tasks.append(('passkey', variant, f'The access key for archive {variant} is {secret}.', + f'What is the access key for archive {variant}?', secret)) + city = ['춘천', '목포', '전주', '속초'][variant] + tasks.append(('korean_table', variant, + f'배송표: 주문 가{variant}=대전; 주문 나{variant}={city}; 주문 다{variant}=부산.', + f'배송표에서 주문 나{variant}의 도착 도시를 찾으세요.', city)) + filler = 'Background: shelves hold ordinary books; this note contains no task answer.\n' + result = [] + for target in [0, 2048, 8192]: + for task, variant, fact, question, expected in tasks: + def render(repeats): + prompt = (filler * (repeats // 2) + fact + '\n' + filler * (repeats - repeats // 2) + + '\n' + question + '\nReturn only a JSON object with one key "answer" and a string value. No explanation.') + ids = tokenizer.apply_chat_template([{'role': 'user', 'content': prompt}], + tokenize=True, return_dict=False, add_generation_prompt=True, enable_thinking=False) + return prompt, [int(x) for x in ids] + low, high = 0, target + if target and len(render(0)[1]) > target: + raise ValueError('base prompt exceeds context target') + while low < high: + mid = (low + high + 1) // 2 + if len(render(mid)[1]) <= target: + low = mid + else: + high = mid - 1 + prompt, ids = render(low) + digest = hashlib.sha256(json.dumps(ids, separators=(',', ':')).encode()).hexdigest() + result.append(dict(case=f'{task}-{variant}-{target}', task=task, target_tokens=target, + expected=expected, prompt=prompt, ids=ids, input_sha256=digest)) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + export = sub.add_parser('fixtures') + export.add_argument('--model', required=True) + export.add_argument('--output', type=Path, required=True) + run = sub.add_parser('run') + run.add_argument('--fixtures', type=Path, required=True) + run.add_argument('--endpoint', required=True) + run.add_argument('--route', required=True) + run.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if args.command == 'fixtures': + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) + data = dict(model=args.model, seed=26320261024, max_tokens=128, + thinking=False, cases=fixtures(tokenizer)) + args.output.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') + return + data = json.loads(args.fixtures.read_text()) + # Exclusive creation prevents silently replacing a prior run after seeing answers. + with args.output.open('x') as output: + for case in data['cases']: + body = dict(model=data['model'], prompt=case['ids'], temperature=0, + max_tokens=data['max_tokens'], ignore_eos=False, stream=False) + request = urllib.request.Request(args.endpoint.rstrip('/') + '/v1/completions', + data=json.dumps(body).encode(), headers={'Content-Type': 'application/json', + 'x-request-id': 'quality263-' + case['case']}) + start = time.perf_counter() + row = dict(case=case['case'], route=args.route, input_sha256=case['input_sha256'], + input_tokens=len(case['ids']), expected=case['expected']) + try: + with urllib.request.urlopen(request, timeout=600) as response: + payload = json.load(response) + choice = payload['choices'][0] + row.update(text=choice['text'], tokens=choice['token_ids'], + finish_reason=choice['finish_reason'], response=payload, + **score(choice['text'], case['expected'])) + except Exception as error: + row.update(error=repr(error), answer_correct=False, format_correct=False) + output.write(json.dumps(row, ensure_ascii=False) + '\n') + output.flush() + raise + row['elapsed_seconds'] = time.perf_counter() - start + output.write(json.dumps(row, ensure_ascii=False) + '\n') + output.flush() + print(case['case'], row['answer_correct'], row['format_correct'], flush=True) + + +if __name__ == '__main__': + main() diff --git a/opt_prime/tests/test_heldout_quality.py b/opt_prime/tests/test_heldout_quality.py new file mode 100644 index 00000000..81a74343 --- /dev/null +++ b/opt_prime/tests/test_heldout_quality.py @@ -0,0 +1,92 @@ +"""Prevent answer substrings, duplicate keys, or missing pairs from inflating scores.""" +import importlib.util +from pathlib import Path +import pytest + +path = Path(__file__).parents[1] / 'examples/pd/http_matrix/heldout_quality.py' + +def module(): + assert path.exists(), 'held-out evaluator is not implemented' + spec = importlib.util.spec_from_file_location('heldout_quality', path) + value = importlib.util.module_from_spec(spec) + spec.loader.exec_module(value) + return value + +@pytest.mark.parametrize('text,correct,formatted', [ + ('{"answer":"73"}', True, True), + ('{"answer":"73","note":"ok"}', True, False), + ('{"answer":73}', False, False), + ('The answer is 73', False, False), + ('{"answer":"173"}', False, True), + ('{"answer":"wrong","answer":"73"}', False, False), + ('{"answer":"73"} trailing', False, False), +]) +def test_score(text, correct, formatted): + got = module().score(text, '73') + assert (got['answer_correct'], got['format_correct']) == (correct, formatted) + +def test_paired_regressions_are_not_hidden_by_equal_aggregate(): + m = module() + base = [{'case': 'a', 'input_sha256': 'x', 'answer_correct': True, 'tokens': [1]}, + {'case': 'b', 'input_sha256': 'y', 'answer_correct': False, 'tokens': [2]}] + other = [{'case': 'a', 'input_sha256': 'x', 'answer_correct': False, 'tokens': [3]}, + {'case': 'b', 'input_sha256': 'y', 'answer_correct': True, 'tokens': [2]}] + got = m.compare(base, other) + assert got == {'cases': 2, 'correct_to_wrong': 1, 'wrong_to_correct': 1, 'token_equal': 1} + with pytest.raises(ValueError): + m.compare(base, other[:1]) + other[0]['input_sha256'] = 'changed' + with pytest.raises(ValueError): + m.compare(base, other) + +def test_context_builder_keeps_retrieval_fact_inside_budget(): + class CharacterTokenizer: + def apply_chat_template(self, messages, **kwargs): + return list(messages[0]['content'].encode()) + m = module() + cases = m.fixtures(CharacterTokenizer()) + assert len(cases) == len({x['case'] for x in cases}) == 36 + for case in cases: + assert len(case['ids']) <= (case['target_tokens'] or 2048) + if case['task'] != 'arithmetic': + assert case['expected'] in case['prompt'] + if case['target_tokens']: + assert len(case['ids']) > case['target_tokens'] - 150 + +def test_http_runner_supplies_required_request_id_and_frozen_tokens(tmp_path, monkeypatch): + import json + import sys + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + seen = [] + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + seen.append((self.headers.get('x-request-id'), body)) + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps({'choices': [{'text': '{"answer":"73"}', + 'token_ids': [9], 'finish_reason': 'stop'}]}).encode()) + def log_message(self, *args): + pass + server = HTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + fixture = tmp_path / 'input.json' + output = tmp_path / 'output.jsonl' + fixture.write_text(json.dumps({'model':'model', 'max_tokens':128, + 'cases':[{'case':'a','ids':[4,5], 'input_sha256':'hash','expected':'73'}]})) + monkeypatch.setattr(sys, 'argv', ['heldout_quality', 'run', '--fixtures',str(fixture), + '--endpoint',f'http://127.0.0.1:{server.server_port}', '--route','test', '--output',str(output)]) + try: + module().main() + finally: + server.shutdown() + thread.join() + server.server_close() + assert seen[0][0], 'stage HTTP API requires a nonempty request ID' + assert seen[0][1]['prompt'] == [4,5] + assert seen[0][1]['ignore_eos'] is False + assert json.loads(output.read_text())['answer_correct'] is True + with pytest.raises(FileExistsError): + module().main()