From e5fa0878272ce49c7ec804a54b9ed865d261b165 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 23 Oct 2026 21:28:04 +0900 Subject: [PATCH 1/2] fix: honor tokenizer turn-end EOS in batched serving --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 4 + .../evidence229/koni-capital-fixture.json | 152 +++++++++++++++++ .../pd/http_matrix/evidence229/koni-http.json | 154 ++++++++++++++++++ .../experiments/2026-10-23-serving-eos229.md | 38 +++++ opt_prime/examples/pd/serve_batched.py | 6 +- opt_prime/examples/pd/validate_http_eos.py | 103 ++++++++++++ opt_prime/tests/test_serving_eos_factory.py | 75 +++++++++ 8 files changed, 532 insertions(+), 1 deletion(-) create mode 100644 opt_prime/examples/pd/http_matrix/evidence229/koni-capital-fixture.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence229/koni-http.json create mode 100644 opt_prime/examples/pd/http_matrix/experiments/2026-10-23-serving-eos229.md create mode 100644 opt_prime/examples/pd/validate_http_eos.py create mode 100644 opt_prime/tests/test_serving_eos_factory.py diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 1d2f906a..3ba94e4d 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -1,2 +1,3 @@ {"id": "b61f99ae005a4258aa7f44d8edeca28d", "ts": "2026-10-23T18:08:44+09:00", "task": "Four-model HTTP matrix (aicomp #211)", "result": "All4 GPU models pass HTTP concurrency4/8, cancel/disconnect/drain/reuse checks. No-cacheBF16 and warm-cacheFP32 each48/48 concurrent outputs exact. Warm-cacheBF16 has31/48 raw equality; actualQwenHTTP tie explains observed branch. Warm-state cancellation isolates KONI cold-cache difference.", "next": "Publish GPUHTTP evidence; continue ATOM HTTP and RNGD routes. Preserve numerical/quality and true-overlap limitations.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"models": "4", "slots": "4", "concurrency": "4,8", "conditions": "bf16_no_cache,bf16_cache,fp32_cache"}, "artifacts": ["evidence/summary.json"], "experiments": ["experiments/2026-10-23-http-matrix211.md"], "labels": ["benchmark"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "GPU host", "branch": "main", "commit": "351d39d-dirty"} {"id": "3563031e43c8478e951fff82f81bfe94", "ts": "2026-10-23T18:38:18+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "ATOM HTTP: all four models pass 128/2048/8192-token lifecycle checks; cache-off concurrent48/48 exact. Both Qwen device/host cache cases pass12/12 each. SIGPIPE candidate suppresses native disconnect backtraces6 to0 with unchanged outputs;14 unit/integration tests pass.", "next": "Publish ATOM HTTP matrix and SIGPIPE fix; continue missing RNGD and hybrid-state routes.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "atom", "input_tokens": "128,2048,8192", "output_tokens": "32", "hardware_batch": "1"}, "artifacts": ["evidence/atom-http-qwen3-4b-host-fixed.json"], "experiments": ["experiments/2026-10-23-atom-http211.md"], "labels": ["experiment", "bug"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "RNGD+ATOM host", "branch": "exp/211-atom-http", "commit": "working candidate"} +{"id": "4c7935b757264687b1310fbe1ec6399a", "ts": "2026-10-23T21:26:38+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Production factory union of generation-config/tokenizer EOS fixes real KONI HTTP turn end: baseline length, fixed stop, ignore_eos length, streaming stop, reuse stop. Baseline factory3fail/3pass ->22related tests pass. Preserved failed initial SSE checker log.", "next": "Publish scoped serving fix; verify NPU HTTP and retain long RNGD semantic validation.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"model": "KONI-8B", "backend": "gpu", "max_tokens": "8"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-http-final.json", "size": 5067, "sha256": "d5be9a4a52a6485f"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index 0ce456da..dad8182e 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -1,3 +1,7 @@ # HTTP model matrix State: review; full goal remains active. GPU evidence remains as recorded. ATOM four-model cache-off lifecycle passes with48/48 concurrent equality; both Qwen device/host cache cases pass12/12 each. SIGPIPE fix tracked separately in #214. Remaining: RNGD routes, hybrid-state restoration, real transfer overlap and broader semantic testing. + +## Production EOS correction (#229) + +The common serve_batched factory now unions model generation EOS and tokenizer turn-end EOS. KONI config128001/tokenizer128009 previously continued into role delimiters; real GPU HTTP negative control ends length, fixed/stream/reuse end stop, ignore_eos remains length.22related CPU tests pass after3failed factory regressions. GPU hardware checked; NPU HTTP revalidation remains open. See serving-eos229 record/evidence229. diff --git a/opt_prime/examples/pd/http_matrix/evidence229/koni-capital-fixture.json b/opt_prime/examples/pd/http_matrix/evidence229/koni-capital-fixture.json new file mode 100644 index 00000000..28380500 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence229/koni-capital-fixture.json @@ -0,0 +1,152 @@ +{ + "eos_ids": [ + 128001, + 128009 + ], + "cases": [ + { + "case": "capital-0", + "ids": [ + 128000, + 128006, + 9125, + 128007, + 271, + 2675, + 527, + 735, + 715, + 40, + 11, + 459, + 15592, + 18328, + 16572, + 3196, + 389, + 445, + 4355, + 4940, + 18, + 13, + 16, + 323, + 3549, + 555, + 735, + 3931, + 40, + 311, + 387, + 11190, + 323, + 10978, + 13, + 4718, + 6677, + 45395, + 264, + 7029, + 2134, + 315, + 13650, + 11, + 10923, + 499, + 311, + 16988, + 304, + 67749, + 21633, + 323, + 3493, + 6492, + 389, + 6485, + 15223, + 13, + 21883, + 374, + 459, + 7754, + 430, + 16964, + 264, + 3465, + 13, + 9842, + 264, + 2077, + 430, + 36001, + 45695, + 279, + 1715, + 13, + 1442, + 499, + 1541, + 956, + 1440, + 279, + 4320, + 11, + 1120, + 2019, + 430, + 499, + 1541, + 956, + 1440, + 627, + 38766, + 1303, + 33025, + 2696, + 25, + 5887, + 220, + 2366, + 19, + 198, + 15724, + 2696, + 25, + 220, + 1544, + 5033, + 220, + 2366, + 19, + 271, + 128009, + 128006, + 882, + 128007, + 271, + 3923, + 374, + 279, + 6864, + 315, + 9822, + 30, + 18321, + 449, + 1193, + 279, + 3363, + 836, + 13, + 128009, + 128006, + 42, + 715, + 40, + 128007, + 271 + ], + "question": "What is the capital of France? Reply with only the city name." + } + ] +} diff --git a/opt_prime/examples/pd/http_matrix/evidence229/koni-http.json b/opt_prime/examples/pd/http_matrix/evidence229/koni-http.json new file mode 100644 index 00000000..5d08acb7 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence229/koni-http.json @@ -0,0 +1,154 @@ +[ + { + "case": "baseline", + "eos_ids": [ + 128001 + ], + "response": { + "id": "baseline", + "object": "text_completion", + "created": 1792758338, + "model": "/tmp/serving-matrix-models/KONI-8B", + "choices": [ + { + "index": 0, + "text": "Parisuser\n\nWhat is", + "token_ids": [ + 60704, + 128009, + 128006, + 882, + 128007, + 271, + 3923, + 374 + ], + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 138, + "completion_tokens": 8, + "total_tokens": 146 + } + }, + "raw": "{\"id\": \"baseline\", \"object\": \"text_completion\", \"created\": 1792758338, \"model\": \"/tmp/serving-matrix-models/KONI-8B\", \"choices\": [{\"index\": 0, \"text\": \"Parisuser\\n\\nWhat is\", \"token_ids\": [60704, 128009, 128006, 882, 128007, 271, 3923, 374], \"finish_reason\": \"length\"}], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 8, \"total_tokens\": 146}}" + }, + { + "case": "fixed", + "eos_ids": [ + 128001, + 128009 + ], + "response": { + "id": "fixed", + "object": "text_completion", + "created": 1792758338, + "model": "/tmp/serving-matrix-models/KONI-8B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 60704, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 138, + "completion_tokens": 2, + "total_tokens": 140 + } + }, + "raw": "{\"id\": \"fixed\", \"object\": \"text_completion\", \"created\": 1792758338, \"model\": \"/tmp/serving-matrix-models/KONI-8B\", \"choices\": [{\"index\": 0, \"text\": \"Paris\", \"token_ids\": [60704, 128009], \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 2, \"total_tokens\": 140}}" + }, + { + "case": "ignore", + "eos_ids": [ + 128001, + 128009 + ], + "response": { + "id": "ignore", + "object": "text_completion", + "created": 1792758338, + "model": "/tmp/serving-matrix-models/KONI-8B", + "choices": [ + { + "index": 0, + "text": "Parisuser\n\nWhat is", + "token_ids": [ + 60704, + 128009, + 128006, + 882, + 128007, + 271, + 3923, + 374 + ], + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 138, + "completion_tokens": 8, + "total_tokens": 146 + } + }, + "raw": "{\"id\": \"ignore\", \"object\": \"text_completion\", \"created\": 1792758338, \"model\": \"/tmp/serving-matrix-models/KONI-8B\", \"choices\": [{\"index\": 0, \"text\": \"Parisuser\\n\\nWhat is\", \"token_ids\": [60704, 128009, 128006, 882, 128007, 271, 3923, 374], \"finish_reason\": \"length\"}], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 8, \"total_tokens\": 146}}" + }, + { + "case": "stream", + "eos_ids": [ + 128001, + 128009 + ], + "response": { + "id": "stream", + "object": "text_completion", + "created": 1792758338, + "model": "/tmp/serving-matrix-models/KONI-8B", + "choices": [ + { + "index": 0, + "text": "", + "finish_reason": "stop" + } + ] + }, + "raw": "data: {\"id\": \"stream\", \"object\": \"text_completion\", \"created\": 1792758338, \"model\": \"/tmp/serving-matrix-models/KONI-8B\", \"choices\": [{\"index\": 0, \"text\": \"Paris\", \"finish_reason\": null}]}\n\ndata: {\"id\": \"stream\", \"object\": \"text_completion\", \"created\": 1792758338, \"model\": \"/tmp/serving-matrix-models/KONI-8B\", \"choices\": [{\"index\": 0, \"text\": \"\", \"finish_reason\": \"stop\"}]}\n\ndata: {\"id\": \"stream\", \"object\": \"text_completion\", \"created\": 1792758338, \"model\": \"/tmp/serving-matrix-models/KONI-8B\", \"choices\": [], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 2, \"total_tokens\": 140}}\n\ndata: [DONE]\n\n" + }, + { + "case": "reuse", + "eos_ids": [ + 128001, + 128009 + ], + "response": { + "id": "reuse", + "object": "text_completion", + "created": 1792758338, + "model": "/tmp/serving-matrix-models/KONI-8B", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 60704, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 138, + "completion_tokens": 2, + "total_tokens": 140 + } + }, + "raw": "{\"id\": \"reuse\", \"object\": \"text_completion\", \"created\": 1792758338, \"model\": \"/tmp/serving-matrix-models/KONI-8B\", \"choices\": [{\"index\": 0, \"text\": \"Paris\", \"token_ids\": [60704, 128009], \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 2, \"total_tokens\": 140}}" + } +] diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-serving-eos229.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-serving-eos229.md new file mode 100644 index 00000000..9f4ae362 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-serving-eos229.md @@ -0,0 +1,38 @@ +# Honor tokenizer turn-end EOS in production batched serving + +- **Date**: 2026-10-23 +- **Task**: aicomp #229, parent #201/#227 +- **Host / device**: GPU host A10080GB, BF16, B1/slot cache/context2048, CPU threads4. +- **Commit**: base3ea0493, fix/229-serving-eos working tree. + +## Question + +Does serve_batched stop on a tokenizer's chat EOS when generation_config still specifies a different base-model EOS? + +## Setup + +KONI pinned checkpoint0172bf7e73977ae213613529f3d2d5400a33f678 has configEOS128001, tokenizerEOS128009. The common production factory used only the former. Normalize configured IDs and add tokenizerEOS without duplicates. Explicit ignore_eos remains a per-request override. Six factory tests cover scalar/list/tuple/None/duplicate configurations and ignore_eos. + +Real HTTP uses the production factory, owner, scheduler and server, one loaded KONI model and the capital-0 chat token IDs from #227. Baseline negative control sets the live idle service back to config-only EOS; subsequent requests use the factory's union. Requests sequential, max_tokens8, temperature0, prefix-cacheoff. Check nonstreaming, ignore_eos, streaming and reuse. This experiment covers GPU hardware; the common factory placement covers all backend branches structurally but NPU HTTP hardware is not revalidated here. + +## Command + +```bash +PYTHONPATH=opt_prime python -m pytest -q opt_prime/tests/test_serving_eos_factory.py opt_prime/tests/test_pd_sampling.py opt_prime/tests/test_pd_stream.py opt_prime/tests/test_serve_batched.py +PYTHONPATH=opt_prime CUDA_VISIBLE_DEVICES=0 HF_HUB_OFFLINE=1 python opt_prime/examples/pd/validate_http_eos.py --factory opt_prime/examples/pd/serve_batched.py --model "$KONI_SNAPSHOT" --fixtures "$KONI_FIXTURES" --output eos229-koni-http-final.json +``` + +## Results + +- Initial test used the wrong SamplingParams field name; corrected to stop_token_ids before assessing the regression. Baseline then3failed/3passed at the intended EOS union assertions. +- After factory fix:22related CPU tests pass (one existing Transformers warning). +- Real HTTP baseline finishes length; fixed finishes stop; ignore_eos finishes length; streaming finishes stop; immediate next request finishes stop. +- Initial HTTP checker misread the last usage-only SSE event as a choice and raised IndexError after baseline/fixed/ignore passed. Select the final event containing a finish_reason; complete rerun passes. Preserve both failure and corrected logs. + +Evidence: analysis/serving-matrix-201/eos229-red.log, eos229-tests.log, eos229-koni-http.log (checker failure), eos229-koni-http-final.json/.log. + +## Conclusion + +The production factory now honors both checkpoint and tokenizer EOS, preventing unwanted continuation into chat role delimiters for this KONI case while preserving ignore_eos. No model weights or shared SDK changed. This does not establish all model quality, NPU HTTP behavior, heterogeneous PD or overlap performance. Full four-model goal remains active. + +Selected public evidence: ../evidence229/koni-http.json and koni-capital-fixture.json. Verbose failure logs remain in local research artifacts. diff --git a/opt_prime/examples/pd/serve_batched.py b/opt_prime/examples/pd/serve_batched.py index b8debf46..b0338c36 100644 --- a/opt_prime/examples/pd/serve_batched.py +++ b/opt_prime/examples/pd/serve_batched.py @@ -95,10 +95,14 @@ def build(a): else: b = ContinuousBatcher(model, prefill_chunk=a.chunk, **kw) eos = model.generation_config.eos_token_id + eos = tuple(eos) if isinstance(eos, (list, tuple)) else ((eos,) if eos is not None else ()) + # Some chat checkpoints retain a base-model EOS in generation_config. + if tok.eos_token_id is not None: + eos = tuple(dict.fromkeys((*eos, tok.eos_token_id))) svc = BatchedService(b, model=a.model, vocab_size=model.config.get_text_config().vocab_size, max_output=a.max_output, render=lambda t: tok.decode(t, skip_special_tokens=True), tokenizer=tok, - eos_token_ids=tuple(eos) if isinstance(eos, (list, tuple)) else ((eos,) if eos is not None else ())) + eos_token_ids=eos) return BatchedOwner(svc) diff --git a/opt_prime/examples/pd/validate_http_eos.py b/opt_prime/examples/pd/validate_http_eos.py new file mode 100644 index 00000000..9ec50bcf --- /dev/null +++ b/opt_prime/examples/pd/validate_http_eos.py @@ -0,0 +1,103 @@ +"""KONI real-HTTP EOS negative control, override, streaming and reuse check.""" + +import argparse +import importlib.util +import json +import threading +import urllib.request +from pathlib import Path +from types import SimpleNamespace + +import torch + +p = argparse.ArgumentParser() +p.add_argument("--factory", required=True) +p.add_argument("--model", required=True) +p.add_argument("--fixtures", required=True) +p.add_argument("--output", required=True) +a = p.parse_args() +torch.set_num_threads(4) +spec = importlib.util.spec_from_file_location("factory229", a.factory) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +model = a.model +args = SimpleNamespace( + model=model, + kv_events=None, + backend="torch", + dtype="bfloat16", + device="cuda:0", + slots=1, + max_context=2048, + kv="slot", + page_size=64, + prefix_cache=False, + kv_host_memory_gb=0, + reload_bandwidth=None, + prefill_rate=None, + chunk=256, + max_output=64, + pd_prefill_device=None, + pd_prefill_python=None, + remote_kv=None, +) +owner = module.build(args) +fixed = owner.service.eos_token_ids +assert fixed == (128001, 128009), fixed +worker = threading.Thread(target=owner.run, daemon=True) +worker.start() +server, http = module.serve_stage(owner, "decode", 0) +ids = json.loads(Path(a.fixtures).read_text())["cases"][0]["ids"] +rows = [] +try: + for name, eos, ignore, stream in [ + ("baseline", (128001,), False, False), + ("fixed", fixed, False, False), + ("ignore", fixed, True, False), + ("stream", fixed, False, True), + ("reuse", fixed, False, False), + ]: + owner.service.eos_token_ids = eos + body = { + "model": model, + "prompt": ids, + "max_tokens": 8, + "temperature": 0, + "ignore_eos": ignore, + "stream": stream, + } + req = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}/v1/completions", + data=json.dumps(body).encode(), + headers={"x-request-id": name}, + ) + with urllib.request.urlopen(req, timeout=60) as response: + raw = response.read().decode() + if stream: + events = [ + json.loads(line[6:]) + for line in raw.splitlines() + if line.startswith("data: ") and line[6:] != "[DONE]" + ] + assert "[DONE]" in raw + result = next( + e + for e in reversed(events) + if e.get("choices") and e["choices"][0].get("finish_reason") + ) + else: + result = json.loads(raw) + finish = result["choices"][0]["finish_reason"] + assert finish == ("length" if name in ["baseline", "ignore"] else "stop"), ( + name, + result, + ) + rows.append({"case": name, "eos_ids": eos, "response": result, "raw": raw}) + print(name, finish, flush=True) + Path(a.output).write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n") +finally: + owner.submit("stop") + worker.join(15) + server.shutdown() + server.server_close() + http.join(5) diff --git a/opt_prime/tests/test_serving_eos_factory.py b/opt_prime/tests/test_serving_eos_factory.py new file mode 100644 index 00000000..4ab801aa --- /dev/null +++ b/opt_prime/tests/test_serving_eos_factory.py @@ -0,0 +1,75 @@ +"""The public factory must honor model EOS and tokenizer chat turn end.""" + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.mark.parametrize( + "configured,tokenizer_eos,expected", + [ + (8, 9, (8, 9)), + ([8, 9], 9, (8, 9)), + ((8, 10), 9, (8, 10, 9)), + (None, 9, (9,)), + (8, None, (8,)), + (None, None, ()), + ], +) +def test_factory_preserves_all_eos_and_ignore_override( + monkeypatch, configured, tokenizer_eos, expected +): + path = Path(__file__).parents[1] / "examples/pd/serve_batched.py" + spec = importlib.util.spec_from_file_location("serve_eos_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + tokenizer = SimpleNamespace( + eos_token_id=tokenizer_eos, decode=lambda ids, **kw: str(ids) + ) + model = SimpleNamespace( + generation_config=SimpleNamespace(eos_token_id=configured), + config=SimpleNamespace(get_text_config=lambda: SimpleNamespace(vocab_size=32)), + ) + model.to = lambda device: model + model.eval = lambda: model + monkeypatch.setattr( + module.AutoTokenizer, "from_pretrained", lambda *a, **kw: tokenizer + ) + monkeypatch.setattr( + module.AutoModelForCausalLM, "from_pretrained", lambda *a, **kw: model + ) + monkeypatch.setattr( + module, + "ContinuousBatcher", + lambda *a, **kw: SimpleNamespace(max_context=64, paged=False), + ) + monkeypatch.setattr(module, "prefill_worker", lambda *a: None) + monkeypatch.setattr(module, "remote_tier", lambda *a: None) + args = SimpleNamespace( + model="test", + kv_events=None, + backend="torch", + dtype="float32", + device="cpu", + slots=1, + max_context=64, + kv="slot", + page_size=16, + prefix_cache=False, + kv_host_memory_gb=0, + reload_bandwidth=1, + prefill_rate=1, + chunk=16, + max_output=16, + ) + service = module.build(args).service + request, _ = service._request( + "r", {"model": "test", "prompt": [1], "max_tokens": 4} + ) + assert request.sampling.stop_token_ids == expected + request, _ = service._request( + "r2", {"model": "test", "prompt": [1], "max_tokens": 4, "ignore_eos": True} + ) + assert request.sampling.stop_token_ids == () From f84823a0dbd2c7c4a8bfffb729c09285112b5894 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 23 Oct 2026 21:33:56 +0900 Subject: [PATCH 2/2] test: verify tokenizer EOS handling on ATOM HTTP --- .../pd/http_matrix/.research/events.jsonl | 2 + opt_prime/examples/pd/http_matrix/STATUS.md | 2 +- .../evidence229/koni-atom-http.json | 154 ++++++++++++++++++ .../experiments/2026-10-23-serving-eos229.md | 10 +- opt_prime/examples/pd/validate_http_eos.py | 22 ++- 5 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 opt_prime/examples/pd/http_matrix/evidence229/koni-atom-http.json diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 3ba94e4d..5f51c17b 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -1,3 +1,5 @@ {"id": "b61f99ae005a4258aa7f44d8edeca28d", "ts": "2026-10-23T18:08:44+09:00", "task": "Four-model HTTP matrix (aicomp #211)", "result": "All4 GPU models pass HTTP concurrency4/8, cancel/disconnect/drain/reuse checks. No-cacheBF16 and warm-cacheFP32 each48/48 concurrent outputs exact. Warm-cacheBF16 has31/48 raw equality; actualQwenHTTP tie explains observed branch. Warm-state cancellation isolates KONI cold-cache difference.", "next": "Publish GPUHTTP evidence; continue ATOM HTTP and RNGD routes. Preserve numerical/quality and true-overlap limitations.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"models": "4", "slots": "4", "concurrency": "4,8", "conditions": "bf16_no_cache,bf16_cache,fp32_cache"}, "artifacts": ["evidence/summary.json"], "experiments": ["experiments/2026-10-23-http-matrix211.md"], "labels": ["benchmark"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "GPU host", "branch": "main", "commit": "351d39d-dirty"} {"id": "3563031e43c8478e951fff82f81bfe94", "ts": "2026-10-23T18:38:18+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "ATOM HTTP: all four models pass 128/2048/8192-token lifecycle checks; cache-off concurrent48/48 exact. Both Qwen device/host cache cases pass12/12 each. SIGPIPE candidate suppresses native disconnect backtraces6 to0 with unchanged outputs;14 unit/integration tests pass.", "next": "Publish ATOM HTTP matrix and SIGPIPE fix; continue missing RNGD and hybrid-state routes.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "atom", "input_tokens": "128,2048,8192", "output_tokens": "32", "hardware_batch": "1"}, "artifacts": ["evidence/atom-http-qwen3-4b-host-fixed.json"], "experiments": ["experiments/2026-10-23-atom-http211.md"], "labels": ["experiment", "bug"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "RNGD+ATOM host", "branch": "exp/211-atom-http", "commit": "working candidate"} {"id": "4c7935b757264687b1310fbe1ec6399a", "ts": "2026-10-23T21:26:38+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Production factory union of generation-config/tokenizer EOS fixes real KONI HTTP turn end: baseline length, fixed stop, ignore_eos length, streaming stop, reuse stop. Baseline factory3fail/3pass ->22related tests pass. Preserved failed initial SSE checker log.", "next": "Publish scoped serving fix; verify NPU HTTP and retain long RNGD semantic validation.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"model": "KONI-8B", "backend": "gpu", "max_tokens": "8"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-http-final.json", "size": 5067, "sha256": "d5be9a4a52a6485f"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "e4993823527144b5a14e6441b1cd5c01", "ts": "2026-10-23T21:29:56+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Published production EOS fix and real KONI HTTP evidence in draft PR230 (e5fa087). Final factory/streaming/sampling/batched suite22passed; hardware checker5/5 cases passed; changed files sensitive-clean,4unchanged baseline findings in this branch.", "next": "Verify same HTTP factory on NPU; continue full four-model long semantic and heterogeneous PD validation.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "gpu"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-http-final.json", "size": 5067, "sha256": "d5be9a4a52a6485f"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "328b8349e9974a059ddc7e1a8a485e2c", "ts": "2026-10-23T21:32:23+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Real ATOM KONI HTTP also passes5/5 baseline/fixed/ignore/stream/reuse checks using same production factory: config-only length vs union stop. GPU+ATOM now10/10 expected outcomes; no SDK/artifact rebuild.", "next": "Update PR230 with NPU evidence; finish KONI medium-context RNGD handoff and broader PD work.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "atom", "kv_blocks": "1"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-atom-http.json", "size": 5199, "sha256": "4735255db379ff00"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index dad8182e..3464a402 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -4,4 +4,4 @@ State: review; full goal remains active. GPU evidence remains as recorded. ATOM ## Production EOS correction (#229) -The common serve_batched factory now unions model generation EOS and tokenizer turn-end EOS. KONI config128001/tokenizer128009 previously continued into role delimiters; real GPU HTTP negative control ends length, fixed/stream/reuse end stop, ignore_eos remains length.22related CPU tests pass after3failed factory regressions. GPU hardware checked; NPU HTTP revalidation remains open. See serving-eos229 record/evidence229. +The common serve_batched factory now unions model generation EOS and tokenizer turn-end EOS. KONI config128001/tokenizer128009 previously continued into role delimiters; real GPU HTTP negative control ends length, fixed/stream/reuse end stop, ignore_eos remains length.22related CPU tests pass after3failed factory regressions. GPU and ATOM hardware both pass5/5 EOS/override/stream/reuse checks; NPU uses existing B1/context9216/k1 artifact. See serving-eos229 record/evidence229. diff --git a/opt_prime/examples/pd/http_matrix/evidence229/koni-atom-http.json b/opt_prime/examples/pd/http_matrix/evidence229/koni-atom-http.json new file mode 100644 index 00000000..6c600434 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence229/koni-atom-http.json @@ -0,0 +1,154 @@ +[ + { + "case": "baseline", + "eos_ids": [ + 128001 + ], + "response": { + "id": "baseline", + "object": "text_completion", + "created": 1792758717, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "Parisuser\n\nWhat is", + "token_ids": [ + 60704, + 128009, + 128006, + 882, + 128007, + 271, + 3923, + 374 + ], + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 138, + "completion_tokens": 8, + "total_tokens": 146 + } + }, + "raw": "{\"id\": \"baseline\", \"object\": \"text_completion\", \"created\": 1792758717, \"model\": \"KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024\", \"choices\": [{\"index\": 0, \"text\": \"Parisuser\\n\\nWhat is\", \"token_ids\": [60704, 128009, 128006, 882, 128007, 271, 3923, 374], \"finish_reason\": \"length\"}], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 8, \"total_tokens\": 146}}" + }, + { + "case": "fixed", + "eos_ids": [ + 128001, + 128009 + ], + "response": { + "id": "fixed", + "object": "text_completion", + "created": 1792758717, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 60704, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 138, + "completion_tokens": 2, + "total_tokens": 140 + } + }, + "raw": "{\"id\": \"fixed\", \"object\": \"text_completion\", \"created\": 1792758717, \"model\": \"KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024\", \"choices\": [{\"index\": 0, \"text\": \"Paris\", \"token_ids\": [60704, 128009], \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 2, \"total_tokens\": 140}}" + }, + { + "case": "ignore", + "eos_ids": [ + 128001, + 128009 + ], + "response": { + "id": "ignore", + "object": "text_completion", + "created": 1792758718, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "Parisuser\n\nWhat is", + "token_ids": [ + 60704, + 128009, + 128006, + 882, + 128007, + 271, + 3923, + 374 + ], + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 138, + "completion_tokens": 8, + "total_tokens": 146 + } + }, + "raw": "{\"id\": \"ignore\", \"object\": \"text_completion\", \"created\": 1792758718, \"model\": \"KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024\", \"choices\": [{\"index\": 0, \"text\": \"Parisuser\\n\\nWhat is\", \"token_ids\": [60704, 128009, 128006, 882, 128007, 271, 3923, 374], \"finish_reason\": \"length\"}], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 8, \"total_tokens\": 146}}" + }, + { + "case": "stream", + "eos_ids": [ + 128001, + 128009 + ], + "response": { + "id": "stream", + "object": "text_completion", + "created": 1792758718, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "", + "finish_reason": "stop" + } + ] + }, + "raw": "data: {\"id\": \"stream\", \"object\": \"text_completion\", \"created\": 1792758718, \"model\": \"KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024\", \"choices\": [{\"index\": 0, \"text\": \"Paris\", \"finish_reason\": null}]}\n\ndata: {\"id\": \"stream\", \"object\": \"text_completion\", \"created\": 1792758718, \"model\": \"KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024\", \"choices\": [{\"index\": 0, \"text\": \"\", \"finish_reason\": \"stop\"}]}\n\ndata: {\"id\": \"stream\", \"object\": \"text_completion\", \"created\": 1792758718, \"model\": \"KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024\", \"choices\": [], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 2, \"total_tokens\": 140}}\n\ndata: [DONE]\n\n" + }, + { + "case": "reuse", + "eos_ids": [ + 128001, + 128009 + ], + "response": { + "id": "reuse", + "object": "text_completion", + "created": 1792758718, + "model": "KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024", + "choices": [ + { + "index": 0, + "text": "Paris", + "token_ids": [ + 60704, + 128009 + ], + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 138, + "completion_tokens": 2, + "total_tokens": 140 + } + }, + "raw": "{\"id\": \"reuse\", \"object\": \"text_completion\", \"created\": 1792758718, \"model\": \"KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024\", \"choices\": [{\"index\": 0, \"text\": \"Paris\", \"token_ids\": [60704, 128009], \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 138, \"completion_tokens\": 2, \"total_tokens\": 140}}" + } +] diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-serving-eos229.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-serving-eos229.md index 9f4ae362..7e535ae2 100644 --- a/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-serving-eos229.md +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-serving-eos229.md @@ -13,7 +13,7 @@ Does serve_batched stop on a tokenizer's chat EOS when generation_config still s KONI pinned checkpoint0172bf7e73977ae213613529f3d2d5400a33f678 has configEOS128001, tokenizerEOS128009. The common production factory used only the former. Normalize configured IDs and add tokenizerEOS without duplicates. Explicit ignore_eos remains a per-request override. Six factory tests cover scalar/list/tuple/None/duplicate configurations and ignore_eos. -Real HTTP uses the production factory, owner, scheduler and server, one loaded KONI model and the capital-0 chat token IDs from #227. Baseline negative control sets the live idle service back to config-only EOS; subsequent requests use the factory's union. Requests sequential, max_tokens8, temperature0, prefix-cacheoff. Check nonstreaming, ignore_eos, streaming and reuse. This experiment covers GPU hardware; the common factory placement covers all backend branches structurally but NPU HTTP hardware is not revalidated here. +Real HTTP uses the production factory, owner, scheduler and server, one loaded KONI model and the capital-0 chat token IDs from #227. Baseline negative control sets the live idle service back to config-only EOS; subsequent requests use the factory's union. Requests sequential, max_tokens8, temperature0, prefix-cacheoff. Check nonstreaming, ignore_eos, streaming and reuse. This experiment covers GPU and ATOM hardware. ATOM uses the existing B1/context9216/k1 KONI artifact under exclusive allocation; a process-local engine subclass supplies only its artifact cache directory to the unchanged factory. ## Command @@ -26,13 +26,15 @@ PYTHONPATH=opt_prime CUDA_VISIBLE_DEVICES=0 HF_HUB_OFFLINE=1 python opt_prime/ex - Initial test used the wrong SamplingParams field name; corrected to stop_token_ids before assessing the regression. Baseline then3failed/3passed at the intended EOS union assertions. - After factory fix:22related CPU tests pass (one existing Transformers warning). -- Real HTTP baseline finishes length; fixed finishes stop; ignore_eos finishes length; streaming finishes stop; immediate next request finishes stop. +- Real HTTP on both GPU and ATOM: baseline finishes length; fixed finishes stop; ignore_eos finishes length; streaming finishes stop; immediate next request finishes stop. Both backends pass5/5 checks. GPU baseline returns Paris followed by chat-role continuation (8tokens); fixed returns Paris and stops (2tokens including EOS). - Initial HTTP checker misread the last usage-only SSE event as a choice and raised IndexError after baseline/fixed/ignore passed. Select the final event containing a finish_reason; complete rerun passes. Preserve both failure and corrected logs. Evidence: analysis/serving-matrix-201/eos229-red.log, eos229-tests.log, eos229-koni-http.log (checker failure), eos229-koni-http-final.json/.log. ## Conclusion -The production factory now honors both checkpoint and tokenizer EOS, preventing unwanted continuation into chat role delimiters for this KONI case while preserving ignore_eos. No model weights or shared SDK changed. This does not establish all model quality, NPU HTTP behavior, heterogeneous PD or overlap performance. Full four-model goal remains active. +The production factory now honors both checkpoint and tokenizer EOS, preventing unwanted continuation into chat role delimiters for this KONI case while preserving ignore_eos. No model weights or shared SDK changed. This does not establish all model quality, all-model NPU HTTP behavior, heterogeneous PD or overlap performance. Full four-model goal remains active. -Selected public evidence: ../evidence229/koni-http.json and koni-capital-fixture.json. Verbose failure logs remain in local research artifacts. +ATOM command: `bash run-http-eos229.sh --backend atom --cache-dir atom-artifacts --model KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024 --factory serve_batched229.py --fixtures semantic227-koni-fixtures-eos.json --output eos229-koni-atom-http.json`. Same validator, production factory/owner/server, existing isolated bootstrap. Evidence: analysis/serving-matrix-201/eos229-koni-atom-http.json/.log. + +Selected public evidence: ../evidence229/koni-http.json, koni-atom-http.json and koni-capital-fixture.json. Verbose failure logs remain local. diff --git a/opt_prime/examples/pd/validate_http_eos.py b/opt_prime/examples/pd/validate_http_eos.py index 9ec50bcf..c33fd321 100644 --- a/opt_prime/examples/pd/validate_http_eos.py +++ b/opt_prime/examples/pd/validate_http_eos.py @@ -13,6 +13,8 @@ p = argparse.ArgumentParser() p.add_argument("--factory", required=True) p.add_argument("--model", required=True) +p.add_argument("--backend", choices=["torch", "atom"], default="torch") +p.add_argument("--cache-dir") p.add_argument("--fixtures", required=True) p.add_argument("--output", required=True) a = p.parse_args() @@ -21,14 +23,27 @@ module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) model = a.model +if a.backend == "atom": + if not a.cache_dir: + p.error("ATOM validation requires an existing artifact cache") + import opt_prime.accel.platforms.atom.engine as atom_engine + + nodes = list(Path("/dev").glob("rbln[0-9]*")) + assert len(nodes) == 1, "requires exclusive single-ATOM allocation" + + class ArtifactEngine(atom_engine.AtomRblnEngine): + def __init__(self, **kwargs): + super().__init__(**kwargs, cache_dir=a.cache_dir) + + atom_engine.AtomRblnEngine = ArtifactEngine args = SimpleNamespace( model=model, kv_events=None, - backend="torch", + backend=a.backend, dtype="bfloat16", device="cuda:0", slots=1, - max_context=2048, + max_context=9216 if a.backend == "atom" else 2048, kv="slot", page_size=64, prefix_cache=False, @@ -40,6 +55,9 @@ pd_prefill_device=None, pd_prefill_python=None, remote_kv=None, + devices=[int(nodes[0].name[4:])] if a.backend == "atom" else [], + atom_kv_blocks=1, + pd_remote_only=False, ) owner = module.build(args) fixed = owner.service.eos_token_ids