diff --git a/opt_prime/examples/pd/http_matrix/README.md b/opt_prime/examples/pd/http_matrix/README.md index 6576bad8..e2b28516 100644 --- a/opt_prime/examples/pd/http_matrix/README.md +++ b/opt_prime/examples/pd/http_matrix/README.md @@ -17,3 +17,17 @@ Use `--backend atom` in an isolated allocation exposing exactly one ATOM. Existi Inputs128/2048/8192 and clients4/8 are queued over one ATOM execution lane, not hardware batching. All four cache-off runs pass lifecycle checks and48/48 concurrent/serial comparisons. Both Qwen device/host cache cases pass12/12 each. Gemma requires the causal-prefill SDK graph correction and #210 hybrid local-table/mask fix; its prefix/host/PD restoration remains unsupported. KONI's one-block configuration has no spare prefix block. See experiments/2026-10-23-atom-http211.md and evidence/atom-http-*.json. Files ending `fixed` include #214's HTTP MSG_NOSIGNAL fix; the Qwen0.6 matched rerun preserves all output sequences and eliminates the observed native disconnect backtraces. This does not establish all RNGD/heterogeneous routes or real transfer overlap. + +### Torch PD producer reproducibility controls + +`serve_batched.py --pd-prefill-device cuda:0` keeps producer prefix caching enabled and uses a 2048-token producer chunk by default. These are independent of decoder `--prefix-cache` and local `--chunk`. + +To compare BF16 native and PD under uncached, matching chunk execution: + +```bash +python examples/pd/serve_batched.py --model "$MODEL" --device cuda:1 --slots 4 \ + --max-context 9216 --chunk 256 --pd-prefill-device cuda:0 \ + --pd-prefill-no-prefix-cache --pd-prefill-chunk 256 +``` + +The new flags affect the in-process Torch producer only. Nonpositive producer chunks are rejected. Disabling reuse or using smaller chunks can increase prefill work/overhead; these options do not promise a speedup or universal bitwise equality. CPU factory and four-model GPU controls are recorded under issue #241. diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index f93eb16c..4d6e5fe7 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -21,3 +21,7 @@ Static B1 with one KV block now supports cache-off full-prefix PD. KONI live RNG ## Subprocess failure lifecycle (#238) Structured SDK logs and cancelled/malformed replies previously killed the receiver and could strand later requests. Terminal errors now settle all queued Futures and reject subsequent submits. Failed startup and bounded close release the owned process and slab; fresh work succeeds on an explicitly created replacement worker. Nine CPU fault scenarios plus the existing tiny Torch subprocess path pass in the 64-test focused set. Full suite: 460 passed, 15 skipped, 135 subtests passed. Accelerator fault recovery and service-level automatic restart remain separate acceptance items. See prefill-worker-failure238 record. + +## Explicit Torch producer policy (#241) + +New opt-in controls disable producer prefix reuse and select producer chunk size; defaults remain cache-on/chunk2048. Public cache-off/chunk256 configuration passes all four real GPU PD HTTP paths:48/48 concurrent equality,12/12 native-PD serial equality at128/2048/8192tokens,84remote prefills and0producer hits. Lifecycle checks pass.20focused and464full tests pass (15skipped,135subtests). This is tested numerical reproducibility, not a speedup or arbitrary-input equality guarantee; original default-policy discrepancies remain in#240. diff --git a/opt_prime/examples/pd/http_matrix/evidence241/events.jsonl b/opt_prime/examples/pd/http_matrix/evidence241/events.jsonl new file mode 100644 index 00000000..b90b9f9f --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence241/events.jsonl @@ -0,0 +1,2 @@ +{"id": "bb5828e86f364b2f8bc860f5f50d6e4a", "ts": "2026-10-23T23:41:12+09:00", "task": "Torch PD producer cache control (aicomp #241)", "result": "Added explicit producer cache opt-out; actual tiny-model factory red1/3 -> green3/3,19focused tests pass. Qwen0.6B/KONI GPU controls recover concurrent12/12 with cache off; public-flag GPU validation pending.", "next": "Complete full suite and run new flag on GPU; retain enabled default and independent decoder policy.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [], "experiments": ["experiments/2026-10-23-pd-producer-cache241.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "8f4709fcaa1146a3b7e60d4e4e12bbf4", "ts": "2026-10-23T23:47:49+09:00", "task": "Torch PD producer cache control (aicomp #241)", "result": "Public cache-off/chunk256 flags: four-model GPU PD48/48 concurrent and12/12 native-PD serial equality;84remote prefills,0producer cache hits; all lifecycle checks pass.20focused tests pass.", "next": "Publish configuration controls and full comparison evidence; measure performance cost separately.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"dtype": "bfloat16", "producer_prefix_cache": "false", "producer_chunk": "256"}, "artifacts": [{"path": "analysis/serving-matrix-201/gpu241-summary.json", "size": 1185, "sha256": "d63548c8bc387463"}], "experiments": ["experiments/2026-10-23-pd-producer-cache241.md"], "labels": ["experiment"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/examples/pd/http_matrix/evidence241/gpu241-gemma-public.json b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-gemma-public.json new file mode 100644 index 00000000..40c7034b --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-gemma-public.json @@ -0,0 +1,777 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.2480425540124997, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.5971348389866762, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 2.097526551980991, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 3.246655499038752, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.1564620950375684, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 6.086662932008039, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.7346730430144817, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 3.8586602420546114, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 3.244282880972605, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 4.3633476630202495, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 4.313681842992082, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 5.79393796203658, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 5.8194297780282795, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 6.077404965006281, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 256, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 3.704238993988838 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 1.0592532549635507 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.746963998011779 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 30.377107114181854\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 14\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 0.7733828995448175\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 54.3519770000712\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"Gemma3-1B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 470, + "prefill_chunks": 0, + "decode_steps": 418, + "decode_rows": 594, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 9.051595669065136, + "transfer_s": 0.19907469901954755 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence241/gpu241-koni-public.json b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-koni-public.json new file mode 100644 index 00000000..dc2b5743 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-koni-public.json @@ -0,0 +1,777 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.3100038830307312, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 1.0760798280243762, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.5293093509972095, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 3.3086314600077458, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 1.5736503399675712, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 6.177591815998312, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 0.9205390079878271, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.396132203983143, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 3.516242377983872, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 1.4329761550179683, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 3.865869867033325, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 5.9176488200319, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 5.940213277994189, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 6.1712923960294574, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 256, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.8500524410046637 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.143585175042972 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.969620059011504 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 29.515059162979014\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 0.5406309075450765\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 46.274617296876386\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"KONI-8B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"KONI-8B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"KONI-8B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 569, + "prefill_chunks": 0, + "decode_steps": 433, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 11.160848259052727, + "transfer_s": 0.7520894939661957 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence241/gpu241-qwen06b-public.json b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-qwen06b-public.json new file mode 100644 index 00000000..43a2108a --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-qwen06b-public.json @@ -0,0 +1,777 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.864880312990863, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2533477320102975, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.7543227199930698, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.863383724994492, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.796506516984664, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 5.791821859020274, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.4773476280388422, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 3.5579012680100277, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.044472533976659, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.6066554809804074, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 4.067085198999848, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 5.526760092005134, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 5.54846198199084, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 5.7823985400027595, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 256, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.425002356991172 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.9895333859603852 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.1251549210282974 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.02\"} 1\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 29.55264288606122\nvllm:time_to_first_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.6429617053870442\nvllm:time_per_output_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.0\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 49.48445575305959\nvllm:e2e_request_latency_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 494, + "prefill_chunks": 0, + "decode_steps": 421, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 8.7879436298972, + "transfer_s": 0.7402957379817963 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence241/gpu241-qwen4b-public.json b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-qwen4b-public.json new file mode 100644 index 00000000..3e6ae728 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-qwen4b-public.json @@ -0,0 +1,777 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.3558407049858943, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 1.3509710889775306, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.7663904979708605, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 3.347964139014948, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 1.797310939989984, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 6.245678303996101, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 1.5209945480455644, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.9452596239862032, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 3.9197489300277084, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 3.9578521240036935, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 4.362375903001521, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 5.992897638003342, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 6.016431019990705, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 6.232266301987693, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 256, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 4.184855234983843 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.3966370840207674 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 3.2950828099856153 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 31.681192018964794\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 1\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 0.7286234277454507\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 1\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 54.26851827907376\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"Qwen3-4B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"Qwen3-4B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"Qwen3-4B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 535, + "prefill_chunks": 0, + "decode_steps": 418, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 10.656214797811117, + "transfer_s": 1.2329233231721446 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence241/gpu241-summary.json b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-summary.json new file mode 100644 index 00000000..de91c3d0 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence241/gpu241-summary.json @@ -0,0 +1,50 @@ +[ + { + "model": "qwen06b", + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "native_pd_serial_exact": 3, + "native_pd_serial_total": 3, + "producer_prefix_cache": false, + "producer_chunk": 256, + "remote_prefills": 21, + "producer_cached_tokens": 0 + }, + { + "model": "qwen4b", + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "native_pd_serial_exact": 3, + "native_pd_serial_total": 3, + "producer_prefix_cache": false, + "producer_chunk": 256, + "remote_prefills": 21, + "producer_cached_tokens": 0 + }, + { + "model": "koni", + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "native_pd_serial_exact": 3, + "native_pd_serial_total": 3, + "producer_prefix_cache": false, + "producer_chunk": 256, + "remote_prefills": 21, + "producer_cached_tokens": 0 + }, + { + "model": "gemma", + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "native_pd_serial_exact": 3, + "native_pd_serial_total": 3, + "producer_prefix_cache": false, + "producer_chunk": 256, + "remote_prefills": 21, + "producer_cached_tokens": 0 + } +] diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-pd-producer-cache241.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-pd-producer-cache241.md new file mode 100644 index 00000000..603fdc24 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-pd-producer-cache241.md @@ -0,0 +1,35 @@ +# Explicit Torch PD producer cache and chunk policy + +- **Date**: 2026-10-23 +- **Task**: aicomp #241 +- **Host / device**: CPU factory tests; GPU controls on two A10080GB PCIe +- **Commit**: baseline369c8ff; feat/241-pd-producer-cache candidate + +## Question + +Can serving explicitly disable only the Torch producer's prefix cache, without changing the existing default or decoder cache policy? + +## Setup + +The public Torch PD producer previously always used TorchPrefillEngine's prefix_cache=True default. Add --pd-prefill-no-prefix-cache, default false, and --pd-prefill-chunk, default2048. Reject nonpositive chunks before loading the model. The flag affects only --pd-prefill-device; vendor/subprocess producers retain their own configuration. Existing SimpleNamespace callers without this attribute retain the original default. + +## Command + +```bash +PYTHONPATH=opt_prime python -m pytest -q opt_prime/tests/test_serving_prefill_cache.py opt_prime/tests/test_serve_batched.py opt_prime/tests/test_pd_batching_disagg.py +# From opt_prime/: +PYTHONPATH=. python -m pytest -q tests +# Public serving flag: +python examples/pd/serve_batched.py --model "$MODEL" --device cuda:1 --pd-prefill-device cuda:0 --pd-prefill-no-prefix-cache --pd-prefill-chunk 256 +``` + +## Results + +- Baseline:1failed,2passed. Explicit cache-off still reused64tokens on a repeated128-token prompt. +- Candidate focused set:19passed,1existing flex_attention warning. Actual tiny Llama model checkpoint is loaded by the public factory; first prefill has0hits, repeated prefill has64hits by default/explicit false and0hits with opt-out. First token and KV shape remain valid. No mock engine. +- Chunk control red:1failed,3passed (64tokens executed as one chunk instead of two32-token chunks). Final focused suite20passed,1existing warning. Final full suite464passed,15skipped,135subtests passed,1existing flex_attention warning,142.43s. Previous cache-only full suite463passed,15skipped,135subtests. +- Supporting hardware controls under#240: Qwen3-0.6B BF16 concurrent equality8/12 with producer cache on versus12/12 with cache off; KONI7/12 versus12/12. Both FP32 cache-on controls also reach12/12. These controls used a process-local constructor override; cache-off alone leaves native/PD2K differences for Qwen0.6B and Gemma. Matching chunk256 additionally restores both3/3native/PD serial comparisons. Public factory/validator flags now pass all4models:84remote prefills,0producer cache-hit tokens,48/48concurrent equality,12/12native/PD serial equality; all cancel/disconnect/invalid-input checks pass. Inputs128/2048/8192,32outputs ignoring EOS,decode slots4. This uses actual GPU0-prefill/GPU1-decode, BF16, producer cache off/chunk256. + +## Conclusion + +The option provides a reproducible cache-off producer path without silently trading cache throughput for numerical reproducibility. It does not guarantee equality for all inputs, batches or devices and is not a model-quality or performance improvement claim. GPU public-flag validation is complete for this matrix. Production defaults remain cache-on/chunk2048; reproducible uncached256 execution must be selected explicitly. diff --git a/opt_prime/examples/pd/serve_batched.py b/opt_prime/examples/pd/serve_batched.py index b0338c36..5f3f38cf 100644 --- a/opt_prime/examples/pd/serve_batched.py +++ b/opt_prime/examples/pd/serve_batched.py @@ -30,9 +30,14 @@ def prefill_worker(a, slots, config): """the PD prefill side from the flags, or None (aggregated serving)""" if a.pd_prefill_device: + chunk = getattr(a, "pd_prefill_chunk", 2048) + if chunk <= 0: + raise ValueError("PD prefill chunk must be positive") from opt_prime.accel.pd.pd_batching import PrefillWorker, TorchPrefillEngine m = AutoModelForCausalLM.from_pretrained(a.model, dtype=getattr(torch, a.dtype)).to(a.pd_prefill_device).eval() - return PrefillWorker(TorchPrefillEngine(m, max_context=a.max_context, out_device=a.device + return PrefillWorker(TorchPrefillEngine(m, max_context=a.max_context, chunk=chunk, + prefix_cache=not getattr(a, "pd_prefill_no_prefix_cache", False), + out_device=a.device if a.backend == "torch" else "cpu")) if a.pd_prefill_python: import os, shlex @@ -118,6 +123,10 @@ def build(a): ap.add_argument("--fxb", default=None, help="RNGD: furiosa-llm compiled artifact") ap.add_argument("--rngd-devices", default="npu:0"); ap.add_argument("--no-prefix-cache", action="store_true") ap.add_argument("--pd-prefill-device", default=None, help="PD: torch prefill on this device (e.g. cuda:0)") + ap.add_argument("--pd-prefill-chunk", type=int, default=2048, + help="Torch --pd-prefill-device producer chunk size (default: 2048)") + ap.add_argument("--pd-prefill-no-prefix-cache", action="store_true", + help="disable prefix reuse in the Torch --pd-prefill-device producer (decoder policy unchanged)") ap.add_argument("--pd-prefill-python", default=None, help="PD: prefill_server under this interpreter") ap.add_argument("--pd-prefill-args", default="--engine torch --device cpu") ap.add_argument("--pd-prefill-pythonpath", default=None) diff --git a/opt_prime/tests/test_serving_prefill_cache.py b/opt_prime/tests/test_serving_prefill_cache.py new file mode 100644 index 00000000..d2d4f8af --- /dev/null +++ b/opt_prime/tests/test_serving_prefill_cache.py @@ -0,0 +1,74 @@ +"""The public factory must make remote producer cache policy controllable.""" +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from opt_prime.accel.pd import PDRequest +from transformers import LlamaConfig, LlamaForCausalLM + + +@pytest.mark.parametrize("disabled,expected_hit", [(None, 64), (False, 64), (True, 0)]) +def test_producer_cache_policy_controls_real_repeated_prefill(tmp_path, disabled, expected_hit): + torch.set_num_threads(2) + torch.manual_seed(241) + model = LlamaForCausalLM(LlamaConfig( + vocab_size=97, hidden_size=32, intermediate_size=48, num_hidden_layers=1, + num_attention_heads=4, num_key_value_heads=2, max_position_embeddings=256, + )).eval() + model.save_pretrained(tmp_path) + path = Path(__file__).parents[1] / "examples/pd/serve_batched.py" + spec = importlib.util.spec_from_file_location("serve_cache_test", path) + factory = importlib.util.module_from_spec(spec) + spec.loader.exec_module(factory) + args = SimpleNamespace(model=str(tmp_path), pd_prefill_device="cpu", dtype="float32", + backend="torch", device="cpu", max_context=256) + if disabled is not None: + args.pd_prefill_no_prefix_cache = disabled + worker = factory.prefill_worker(args, 1, model.config) + try: + prompt = tuple((i % 90) + 1 for i in range(128)) + first = worker.submit(PDRequest("first", prompt, 1)).result(timeout=10) + repeated = worker.submit(PDRequest("repeated", prompt, 1)).result(timeout=10) + assert first.prefill_cached == 0 + assert repeated.prefill_cached == expected_hit + assert first.token == repeated.token + assert first.keys.shape == repeated.keys.shape == (1, 2, 128, 8) + finally: + worker.close() + + +def test_producer_chunk_controls_real_prefill_boundaries(tmp_path, monkeypatch): + import opt_prime.accel.pd.pd_batching as pd + + model = LlamaForCausalLM(LlamaConfig( + vocab_size=97, hidden_size=32, intermediate_size=48, num_hidden_layers=1, + num_attention_heads=4, num_key_value_heads=2, max_position_embeddings=256, + )).eval() + model.save_pretrained(tmp_path) + path = Path(__file__).parents[1] / "examples/pd/serve_batched.py" + spec = importlib.util.spec_from_file_location("serve_chunk_test", path) + factory = importlib.util.module_from_spec(spec) + spec.loader.exec_module(factory) + spans = [] + original = pd.prefill_chunk + + def observe(model, kv, slot, ids, start, end, device): + spans.append((start, end)) + return original(model, kv, slot, ids, start, end, device) + + monkeypatch.setattr(pd, "prefill_chunk", observe) + args = SimpleNamespace(model=str(tmp_path), pd_prefill_device="cpu", dtype="float32", + backend="torch", device="cpu", max_context=256, pd_prefill_chunk=32) + worker = factory.prefill_worker(args, 1, model.config) + try: + out = worker.submit(PDRequest("chunk", tuple(range(1, 65)), 1)).result(timeout=10) + assert spans == [(0, 32), (32, 64)] + assert out.keys.shape == (1, 2, 64, 8) + finally: + worker.close() + for chunk in (0, -1): + args.pd_prefill_chunk = chunk + with pytest.raises(ValueError, match="positive"): + factory.prefill_worker(args, 1, model.config)