From 3ea0493322f53207627580db504f42947622cbe7 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 23 Oct 2026 18:40:14 +0900 Subject: [PATCH] test: extend HTTP matrix to four ATOM models and host cache (#211) --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/README.md | 10 +- opt_prime/examples/pd/http_matrix/STATUS.md | 2 +- .../atom-http-gemma3-1b-no-cache.json | 760 +++++++++++++++++ .../evidence/atom-http-koni-8b-no-cache.json | 761 +++++++++++++++++ .../atom-http-qwen3-06b-cache-fixed.json | 761 +++++++++++++++++ .../evidence/atom-http-qwen3-06b-cache.json | 761 +++++++++++++++++ .../evidence/atom-http-qwen3-06b-host.json | 762 ++++++++++++++++++ .../atom-http-qwen3-06b-no-cache.json | 760 +++++++++++++++++ .../evidence/atom-http-qwen3-4b-cache.json | 761 +++++++++++++++++ .../atom-http-qwen3-4b-host-fixed.json | 762 ++++++++++++++++++ .../evidence/atom-http-qwen3-4b-no-cache.json | 760 +++++++++++++++++ .../experiments/2026-10-23-atom-http211.md | 56 ++ opt_prime/examples/pd/http_matrix/validate.py | 41 +- 14 files changed, 6951 insertions(+), 7 deletions(-) create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-gemma3-1b-no-cache.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-koni-8b-no-cache.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-cache-fixed.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-cache.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-host.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-no-cache.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-cache.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-host-fixed.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-no-cache.json create mode 100644 opt_prime/examples/pd/http_matrix/experiments/2026-10-23-atom-http211.md diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index c7f9dd30..1d2f906a 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -1 +1,2 @@ {"id": "b61f99ae005a4258aa7f44d8edeca28d", "ts": "2026-10-23T18:08:44+09:00", "task": "Four-model HTTP matrix (aicomp #211)", "result": "All4 GPU models pass HTTP concurrency4/8, cancel/disconnect/drain/reuse checks. No-cacheBF16 and warm-cacheFP32 each48/48 concurrent outputs exact. Warm-cacheBF16 has31/48 raw equality; actualQwenHTTP tie explains observed branch. Warm-state cancellation isolates KONI cold-cache difference.", "next": "Publish GPUHTTP evidence; continue ATOM HTTP and RNGD routes. Preserve numerical/quality and true-overlap limitations.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"models": "4", "slots": "4", "concurrency": "4,8", "conditions": "bf16_no_cache,bf16_cache,fp32_cache"}, "artifacts": ["evidence/summary.json"], "experiments": ["experiments/2026-10-23-http-matrix211.md"], "labels": ["benchmark"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "GPU host", "branch": "main", "commit": "351d39d-dirty"} +{"id": "3563031e43c8478e951fff82f81bfe94", "ts": "2026-10-23T18:38:18+09:00", "task": "Four-model serving matrix (aicomp #201)", "result": "ATOM HTTP: all four models pass 128/2048/8192-token lifecycle checks; cache-off concurrent48/48 exact. Both Qwen device/host cache cases pass12/12 each. SIGPIPE candidate suppresses native disconnect backtraces6 to0 with unchanged outputs;14 unit/integration tests pass.", "next": "Publish ATOM HTTP matrix and SIGPIPE fix; continue missing RNGD and hybrid-state routes.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "atom", "input_tokens": "128,2048,8192", "output_tokens": "32", "hardware_batch": "1"}, "artifacts": ["evidence/atom-http-qwen3-4b-host-fixed.json"], "experiments": ["experiments/2026-10-23-atom-http211.md"], "labels": ["experiment", "bug"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "RNGD+ATOM host", "branch": "exp/211-atom-http", "commit": "working candidate"} diff --git a/opt_prime/examples/pd/http_matrix/README.md b/opt_prime/examples/pd/http_matrix/README.md index fa515f7c..6576bad8 100644 --- a/opt_prime/examples/pd/http_matrix/README.md +++ b/opt_prime/examples/pd/http_matrix/README.md @@ -1,4 +1,4 @@ -# Four-model GPU HTTP validation (#211/#212) +# Four-model GPU and ATOM HTTP validation (#211/#212) Run `validate.py --model MODEL --output result.json` under the engine environment with a free CUDA device. `--prefix-cache` enables the separate cache case; `--dtype float32` provides the numerical control. Offline immutable model snapshots are required. The default configuration is four paged execution slots, context9216, page64, prefill chunk256, inputs128/2048/8192 and32 fixed output tokens. This exercises the real HTTP service factory and sockets, not a mocked handler. @@ -9,3 +9,11 @@ Run `validate.py --model MODEL --output result.json` under the engine environmen See experiments/2026-10-23-http-matrix211.md and evidence/summary.json. Evidence includes the initial failed KONI cold-baseline cancellation assertion. Earlier runs preceded the warm-state cancellation control and some metadata additions; their commands/configurations are recorded in the experiment. Public copies normalize local model-directory labels in metrics only; raw measurements remain in the research workspace. This GPU experiment does not complete ATOM/RNGD HTTP or PD coverage, long-context RNGD support, hybrid NPU cache restoration, broader semantic quality or true compute/transfer overlap. + +## ATOM extension + +Use `--backend atom` in an isolated allocation exposing exactly one ATOM. Existing artifacts must use B1/context9216 and match `--atom-kv-blocks` (default3; KONI8B requires1). The two Qwen models support `--prefix-cache`; add `--host-kv-gib 2` to exercise the host tier and require positive observed host hits. Artifact precision is fixed; float32 is rejected for ATOM. Set OPT_PRIME_CACHE to the task-owned artifact cache. SDK/device bootstrap is deployment-specific and must map the allocated physical device to runtime-visible0 when required. + +Inputs128/2048/8192 and clients4/8 are queued over one ATOM execution lane, not hardware batching. All four cache-off runs pass lifecycle checks and48/48 concurrent/serial comparisons. Both Qwen device/host cache cases pass12/12 each. Gemma requires the causal-prefill SDK graph correction and #210 hybrid local-table/mask fix; its prefix/host/PD restoration remains unsupported. KONI's one-block configuration has no spare prefix block. + +See experiments/2026-10-23-atom-http211.md and evidence/atom-http-*.json. Files ending `fixed` include #214's HTTP MSG_NOSIGNAL fix; the Qwen0.6 matched rerun preserves all output sequences and eliminates the observed native disconnect backtraces. This does not establish all RNGD/heterogeneous routes or real transfer overlap. diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index ed94583b..0ce456da 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -1,3 +1,3 @@ # HTTP model matrix -State: review; goal remains active. Four GPU models pass no-cache lifecycle checks and48/48 concurrent equality; BF16 warm cache has raw numerical differences. FP32 warm cache passes lifecycle and48/48 concurrent equality. Actual Qwen HTTP trace identifies a BF16 logit tie. Next: ATOM HTTP configurations and missing RNGD routes; broader semantic inputs and cache/host-tier ownership checks. +State: review; full goal remains active. GPU evidence remains as recorded. ATOM four-model cache-off lifecycle passes with48/48 concurrent equality; both Qwen device/host cache cases pass12/12 each. SIGPIPE fix tracked separately in #214. Remaining: RNGD routes, hybrid-state restoration, real transfer overlap and broader semantic testing. diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-gemma3-1b-no-cache.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-gemma3-1b-no-cache.json new file mode 100644 index 00000000..e2462cc6 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-gemma3-1b-no-cache.json @@ -0,0 +1,760 @@ +{ + "model_name": "gemma-3-1b-it", + "prefix_cache": false, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 3, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.8503087810240686, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ], + "e2e_s": 0.40669069392606616, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 0.9704882020596415, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.453182274941355, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ], + "e2e_s": 2.840195677941665, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 5.454201999003999, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ], + "e2e_s": 1.1069058349821717, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 0.6773368080612272, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.876115923980251, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ], + "e2e_s": 1.4994417399866506, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 3.4001172699499875, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 4.549677124014124, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ], + "e2e_s": 4.90743209910579, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 5.434968836954795, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ], + "e2e_s": 0.3936310880817473 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 0.5671621320070699 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.2390507919481024 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ], + "after_tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ], + "after_tokens": [ + 9079, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"gemma-3-1b-it\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"gemma-3-1b-it\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"gemma-3-1b-it\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"gemma-3-1b-it\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"gemma-3-1b-it\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"gemma-3-1b-it\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"gemma-3-1b-it\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"gemma-3-1b-it\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.75\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"1.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"2.5\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"5.0\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"gemma-3-1b-it\"} 27.363004976999946\nvllm:time_to_first_token_seconds_count{model_name=\"gemma-3-1b-it\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.025\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.05\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"gemma-3-1b-it\"} 0.23127831826587356\nvllm:time_per_output_token_seconds_count{model_name=\"gemma-3-1b-it\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.5\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.8\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"1.0\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"1.5\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"2.0\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"2.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"5.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"gemma-3-1b-it\"} 34.53263284324203\nvllm:e2e_request_latency_seconds_count{model_name=\"gemma-3-1b-it\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"gemma-3-1b-it\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"gemma-3-1b-it\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"gemma-3-1b-it\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"gemma-3-1b-it\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"gemma-3-1b-it\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"gemma-3-1b-it\",block_size=\"256\",enable_prefix_caching=\"True\"} 1\n", + "prefix_hit_tokens": 0.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-koni-8b-no-cache.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-koni-8b-no-cache.json new file mode 100644 index 00000000..1a481706 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-koni-8b-no-cache.json @@ -0,0 +1,761 @@ +{ + "model_name": "KONI-Llama3.1-8B-Instruct-20241024", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 1, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 17.8498541880399, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ], + "e2e_s": 2.1889036119682714, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 6.015203595976345, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 15.594296137918718, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ], + "e2e_s": 17.845708256936632, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 39.04894086997956, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ], + "e2e_s": 2.4593210630118847, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 6.622534933965653, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 16.607850600034, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ], + "e2e_s": 32.853210115921684, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 20.552890663035214, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 30.45911693002563, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ], + "e2e_s": 35.16151285509113, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 39.01228911697399, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ], + "e2e_s": 2.182843631017022 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 3.735379097983241 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 9.523682309081778 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 1120 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.04\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.06\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.08\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.1\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.5\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.75\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"1.0\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"2.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"5.0\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"7.5\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"10.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 207.49166697217152\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.05\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.075\"} 15\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 1.3703221580306546\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"1.0\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"1.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"2.0\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"2.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"5.0\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"10.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"15.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"20.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"30.0\"} 15\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 249.97165387112182\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-Llama3.1-8B-Instruct-20241024\"} 0\n", + "prefix_hit_tokens": 0.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-cache-fixed.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-cache-fixed.json new file mode 100644 index 00000000..ad9a24b6 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-cache-fixed.json @@ -0,0 +1,761 @@ +{ + "model_name": "Qwen3-0.6B", + "prefix_cache": true, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 3, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.248584700981155, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.28151508001610637, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.5722270999103785, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.9905206640250981, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2314512348966673, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.522928233956918, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.27896868309471756, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 2.0992609749082476, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.805448218015954, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.6197337730554864, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 2.374678032938391, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.955605463939719, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.208059304044582, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 4.500310557894409, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.2541884219972417 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.466158407041803 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.3094038530252874 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-0.6B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-0.6B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-0.6B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.02\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.04\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.06\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.08\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 22.532509238808416\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 15\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.17089263726806933\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.8\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.5\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 15\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-0.6B\"} 27.830180994118564\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-0.6B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-0.6B\"} 19584\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-0.6B\",block_size=\"128\",enable_prefix_caching=\"True\"} 1\n", + "prefix_hit_tokens": 19584.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-cache.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-cache.json new file mode 100644 index 00000000..7a9d28f5 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-cache.json @@ -0,0 +1,761 @@ +{ + "model_name": "Qwen3-0.6B", + "prefix_cache": true, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 3, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.2128619889263064, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.25869818101637065, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.5717005589976907, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.978555666981265, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.203579452005215, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.730229896027595, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.3080808660015464, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.7815210010157898, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.09453048708383, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.9762356269638985, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 2.676780418958515, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.232980364002287, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.448256555013359, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 4.70400058105588, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.3220145619707182 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.641035900101997 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.3148153700167313 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-0.6B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-0.6B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-0.6B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.02\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.04\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.06\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.08\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 22.62249407276977\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 13\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.18044153184464742\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.8\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-0.6B\"} 28.21618155995384\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-0.6B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-0.6B\"} 19584\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-0.6B\",block_size=\"128\",enable_prefix_caching=\"True\"} 1\n", + "prefix_hit_tokens": 19584.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-host.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-host.json new file mode 100644 index 00000000..4f798a38 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-host.json @@ -0,0 +1,762 @@ +{ + "model_name": "Qwen3-0.6B", + "prefix_cache": true, + "host_kv_gib": 2.0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 3, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.776837351033464, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.4009785130620003, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.7123533319681883, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.7701384159736335, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2193158380687237, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 2.6758189148968086, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.28543873806484044, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.5685252600815147, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.3729288800386712, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8304092199541628, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 1.6564582890132442, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.0843227349687368, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.3504424389684573, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 2.6486748220631853, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.2593580079264939 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.48742022493388504 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.6243514850502834 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-0.6B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-0.6B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-0.6B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.02\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 13.709253596607596\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 15\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.17516412297564168\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.8\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-0.6B\"} 19.139341408852488\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-0.6B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-0.6B\"} 33792\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-0.6B\",block_size=\"128\",enable_prefix_caching=\"True\"} 1\n# HELP airm_kv_host_hit_tokens_total Prompt tokens reloaded from the host tier.\n# TYPE airm_kv_host_hit_tokens_total counter\nairm_kv_host_hit_tokens_total{model_name=\"Qwen3-0.6B\"} 16128\n# HELP airm_kv_offload_blocks_total Blocks written back from the device to the host tier.\n# TYPE airm_kv_offload_blocks_total counter\nairm_kv_offload_blocks_total{model_name=\"Qwen3-0.6B\"} 3\n# HELP airm_kv_reload_blocks_total Blocks reloaded from the host tier to the device.\n# TYPE airm_kv_reload_blocks_total counter\nairm_kv_reload_blocks_total{model_name=\"Qwen3-0.6B\"} 2\n", + "prefix_hit_tokens": 33792.0, + "host_hit_tokens": 16128.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-no-cache.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-no-cache.json new file mode 100644 index 00000000..4c4c759e --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-06b-no-cache.json @@ -0,0 +1,760 @@ +{ + "model_name": "Qwen3-0.6B", + "prefix_cache": false, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 3, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.275814192951657, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.2261266279965639, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.636282664956525, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.270193413947709, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8358748221071437, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.837453335989267, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.6535679040243849, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.43785319302696735, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.099043051013723, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.7397728179348633, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 2.5233542770147324, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.399380394956097, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.9527563330484554, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 4.8296007219469175, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.2767834940459579 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.46634663396980613 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.83812005398795 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-0.6B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-0.6B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-0.6B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.02\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 23.095241924049333\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 15\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.1551467709979343\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.8\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.5\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.0\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-0.6B\"} 27.904791824985296\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-0.6B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-0.6B\",block_size=\"128\",enable_prefix_caching=\"True\"} 1\n", + "prefix_hit_tokens": 0.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-cache.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-cache.json new file mode 100644 index 00000000..5863a805 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-cache.json @@ -0,0 +1,761 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": true, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 3, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 5.370610225945711, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.2549490919336677, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.580465944018215, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 4.1113049370469525, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 5.361873599002138, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 19.83599412906915, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.2714701240183786, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 3.527654197998345, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 10.421250027953647, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 4.826293441001326, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 11.771753660985269, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 18.521116749965586, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 12.998552325065248, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 19.811023617046885, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.3294664029963315 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.387576421024278 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 5.532931986963376 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 86.65211479645222\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 0.768158285566906\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 110.46502164902631\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 19584\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"128\",enable_prefix_caching=\"True\"} 1\n", + "prefix_hit_tokens": 19584.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-host-fixed.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-host-fixed.json new file mode 100644 index 00000000..0d9c3d37 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-host-fixed.json @@ -0,0 +1,762 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": true, + "host_kv_gib": 2.0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 3, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 5.3771037210244685, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.4422351559624076, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.70951707707718, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 4.1945555419661105, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 5.370230198954232, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 10.77056910796091, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.7858462249860168, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.2965235960436985, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 4.345374454976991, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 5.557994809001684, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 9.474480802076869, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 6.989578875014558, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 8.195752296014689, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 10.742149700061418, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.2765175310196355 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.4192551710875705 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 5.704140087007545 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 53.89968488528393\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 0.7480128487354264\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 77.08808319608215\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 33792\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"128\",enable_prefix_caching=\"True\"} 1\n# HELP airm_kv_host_hit_tokens_total Prompt tokens reloaded from the host tier.\n# TYPE airm_kv_host_hit_tokens_total counter\nairm_kv_host_hit_tokens_total{model_name=\"Qwen3-4B\"} 8064\n# HELP airm_kv_offload_blocks_total Blocks written back from the device to the host tier.\n# TYPE airm_kv_offload_blocks_total counter\nairm_kv_offload_blocks_total{model_name=\"Qwen3-4B\"} 3\n# HELP airm_kv_reload_blocks_total Blocks reloaded from the host tier to the device.\n# TYPE airm_kv_reload_blocks_total counter\nairm_kv_reload_blocks_total{model_name=\"Qwen3-4B\"} 1\n", + "prefix_hit_tokens": 33792.0, + "host_hit_tokens": 8064.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-no-cache.json b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-no-cache.json new file mode 100644 index 00000000..d39cdf67 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/atom-http-qwen3-4b-no-cache.json @@ -0,0 +1,760 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": false, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.11.0+cpu", + "dtype": "compiled artifact", + "backend": "atom", + "slots": 1, + "atom_kv_blocks": 3, + "device": "allocated ATOM", + "cases": [ + { + "concurrency": 4, + "wall_s": 11.202900396077894, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.2418438270688057, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 3.519857676932588, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 9.990824809996411, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 11.197645794018172, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 22.908283034921624, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.2329478060128167, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 3.449207398109138, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 9.747248220955953, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 14.40843369893264, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 11.972066989983432, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 20.68713503505569, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 13.183753640041687, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 22.89388387207873, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.2069404810899869 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.213739449973218 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 6.422640762059018 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 115.17362943396438\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 0.7402365743182601\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 138.12096323783044\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"128\",enable_prefix_caching=\"True\"} 1\n", + "prefix_hit_tokens": 0.0 +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-atom-http211.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-atom-http211.md new file mode 100644 index 00000000..9e84621e --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-atom-http211.md @@ -0,0 +1,56 @@ +# Four-model ATOM HTTP lifecycle and cache matrix + +- **Date**: 2026-10-23 +- **Task**: aicomp #211, parent #201; engine prerequisite #207 +- **Host / device**: RNGD+ATOM host, one isolated allocated ATOM; no RNGD allocation for this experiment. rebel-compiler0.11.0, optimum-rbln0.11.0.post1, torch2.11.0+cpu, Transformers5.8.1 +- **Commit**: engine21f2c87 (PR210), GPU HTTP harness16f58f7 (PR213), plus this branch's backend/host-tier options + +## Question + +Do the same four checkpoints handle actual HTTP concurrency, stream cancellation/disconnect and reuse on ATOM? Are Qwen device/host cache paths exercised correctly? + +## Setup + +Reuse the four artifacts from the long-input matrix. All have B1/context9216; Qwen3-0.6B,Qwen3-4B,Gemma3-1B use3 full-context blocks; KONI8B uses1 because its3-block graph did not compile (#208). Gemma artifact includes the upstream causal-prefill correction, and the actual batcher includes PR210 local-table/mask changes. Candidate batching.py SHA2561d2f3d5d7d7e34a766b9104ad172325ac8256caf69d0a95135ac75ab9bc85d81 matches local and deployed copies. + +The production serve_batched.build factory and BatchedOwner serve real loopback HTTP sockets. A process-local module loader selects the candidate engine module; shared installed packages remain unchanged. OPT_PRIME_CACHE points to a task-owned directory whose atom-rbln symlink points at the already compiled artifacts. Bootstrap checks exactly one allocated physical device and maps it to runtime-visible device0. + +Inputs128/2048/8192, outputs32, temperature0, ignore_eos. Concurrent client counts4/8 are **queued requests over one execution lane**, not hardware batch4/8. Same warm-state cancellation control as the GPU harness: explicit cancel/disconnect after first streamed text, confirm request removal, compare before/after outputs, readiness and empty-prompt400. Prefix and host tiers are separate runs; positive observed hit counters are required. + +Checkpoints/revisions match PR203: Qwen3-0.6B c1899de289a04d12100db370d81485cdf75e47ca; Qwen3-4B1cfa9a7208912126459214e8b04321603b3df60c; KONI8B0172bf7e73977ae213613529f3d2d5400a33f678; Gemma3-1B5b11413a10db4e486ef16a20101fd028f8f2499c. + +## Command + +Within the isolated allocation and same SDK/bootstrap environment as PR203, with the task-owned cache symlink configured: + +```bash +bash run-atom-http211.sh --backend atom --model Qwen/Qwen3-0.6B --output atom-http-qwen3-06b-no-cache.json +bash run-atom-http211.sh --backend atom --model Qwen/Qwen3-4B --output atom-http-qwen3-4b-no-cache.json +bash run-atom-http211.sh --backend atom --model unsloth/gemma-3-1b-it --output atom-http-gemma3-1b-no-cache.json +bash run-atom-http211.sh --backend atom --model KISTI-KONI/KONI-Llama3.1-8B-Instruct-20241024 --atom-kv-blocks 1 --output atom-http-koni-8b-no-cache.json +# Qwen cache cases additionally use --prefix-cache; host cases add --host-kv-gib 2. +``` + +The launcher invokes container_bootstrap.py atom_http211.py; atom_http211.py selects the candidate batching module, imports the same process-local SDK correction used for Gemma compilation and invokes validate_http211.py. The last file is the checked-in HTTP harness. No artifact recompile is intended. + +## Results + +All four no-cache models pass HTTP lifecycle checks, with 12/12 concurrent outputs each equal to their own serial baseline (48/48 total). Cancellation/disconnect drain and subsequent reuse pass. Inputs are 128/2048/8192 tokens, output32. + +| Model / cache | Concurrent exact | Prefix hit tokens | Host hit tokens | C4 / C8 wall seconds | +|---|---:|---:|---:|---:| +| atom-http-gemma3-1b-no-cache | 12/12 | 0 | 0 | 2.850 / 5.454 | +| atom-http-koni-8b-no-cache | 12/12 | 0 | 0 | 17.850 / 39.049 | +| atom-http-qwen3-06b-cache-fixed | 12/12 | 19584 | 0 | 1.249 / 4.523 | +| atom-http-qwen3-06b-cache | 12/12 | 19584 | 0 | 1.213 / 4.730 | +| atom-http-qwen3-06b-host | 12/12 | 33792 | 16128 | 1.777 / 2.676 | +| atom-http-qwen3-06b-no-cache | 12/12 | 0 | 0 | 2.276 / 4.837 | +| atom-http-qwen3-4b-cache | 12/12 | 19584 | 0 | 5.371 / 19.836 | +| atom-http-qwen3-4b-host-fixed | 12/12 | 33792 | 8064 | 5.377 / 10.771 | +| atom-http-qwen3-4b-no-cache | 12/12 | 0 | 0 | 11.203 / 22.908 | + +Times are single warm bursts, not statistical performance claims. Four-model cache-off and both Qwen device/host-cache cases pass. Gemma and KONI cache limitations remain below. Original runs emit native SIGPIPE backtraces at closed-socket writes; issue214 fixes this locally using MSG_NOSIGNAL. Qwen0.6 device-cache rerun preserves all 3 serial and 12 concurrent token sequences, with backtrace headers6→0. Qwen4B host-tier candidate also passes with0 headers. See experiments/2026-10-23-http-sigpipe214.md. Raw evidence: analysis/atom-http-211/. + +## Conclusion + +Standalone B1 HTTP lifecycle evidence is progressing. This does not establish hardware B>1, NPU PD, GPU/ATOM numerical equality, semantic quality or real compute/transfer overlap. Gemma hybrid prefix/host/PD restoration remains unsupported; KONI1-block has no spare prefix-cache block. Those configurations must not be counted as validated cache cases. diff --git a/opt_prime/examples/pd/http_matrix/validate.py b/opt_prime/examples/pd/http_matrix/validate.py index 336b99dc..7c907f46 100644 --- a/opt_prime/examples/pd/http_matrix/validate.py +++ b/opt_prime/examples/pd/http_matrix/validate.py @@ -25,15 +25,30 @@ def main(): p.add_argument("--model", required=True) p.add_argument("--output", required=True) p.add_argument("--prefix-cache", action="store_true") + p.add_argument("--backend", choices=["torch", "atom"], default="torch") + p.add_argument("--atom-kv-blocks", type=int, default=3) + p.add_argument("--host-kv-gib", type=float, default=0) p.add_argument("--dtype", choices=["bfloat16", "float32"], default="bfloat16") a = p.parse_args() + if a.backend == "atom" and a.dtype != "bfloat16": + p.error("ATOM precision is determined by the compiled artifact") + if a.host_kv_gib < 0 or (a.host_kv_gib and not a.prefix_cache): + p.error("host KV requires --prefix-cache and a nonnegative capacity") + devices = [] + if a.backend == "atom": + nodes = list(Path("/dev").glob("rbln[0-9]*")) + assert len(nodes) == 1, "requires one isolated allocated ATOM" + devices = [int(nodes[0].name[4:])] torch.set_num_threads(4) args = SimpleNamespace( model=a.model, - backend="torch", + backend=a.backend, device="cuda:0", dtype=a.dtype, - slots=4, + slots=4 if a.backend == "torch" else 1, + devices=devices, + atom_kv_blocks=a.atom_kv_blocks, + pd_remote_only=False, max_context=9216, max_output=256, kv="paged", @@ -41,7 +56,7 @@ def main(): chunk=256, prefix_cache=a.prefix_cache, kv_events=None, - kv_host_memory_gb=0, + kv_host_memory_gb=a.host_kv_gib, reload_bandwidth=None, prefill_rate=None, remote_kv=None, @@ -51,11 +66,17 @@ def main(): result = { "model_name": Path(a.model).name, "prefix_cache": a.prefix_cache, + "host_kv_gib": a.host_kv_gib, "passed": False, "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", "torch": torch.__version__, - "dtype": a.dtype, - "device": torch.cuda.get_device_name(0), + "dtype": a.dtype if a.backend == "torch" else "compiled artifact", + "backend": a.backend, + "slots": args.slots, + "atom_kv_blocks": a.atom_kv_blocks if a.backend == "atom" else None, + "device": torch.cuda.get_device_name(0) + if a.backend == "torch" + else "allocated ATOM", "cases": [], } output = Path(a.output) @@ -210,6 +231,16 @@ def client(): assert result["prefix_hit_tokens"] > 0, ( "cache enabled without observed hits" ) + if a.host_kv_gib: + host_hits = [ + float(line.rsplit(" ", 1)[1]) + for line in result["metrics"].splitlines() + if line.startswith("airm_kv_host_hit_tokens_total{") + ] + result["host_hit_tokens"] = sum(host_hits) + assert host_hits and result["host_hit_tokens"] > 0, ( + "host tier not exercised" + ) result["passed"] = True except Exception as exc: # noqa: BLE001 - preserve failed hardware evidence result["error"] = repr(exc)