diff --git a/helm/dashboards/vllm-dashboard.json b/helm/dashboards/vllm-dashboard.json index 753a9be11..e75a7f2a1 100644 --- a/helm/dashboards/vllm-dashboard.json +++ b/helm/dashboards/vllm-dashboard.json @@ -369,7 +369,7 @@ "targets": [ { "editorMode": "builder", - "expr": "avg(vllm:time_per_output_token_seconds_sum) / avg(vllm:time_per_output_token_seconds_count)", + "expr": "avg(vllm:request_time_per_output_token_seconds_sum) / avg(vllm:request_time_per_output_token_seconds_count)", "legendFormat": "Avg ITL", "range": true, "refId": "A" @@ -656,6 +656,7 @@ "color": { "mode": "palette-classic" }, + "unit": "percentunit", "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -729,7 +730,7 @@ "targets": [ { "editorMode": "builder", - "expr": "vllm:gpu_cache_usage_perc", + "expr": "vllm:kv_cache_usage_perc", "legendFormat": "{{instance}}", "range": true, "refId": "A" @@ -834,70 +835,6 @@ "title": "GPU KV Cache Hit Rate", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Requests moved from GPU to CPU", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 0 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 21, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.0.0+security-01", - "targets": [ - { - "editorMode": "builder", - "expr": "vllm:num_requests_swapped", - "legendFormat": "Swapped Requests", - "range": true, - "refId": "A" - } - ], - "title": "Number of Swapped Requests", - "type": "stat" - }, { "collapsed": false, "gridPos": { @@ -921,6 +858,7 @@ "color": { "mode": "palette-classic" }, + "unit": "percentunit", "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -995,7 +933,7 @@ { "disableTextWrap": false, "editorMode": "builder", - "expr": "vllm:gpu_cache_usage_perc", + "expr": "vllm:kv_cache_usage_perc", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "GPU Usage", diff --git a/helm/dashboards/vllm-model-metrics-dashboard.json b/helm/dashboards/vllm-model-metrics-dashboard.json index e3e579391..c2b9df991 100644 --- a/helm/dashboards/vllm-model-metrics-dashboard.json +++ b/helm/dashboards/vllm-model-metrics-dashboard.json @@ -402,7 +402,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:request_time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "instant": false, @@ -418,7 +418,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:request_time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -435,7 +435,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:request_time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -452,7 +452,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:request_time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -468,7 +468,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(vllm:time_per_output_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])", + "expr": "rate(vllm:request_time_per_output_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:request_time_per_output_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])", "hide": false, "instant": false, "legendFormat": "Mean", @@ -484,7 +484,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "Number of requests in RUNNING, WAITING, and SWAPPED state", + "description": "Number of requests in RUNNING, and WAITING", "fieldConfig": { "defaults": { "color": { @@ -577,23 +577,6 @@ "refId": "A", "useBackend": false }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "vllm:num_requests_swapped{model_name=\"$model_name\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Num Swapped", - "range": true, - "refId": "B", - "useBackend": false - }, { "datasource": { "type": "prometheus", @@ -608,7 +591,7 @@ "instant": false, "legendFormat": "Num Waiting", "range": true, - "refId": "C", + "refId": "B", "useBackend": false } ], @@ -869,24 +852,11 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "vllm:gpu_cache_usage_perc{model_name=\"$model_name\"}", + "expr": "vllm:kv_cache_usage_perc{model_name=\"$model_name\"}", "instant": false, "legendFormat": "GPU Cache Usage", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "vllm:cpu_cache_usage_perc{model_name=\"$model_name\"}", - "hide": false, - "instant": false, - "legendFormat": "CPU Cache Usage", - "range": true, - "refId": "B" } ], "title": "Cache Utilization",