diff --git a/deploy_common/__init__.py b/deploy_common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deploy_common/reject_drain.py b/deploy_common/reject_drain.py new file mode 100644 index 0000000..321dbf4 --- /dev/null +++ b/deploy_common/reject_drain.py @@ -0,0 +1,105 @@ +"""Bounded, body-blind draining after an early gateway rejection.""" +from contextlib import suppress +import queue +import select +import socket +import threading +import time + + +# Keep a full forwarding wave without exceeding Windows select()'s 512-fd set. +MAX_REJECTIONS = 480 +MAX_DRAIN_BYTES = 128 * 1024 * 1024 +DRAIN_TIMEOUT = 30 + + +class RejectionDrainer: + """Keep rejected sockets readable without holding a request admission slot.""" + + def __init__(self): + self._slots = threading.BoundedSemaphore(MAX_REJECTIONS) + self._pending = queue.SimpleQueue() + self._stop = threading.Event() + self._lock = threading.Lock() + self.full = 0 + self.duplicate_failures = 0 + self.thread = threading.Thread(target=self._run, daemon=True) + self.thread.start() + + def submit(self, connection): + with self._lock: + if self._stop.is_set(): + return False + if not self._slots.acquire(blocking=False): + self.full += 1 + return False + duplicate = None + try: + duplicate = connection.dup() + duplicate.setblocking(False) + self._pending.put(duplicate) + return True + except OSError: + self.duplicate_failures += 1 + if duplicate is not None: + duplicate.close() + self._slots.release() + return False + + def close(self): + with self._lock: + self._stop.set() + self.thread.join(1) + + def _run(self): + active = {} + + def release(conn): + active.pop(conn, None) + conn.close() + self._slots.release() + + while not self._stop.is_set(): + while True: + try: + conn = self._pending.get_nowait() + except queue.Empty: + break + active[conn] = [time.monotonic() + DRAIN_TIMEOUT, MAX_DRAIN_BYTES] + if not active: + self._stop.wait(.02) + continue + try: + readable, _, failed = select.select(list(active), [], list(active), .02) + except (OSError, ValueError): + readable, failed = [], list(active) + now = time.monotonic() + for conn in set(readable + failed + [conn for conn, (deadline, _) in active.items() + if deadline <= now]): + if conn not in active: + continue + deadline, remaining = active[conn] + if conn in failed or deadline <= now: + release(conn) + continue + try: + data = conn.recv(min(65536, remaining)) + except BlockingIOError: + continue + except OSError: + data = b'' + remaining -= len(data) + if not data or remaining <= 0: + release(conn) + else: + active[conn][1] = remaining + for conn in list(active): + release(conn) + while True: + try: + conn = self._pending.get_nowait() + except queue.Empty: + break + with suppress(OSError): + conn.close() + self._slots.release() diff --git a/deploy_gateway/server.py b/deploy_gateway/server.py index d0dffed..2773cd5 100644 --- a/deploy_gateway/server.py +++ b/deploy_gateway/server.py @@ -4,29 +4,98 @@ Client Authorization is consumed here and never forwarded to the model. """ import argparse +from concurrent.futures import ThreadPoolExecutor import hashlib import hmac from http.client import HTTPConnection, HTTPException, IncompleteRead from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import ipaddress import json import math from pathlib import Path import re import select import socket +import sys import threading import time from urllib.parse import urlsplit from .routing import Busy, SessionRouter, Unavailable, affinity_key, prefix_rank +from deploy_common.reject_drain import RejectionDrainer HOP_HEADERS = {'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailer', 'transfer-encoding', 'upgrade'} +def _check_statistics_paths(database, key_store, bindings_file, state_path, control_file): + """Reject conflicts before opening SQLite or creating its sidecars.""" + database = Path(database) + sqlite_paths = [database, *(Path(str(database) + suffix) for suffix in ('-wal', '-shm', '-journal'))] + protected = [Path(key_store), Path(bindings_file), Path(str(bindings_file) + '.new')] + if state_path: + protected += [Path(state_path), Path(state_path).with_suffix('.new')] + if control_file: + protected += [Path(control_file), Path(str(control_file) + '.new')] + for left in sqlite_paths: + for right in protected: + if left.resolve() == right.resolve(): + raise ValueError('STATISTICS_PATH_OVERLAP') + try: + if left.samefile(right): + raise ValueError('STATISTICS_PATH_OVERLAP') + except FileNotFoundError: + pass + + +def pd_instance_healthy(row, cache): + """Probe every reported rank; cache observations within one gateway cycle.""" + endpoints = row.get('endpoints') + if not isinstance(endpoints, list) or not 0 < len(endpoints) <= 1024: + return False + targets = [] + for endpoint in endpoints: + if not isinstance(endpoint, dict) or endpoint.get('headless') is True: + return False + try: + host_value = endpoint.get('ip') + if not isinstance(host_value, str): + return False + host = str(ipaddress.ip_address(host_value)) + port = endpoint.get('business_port') + if isinstance(port, str) and port.isdecimal(): + port = int(port) + if type(port) is not int or not 1 <= port <= 65535: + return False + except (ValueError, TypeError): + return False + targets.append((host, port)) + if len(set(targets)) != len(targets): + return False + + def probe(target): + connection = HTTPConnection(target[0], target[1], timeout=2) + try: + connection.request('GET', '/health', headers={'Connection': 'close'}) + return connection.getresponse().status == 200 + except (OSError, HTTPException, ValueError): + return False + finally: + connection.close() + + missing = [target for target in targets if target not in cache] + if missing: + with ThreadPoolExecutor(max_workers=min(16, len(missing))) as pool: + cache.update(zip(missing, pool.map(probe, missing))) + return all(cache[target] for target in targets) + + class Backends(SessionRouter): - def __init__(self, ports, state_path, bindings_file, max_active=32, max_waiting=32, queue_timeout=5, control_file=None): + def __init__(self, ports, state_path, bindings_file, max_active=32, max_waiting=32, queue_timeout=5, control_file=None, readiness_ports=None, pd_model=None, pd_probe_endpoints=False): super().__init__(ports, bindings_file, max_active=max_active, max_waiting=max_waiting, queue_timeout=queue_timeout) + self.readiness_ports = dict(readiness_ports or {}) + self.pd_model = pd_model + self.pd_probe_endpoints = pd_probe_endpoints self.control_file = Path(control_file) if control_file else None self.stop = threading.Event() self.state_path = Path(state_path) if state_path else None @@ -56,14 +125,40 @@ def probe(self, bootstrap=False): with self.lock: self.enabled_ports = enabled versions = dict(self.versions) - observed = {} + observed, endpoint_cache = {}, {} for port in self.ports: - connection = HTTPConnection('127.0.0.1', port, timeout=1) + readiness_port = self.readiness_ports.get(port) + connection = HTTPConnection('127.0.0.1', readiness_port or port, timeout=1) try: - connection.request('GET', '/health', headers={'Connection': 'close'}) + connection.request('GET', '/readiness' if readiness_port else '/health', headers={'Connection': 'close'}) response = connection.getresponse() - observed[port] = response.status == 200 - except (OSError, HTTPException): + if readiness_port and response.status == 200: + body = response.read(65537) + data = json.loads(body) if len(body) <= 65536 else None + observed[port] = isinstance(data, dict) and data.get('ready') is True + if observed[port] and self.pd_model is not None: + connection.close() + connection.request('GET', '/instances', headers={'Connection': 'close'}) + response = connection.getresponse() + body = response.read(1048577) + data = json.loads(body) if response.status == 200 and len(body) <= 1048576 else None + instances = data.get('instances') if isinstance(data, dict) else None + candidates = [row for row in instances if isinstance(row, dict) + and row.get('role') in ('prefill', 'decode') + and row.get('model_name') == self.pd_model + and row.get('healthy') is True and row.get('pool') == 'available' + and row.get('status') == 'active'] if isinstance(instances, list) else [] + roles = set() + for row in candidates: + if row['role'] not in roles and (not self.pd_probe_endpoints + or pd_instance_healthy(row, endpoint_cache)): + roles.add(row['role']) + if roles == {'prefill', 'decode'}: + break + observed[port] = {'prefill', 'decode'} <= roles + else: + observed[port] = response.status == 200 and not readiness_port + except (OSError, HTTPException, ValueError, RuntimeError, RecursionError): observed[port] = False finally: connection.close() @@ -92,7 +187,13 @@ class Gateway(ThreadingHTTPServer): allow_reuse_address = True request_queue_size = 256 - def __init__(self, address, key_store, upstream_port=19001, backup_port=19002, state_path=None, bindings_file=None, upstream_timeout=10800, model_context_limit=None, backend_context_limits=None, *, backend_ports=None, max_active=32, max_waiting=32, queue_timeout=5, control_file=None, backend_managed_affinity=False): + def server_bind(self): + if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) + self.allow_reuse_address = False + super().server_bind() + + def __init__(self, address, key_store, upstream_port=19001, backup_port=19002, state_path=None, bindings_file=None, upstream_timeout=10800, model_context_limit=None, backend_context_limits=None, *, backend_ports=None, max_active=32, max_waiting=32, queue_timeout=5, control_file=None, backend_managed_affinity=False, backend_readiness_ports=None, pd_model=None, pd_probe_endpoints=False): if bindings_file is None or Path(bindings_file).resolve() == Path(key_store).resolve(): raise ValueError('A separate private bindings file is required') if state_path and Path(state_path).resolve() in (Path(key_store).resolve(), Path(bindings_file).resolve()): @@ -113,6 +214,15 @@ def __init__(self, address, key_store, upstream_port=19001, backup_port=19002, s ports = list(backend_ports) if backend_ports is not None else [upstream_port, backup_port] if not ports or len(set(ports)) != len(ports) or any(type(p) is not int or not 1 <= p <= 65535 for p in ports): raise ValueError('Invalid backend ports') + readiness_ports = dict(backend_readiness_ports or {}) + if any(type(p) is not int or p not in ports or type(target) is not int or not 1 <= target <= 65535 + for p, target in readiness_ports.items()): + raise ValueError('Invalid backend readiness mapping') + if pd_model is not None and (not isinstance(pd_model, str) or not pd_model.strip() + or set(readiness_ports) != set(ports)): + raise ValueError('PD model requires a management readiness port for each backend') + if type(pd_probe_endpoints) is not bool or (pd_probe_endpoints and pd_model is None): + raise ValueError('PD endpoint probing requires a PD model') self.backend_context_limits = dict(backend_context_limits or {}) if self.backend_context_limits and (set(self.backend_context_limits) != set(ports) or any(type(p) is not int or type(limit) is not int or not 0 < limit <= 10**9 for p, limit in self.backend_context_limits.items())): @@ -131,12 +241,21 @@ def __init__(self, address, key_store, upstream_port=19001, backup_port=19002, s raise ValueError('No active gateway keys') super().__init__(address, Handler) try: - self.backends = Backends(ports, state_path, bindings_file, max_active, max_waiting, queue_timeout, control_file) + self.backends = Backends(ports, state_path, bindings_file, max_active, max_waiting, queue_timeout, control_file, readiness_ports, pd_model, pd_probe_endpoints) + try: + self.rejection_drainer = RejectionDrainer() + except Exception: + self.backends.stop.set() + raise except Exception: super().server_close() raise def server_close(self): + if not hasattr(self, 'backends'): + return super().server_close() + if hasattr(self, 'rejection_drainer'): + self.rejection_drainer.close() self.backends.stop.set() super().server_close() @@ -181,7 +300,26 @@ class Handler(BaseHTTPRequestHandler): def log_message(self, format, *args): pass + def _observe(self, event, *args): + observer = getattr(self, '_stats_observer', None) + if observer is not None: + try: + getattr(observer, event)(*args) + except Exception: + try: + observer.fail() + except Exception: + pass + def error(self, status, message): + lengths = self.headers.get_all('Content-Length', []) + has_unread_body = bool(self.headers.get('Transfer-Encoding')) + if lengths: + try: + has_unread_body |= len(lengths) != 1 or int(lengths[0]) != 0 + except ValueError: + has_unread_body = True + self._drain_rejection = has_unread_body and not getattr(self, '_request_body_read', False) body = json.dumps({'error': {'message': message, 'type': 'gateway_error'}}).encode() self.send_response(status) self.send_header('Content-Type', 'application/json') @@ -194,6 +332,13 @@ def error(self, status, message): if self.command != 'HEAD': self.wfile.write(body) + def finish(self): + try: + super().finish() + finally: + if getattr(self, '_drain_rejection', False): + self.server.rejection_drainer.submit(self.connection) + def do_GET(self): self.proxy() @@ -253,6 +398,7 @@ def proxy_request(self): if len(body) != length: self.error(400, 'Incomplete request body') return + self._request_body_read = True headers = {'Accept-Encoding': 'identity', 'Connection': 'close'} for name in ['Content-Type', 'Accept', 'User-Agent']: value = self.headers.get(name) @@ -262,6 +408,7 @@ def proxy_request(self): try: eligible, context_limit = self.server.context_route(self.headers) key = affinity_key(self.headers, body, hashlib.sha256(fields[1].encode()).hexdigest()) if balanced else None + self._observe('session', key) except ValueError: self.error(400, 'Invalid session identifier or context window') return @@ -346,6 +493,8 @@ def watch_disconnect(): self.close_connection = True return sent_headers = True + self._observe('response', response.status, response.getheader('Content-Type'), + response.getheader('Content-Encoding')) self.close_connection = True if model_body is not None: try: @@ -359,10 +508,12 @@ def watch_disconnect(): if not chunk: if response.length: raise IncompleteRead(b'', response.length) + self._observe('complete', not disconnected.is_set()) break try: self.wfile.write(chunk) self.wfile.flush() + self._observe('chunk', chunk) except OSError: self.close_connection = True return @@ -407,6 +558,10 @@ def main(): parser.add_argument('--upstream-timeout', type=float, default=10800) parser.add_argument('--model-context-limit', type=int) parser.add_argument('--backend-context-limit', action='append', default=[], metavar='PORT=TOKENS') + parser.add_argument('--backend-readiness-port', action='append', default=[], metavar='PORT=MANAGEMENT_PORT') + parser.add_argument('--statistics-database', type=Path) + parser.add_argument('--backend-pd-model', help='Require healthy Motor prefill and decode instances for this model') + parser.add_argument('--backend-pd-probe-endpoints', action='store_true', help='Probe every rank of each candidate PD instance') args = parser.parse_args() limits = {} for item in args.backend_context_limit: @@ -416,11 +571,45 @@ def main(): if port in limits: parser.error('Duplicate backend context limit') limits[port] = limit - server = Gateway((args.listen, args.port), args.key_store, args.upstream_port, args.backup_port, args.backend_state, args.bindings_file, args.upstream_timeout, args.model_context_limit, limits, - backend_ports=args.backend_port, max_active=args.max_active_per_backend, max_waiting=args.max_waiting, - queue_timeout=args.queue_timeout, control_file=args.control_file, backend_managed_affinity=args.backend_managed_affinity) - print(json.dumps({'state': 'LISTENING', 'listen': args.listen, 'port': args.port}), flush=True) - server.serve_forever() + readiness_ports = {} + for item in args.backend_readiness_port: + if not re.fullmatch(r'[0-9]{1,5}=[0-9]{1,5}', item): + parser.error('Backend readiness mappings must use PORT=MANAGEMENT_PORT') + port, management_port = map(int, item.split('=')) + if port in readiness_ports: + parser.error('Duplicate backend readiness mapping') + readiness_ports[port] = management_port + if args.statistics_database is not None: + try: + _check_statistics_paths(args.statistics_database, args.key_store, args.bindings_file, + args.backend_state, args.control_file) + except (OSError, RuntimeError, ValueError): + parser.error('Statistics database and gateway runtime paths must be distinct') + module = sys.modules[__name__] + original_handler = module.Handler + recorder = None + if args.statistics_database is not None: + try: + from deploy_stats.inprocess import install + recorder = install(module, args.statistics_database, source='gateway-inprocess-v1') + except Exception: + recorder = None + server = None + try: + server = Gateway((args.listen, args.port), args.key_store, args.upstream_port, args.backup_port, args.backend_state, args.bindings_file, args.upstream_timeout, args.model_context_limit, limits, + backend_ports=args.backend_port, max_active=args.max_active_per_backend, max_waiting=args.max_waiting, + queue_timeout=args.queue_timeout, control_file=args.control_file, backend_managed_affinity=args.backend_managed_affinity, + backend_readiness_ports=readiness_ports, pd_model=args.backend_pd_model, + pd_probe_endpoints=args.backend_pd_probe_endpoints) + print(json.dumps({'state': 'LISTENING', 'listen': args.listen, 'port': args.port, + 'statistics': 'enabled' if recorder else ('unavailable' if args.statistics_database else 'disabled')}), flush=True) + server.serve_forever() + finally: + if server is not None: + server.server_close() + if recorder is not None: + module.Handler = original_handler + recorder.close() if __name__ == '__main__': diff --git a/deploy_stats/__init__.py b/deploy_stats/__init__.py new file mode 100644 index 0000000..afc7d33 --- /dev/null +++ b/deploy_stats/__init__.py @@ -0,0 +1 @@ +"""Local, metadata-only accounting for an authenticated model gateway.""" diff --git a/deploy_stats/__main__.py b/deploy_stats/__main__.py new file mode 100644 index 0000000..0de0f39 --- /dev/null +++ b/deploy_stats/__main__.py @@ -0,0 +1,38 @@ +"""Run the accounting hop or the local read-only dashboard.""" +import argparse +from pathlib import Path + +from .proxy import Proxy +from .web import Dashboard + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + proxy = commands.add_parser('proxy') + proxy.add_argument('--port', type=int, default=19882) + proxy.add_argument('--upstream-port', type=int, required=True) + proxy.add_argument('--database', type=Path, required=True) + dashboard = commands.add_parser('dashboard') + dashboard.add_argument('--port', type=int, default=19881) + dashboard.add_argument('--database', type=Path, required=True) + dashboard.add_argument('--backend-port', type=int, action='append', default=[]) + dashboard.add_argument('--telemetry-port', type=int) + dashboard.add_argument('--motor-management-port', type=int, action='append', default=[]) + dashboard.add_argument('--motor-history-database', type=Path) + args = parser.parse_args() + server = (Proxy(args.port, args.upstream_port, args.database) if args.command == 'proxy' + else Dashboard(args.port, args.database, args.backend_port, args.telemetry_port, + motor_management_ports=args.motor_management_port, + motor_history_path=args.motor_history_database)) + print(f'Listening on http://127.0.0.1:{server.server_port}', flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + + +if __name__ == '__main__': + main() diff --git a/deploy_stats/backends.py b/deploy_stats/backends.py new file mode 100644 index 0000000..806f9b5 --- /dev/null +++ b/deploy_stats/backends.py @@ -0,0 +1,77 @@ +"""Read the bounded, public numeric subset of local backend metrics.""" +from http.client import HTTPConnection, HTTPException +import json +import math +import time + +METRICS = { + 'deepinfer:requests_running': 'running', + 'deepinfer:requests_waiting': 'waiting', + 'deepinfer:workers_alive': 'workers_alive', + 'deepinfer:admission_ready': 'admission_ready', + 'deepinfer:prompt_tokens_total': 'input_total', + 'deepinfer:prompt_cache_hit_tokens_total': 'cached_total', + 'deepinfer:prompt_cache_hit_requests_total': 'cache_hit_requests_total', +} + + +def read_backend(port): + result = {'port': port, 'sampled_at': time.time(), 'available': False} + connection = HTTPConnection('127.0.0.1', port, timeout=1) + try: + connection.request('GET', '/metrics') + response = connection.getresponse() + if response.status != 200: + return result + payload = response.read(1024 * 1024 + 1) + if len(payload) > 1024 * 1024: + return result + values = {} + for line in payload.decode('utf-8').splitlines(): + parts = line.split() + if len(parts) != 2 or parts[0] not in METRICS: + continue + name = METRICS[parts[0]] + value = float(parts[1]) + if name in values or not math.isfinite(value) or value < 0: + return result + values[name] = value + if set(values) != set(METRICS.values()) or values['admission_ready'] not in (0, 1): + return result + result.update(values, available=True) + result['hit_rate'] = (values['cached_total'] / values['input_total'] + if values['input_total'] and values['cached_total'] <= values['input_total'] else None) + return result + except (OSError, HTTPException, UnicodeError, ValueError, OverflowError): + return result + finally: + connection.close() + + +def read_telemetry(port): + unavailable = {'available': False, 'nodes': []} + if port is None: + return unavailable + connection = HTTPConnection('127.0.0.1', port, timeout=2) + try: + connection.request('GET', '/snapshot') + response = connection.getresponse() + if response.status != 200: + return unavailable + payload = response.read(1024 * 1024 + 1) + if len(payload) > 1024 * 1024: + return unavailable + value = json.loads(payload) + now = time.time() + stamp = value.get('sampled_at') if isinstance(value, dict) else None + if (not isinstance(value, dict) or value.get('schema') != 1 or value.get('kind') != 'deepinfer-telemetry' or + type(stamp) not in (int, float) or not math.isfinite(stamp) or not 0 <= now - stamp <= 620 or + not isinstance(value.get('nodes'), list) or len(value['nodes']) > 8): + return unavailable + value['available'] = True + json.dumps(value, allow_nan=False) + return value + except (OSError, HTTPException, UnicodeError, ValueError, OverflowError): + return unavailable + finally: + connection.close() diff --git a/deploy_stats/capture.py b/deploy_stats/capture.py new file mode 100644 index 0000000..54a63c3 --- /dev/null +++ b/deploy_stats/capture.py @@ -0,0 +1,380 @@ +"""Extract final usage without changing a JSON or SSE response.""" +import json + +MAX_FRAME = 4 * 1024 * 1024 +MAX_COUNT = 2**53 - 1 +MAX_DATA_LINES = 4096 +MAX_JSON_ESCAPES = 65536 + + +def _too_complex(payload, limit): + """Bound JSON structure and quote scanning without counting string contents.""" + if limit is None: + return False + if payload.count(92) > MAX_JSON_ESCAPES: + return True + if sum(payload.count(mark) for mark in b'{}[],:') <= limit: + return False + size, position, structural, strings = len(payload), 0, 0, 0 + while position < size: + opening = payload.find(b'"', position) + end = size if opening < 0 else opening + structural += sum(payload.count(mark, position, end) for mark in b'{}[],:') + if structural > limit: + return True + if opening < 0: + break + strings += 1 + if strings > limit: + return True + position = opening + 1 + while position < size: + closing = payload.find(b'"', position) + if closing < 0: + return False # The JSON parser will reject the unterminated string. + index = closing - 1 + while index >= position and payload[index] == 92: + index -= 1 + position = closing + 1 + if (closing - index - 1) % 2 == 0: + break + return False + + +def _count(value): + return value if type(value) is int and 0 <= value <= MAX_COUNT else None + + +def _may_have_non_null(payload, field): + name = b'"' + field + b'"' + offset = 0 + for _ in range(1024): + marker = payload.find(name, offset) + if marker < 0: + return False + position = marker + len(name) + while position < len(payload) and payload[position] in b' \t\r\n': + position += 1 + if position - marker > 128: + return True + if position >= len(payload) or payload[position] != 58: + return True + position += 1 + while position < len(payload) and payload[position] in b' \t\r\n': + position += 1 + if position - marker > 128: + return True + if payload[position:position + 4] != b'null': + return True + offset = position + 4 + if offset < len(payload) and payload[offset] not in b' \t\r\n,}]': + return True + return True + + +def _may_have_final_usage(protocol, payload, chat_finished): + if protocol == 'chat' and not chat_finished and not _may_have_non_null(payload, b'finish_reason'): + return False + if protocol == 'responses' and b'response.completed' not in payload and b'response.incomplete' not in payload: + return False + if protocol == 'messages' and b'message_delta' not in payload: + return False + return _may_have_non_null(payload, b'usage') + + +def _unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('Duplicate JSON key') + result[key] = value + return result + + +def extract_usage(protocol, value, *, allow_partial=False): + """Return inclusive input, output and cached counts, or None if unknown.""" + if not isinstance(value, dict): + return None + if protocol == 'responses' and isinstance(value.get('response'), dict): + value = value['response'] + usage = value.get('usage') + if not isinstance(usage, dict): + return None + if protocol == 'messages': + uncached = _count(usage.get('input_tokens')) + cached = _count(usage.get('cache_read_input_tokens')) + created = _count(usage.get('cache_creation_input_tokens', 0)) + output = _count(usage.get('output_tokens')) + if None in (uncached, cached, created, output): + return None + prompt = uncached + cached + created + elif protocol == 'responses': + prompt = _count(usage.get('input_tokens')) + output = _count(usage.get('output_tokens')) + details = usage.get('input_tokens_details') + if details is not None and not isinstance(details, dict): + return None + cached = _count(details.get('cached_tokens')) if isinstance(details, dict) else None + if isinstance(details, dict) and details.get('cached_tokens') is not None and cached is None: + return None + else: + prompt = _count(usage.get('prompt_tokens')) + output = _count(usage.get('completion_tokens')) + details = usage.get('prompt_tokens_details') + if details is not None and not isinstance(details, dict): + return None + cached = _count(details.get('cached_tokens')) if isinstance(details, dict) else None + alternate = _count(usage.get('prompt_cache_hit_tokens')) + if ((isinstance(details, dict) and details.get('cached_tokens') is not None and cached is None) or + (usage.get('prompt_cache_hit_tokens') is not None and alternate is None)): + return None + if cached is None: + cached = alternate + elif alternate is not None and alternate != cached: + return None + miss = usage.get('prompt_cache_miss_tokens') + if miss is not None and (_count(miss) is None or prompt is None or cached is None or miss != prompt - cached): + return None + if prompt is None or output is None or prompt > MAX_COUNT: + return None + if cached is None and (not allow_partial or protocol not in ('chat', 'responses')): + return None + if cached is not None and cached > prompt: + return None + total = usage.get('total_tokens') + if total is not None and _count(total) != prompt + output: + return None + return prompt, output, cached + + +def has_visible_delta(protocol, value): + """Recognize a nonempty output delta, not a role or lifecycle event.""" + if not isinstance(value, dict): + return False + if protocol == 'chat': + choices = value.get('choices') + for choice in choices if isinstance(choices, list) else (): + delta = choice.get('delta') if isinstance(choice, dict) else None + if not isinstance(delta, dict): + continue + if any(isinstance(delta.get(k), str) and delta[k] for k in ('content', 'reasoning', 'reasoning_content')): + return True + for call in delta.get('tool_calls', ()) if isinstance(delta.get('tool_calls'), list) else (): + fn = call.get('function') if isinstance(call, dict) else None + if isinstance(fn, dict) and any(isinstance(fn.get(k), str) and fn[k] for k in ('name', 'arguments')): + return True + return False + event_type = value.get('type') + if protocol == 'responses': + return (isinstance(event_type, str) and event_type.startswith('response.') and + event_type.endswith('.delta') and isinstance(value.get('delta'), str) and bool(value['delta'])) + if protocol == 'messages' and event_type == 'content_block_delta': + delta = value.get('delta') + return isinstance(delta, dict) and any(isinstance(delta.get(k), str) and delta[k] + for k in ('text', 'thinking', 'partial_json')) + return False + + +class UsageCapture: + """Bound memory and retain only numerical usage after each response frame.""" + + def __init__(self, protocol, content_type, content_encoding='', on_first_output=None, *, + max_frame=MAX_FRAME, max_structure=None, max_delta=None, allow_partial=False): + self.protocol = protocol + self.allow_partial = allow_partial + self.max_frame = max_frame + self.max_structure = max_structure + self.max_delta = max_delta + media_type = (content_type or '').split(';', 1)[0].strip().lower() + self.kind = ('sse' if media_type == 'text/event-stream' else + 'json' if media_type == 'application/json' else 'other') + if (content_encoding or '').strip().lower() not in ('', 'identity'): + self.kind = 'other' + self.buffer = bytearray() + self.line_scan_start = 0 + self.event_data = bytearray() + self.event_bytes = 0 + self.event_lines = 0 + self.discard_line = False + self.discard_event = False + self.too_large = False + self.incomplete = False + self.usage = None + self.terminal = False + self.on_first_output = on_first_output + self.saw_output = False + self.first_unavailable = False + self.skipped = False + self.chat_finished = False + + def feed(self, data): + if self.kind == 'other': + return + if self.kind == 'json': + if not self.too_large and len(self.buffer) + len(data) <= self.max_frame: + self.buffer.extend(data) + else: + self.buffer.clear() + self.too_large = True + return + if self.discard_line: + end = data.find(b'\n') + if end < 0: + return + data = data[end + 1:] + self.discard_line = False + self.buffer.extend(data) + while True: + end = self.buffer.find(b'\n', self.line_scan_start) + if end < 0: + if len(self.buffer) > self.max_frame: + self.buffer.clear() + self.line_scan_start = 0 + self.event_data.clear() + self.event_bytes = 0 + self.event_lines = 0 + self.discard_line = True + self.discard_event = True + self.incomplete = True + else: + self.line_scan_start = len(self.buffer) + break + line = bytes(self.buffer[:end]).removesuffix(b'\r') + del self.buffer[:end + 1] + self.line_scan_start = 0 + if self.discard_line: + self.discard_line = False + continue + if not line: + self._event() + elif line.startswith(b'data:') and not self.discard_event: + item = line[5:] + if item.startswith(b' '): + item = item[1:] + self.event_lines += 1 + self.event_bytes += len(item) + (self.event_lines > 1) + if self.event_bytes > self.max_frame or self.event_lines > MAX_DATA_LINES: + self.discard_event = True + self.incomplete = True + self.event_data.clear() + self.event_bytes = 0 + self.event_lines = 0 + else: + if self.event_lines > 1: + self.event_data.append(10) + self.event_data.extend(item) + + def _event(self): + if not self.discard_event and self.event_lines: + payload = self.event_data + if payload == b'[DONE]': + if self.protocol == 'chat': + self.terminal = True + elif not self.terminal: + self.incomplete = True + elif self.terminal: + self.incomplete = True + elif (self.max_delta is not None and len(payload) > self.max_delta and + not _may_have_final_usage(self.protocol, payload, self.chat_finished)): + # A later valid final usage can still account for the request, + # but skipped output cannot establish the first output time. + self.usage = None + self.skipped = True + self.chat_finished = False + if not self.saw_output: + self.first_unavailable = True + else: + if (self.protocol == 'chat' and self.max_delta is not None and len(payload) > self.max_delta + and not self.chat_finished): + self.usage = None + self.skipped = True + value = self._decode(payload) + if not isinstance(value, dict): + self.incomplete = True + else: + event_type = value.get('type') + # A prior usage snapshot is no longer final when generation + # continues. A later valid usage event can restore it. + if self.protocol == 'chat': + choices = value.get('choices') + self.usage = None + elif self.protocol == 'messages' and event_type != 'message_stop': + self.usage = None + elif self.protocol == 'messages' and value.get('usage') is not None: + self.incomplete = True + if self.protocol == 'chat': + if isinstance(choices, list) and any( + isinstance(choice, dict) and isinstance(choice.get('finish_reason'), str) + and choice['finish_reason'] for choice in choices): + self.chat_finished = True + if (not self.saw_output and not self.first_unavailable and not self.incomplete and + has_visible_delta(self.protocol, value)): + self.saw_output = True + if self.on_first_output: + self.on_first_output() + final = (self.protocol == 'chat' or + self.protocol == 'messages' and event_type == 'message_delta' or + self.protocol == 'responses' and event_type in + ('response.completed', 'response.incomplete')) + if final: + usage = extract_usage(self.protocol, value, allow_partial=self.allow_partial) + if usage is not None and (not self.skipped or self.protocol != 'chat' or self.chat_finished): + self.usage = usage + self.skipped = False + elif usage is None: + source = value.get('response') if self.protocol == 'responses' else value + if isinstance(source, dict) and source.get('usage') is not None: + self.incomplete = True + if (self.protocol == 'messages' and event_type == 'message_stop' or + self.protocol == 'responses' and event_type in + ('response.completed', 'response.incomplete')): + self.terminal = True + self.event_data.clear() + self.event_bytes = 0 + self.event_lines = 0 + self.discard_event = False + + def _decode(self, payload): + if _too_complex(payload, self.max_structure): + self.incomplete = True + return None + try: + return json.loads(payload, object_pairs_hook=_unique_pairs) + except (ValueError, UnicodeDecodeError): + self.incomplete = True + return None + + def finish(self): + if self.kind == 'json' and not self.too_large: + return extract_usage(self.protocol, self._decode(self.buffer), allow_partial=self.allow_partial) + if (self.kind == 'sse' and self.terminal and not self.incomplete and not self.buffer and + not self.event_lines and not self.discard_line and not self.discard_event): + return self.usage + return None + + +class SafeCapture: + """A statistics parser error must never interrupt the user's response.""" + + def __init__(self, protocol, content_type, content_encoding='', on_first_output=None, *, + max_frame=MAX_FRAME, max_structure=None, max_delta=None, allow_partial=False): + self.reader = UsageCapture(protocol, content_type, content_encoding, on_first_output, + max_frame=max_frame, max_structure=max_structure, max_delta=max_delta, + allow_partial=allow_partial) + self.failed = False + + def feed(self, data): + if not self.failed: + try: + self.reader.feed(data) + except Exception: + self.failed = True + + def finish(self): + if self.failed: + return None + try: + return self.reader.finish() + except Exception: + self.failed = True + return None diff --git a/deploy_stats/inprocess.py b/deploy_stats/inprocess.py new file mode 100644 index 0000000..662e1a2 --- /dev/null +++ b/deploy_stats/inprocess.py @@ -0,0 +1,209 @@ +"""Observe gateway responses behind Caddy's verified default forwarding policy.""" +import time +import threading +import uuid +from urllib.parse import urlsplit + +from .capture import SafeCapture +from .proxy import PROTOCOLS, client_ip +from .session import BodyBudget +from .store import Recorder + + +CAPTURE_BUDGET = 32 * 1024 * 1024 +MAX_CAPTURE_FRAME = 4 * 1024 * 1024 +MAX_CAPTURE_STRUCTURE = 16384 +MAX_CAPTURE_DELTA = 64 * 1024 + + +class _Observer: + def __init__(self, recorder, budget, headers, protocol): + self.recorder = recorder + self.budget = budget + self.protocol = protocol + self.started = time.perf_counter() + self.request_id = uuid.uuid4().hex + # Caddy must be verified to overwrite untrusted X-Forwarded-For. + # Direct loopback callers do not have that provenance guarantee. + self.source = client_ip(headers, header='X-Forwarded-For') + self.session_id = None + self.first = None + self.status = None + self.completed = False + self.capture = None + self.reserved = 0 + self.failed = False + self.recorder.begin(self.request_id, time.time(), self.source, None) + + def fail(self): + if not self.failed: + self.failed = True + if self.reserved: + self.budget.release(self.reserved) + self.reserved = 0 + self.capture = None + try: + self.recorder.note_drop() + except Exception: + pass + + def session(self, value): + if not self.failed and value is not None: + self.session_id = value + self.recorder.set_session(self.request_id, value) + + def error(self, status): + self.status = status + + def response(self, status, content_type, content_encoding): + self.status = status + if not self.failed and status < 400 and self.protocol: + def first_output(): + if self.first is None: + self.first = max(0, time.perf_counter() - self.started) + self.recorder.first_output(self.request_id, self.first) + self.capture = SafeCapture(self.protocol, content_type, content_encoding, first_output, + max_frame=MAX_CAPTURE_FRAME, max_structure=MAX_CAPTURE_STRUCTURE, + max_delta=MAX_CAPTURE_DELTA, + allow_partial=getattr(self.recorder, 'source', None) == 'gateway-inprocess-v1') + + def chunk(self, data): + if self.failed or self.capture is None: + return + if not self.budget.reserve(len(data)): + self.fail() + return + self.reserved += len(data) + self.capture.feed(data) + if self.capture.failed: + self.fail() + return + retained = len(self.capture.reader.buffer) + self.capture.reader.event_bytes + released = self.reserved - retained + if released > 0: + self.budget.release(released) + self.reserved -= released + + def complete(self, clean): + self.completed = clean + + def finish(self): + state, values = 'interrupted', None + try: + if self.status is not None and self.status >= 400: + state = 'failed' + elif self.completed: + if not self.failed and self.capture is not None: + values = self.capture.finish() + state = ('partial' if values is not None and values[2] is None else + 'known' if values is not None else 'missing') + except Exception: + self.fail() + state, values = 'missing', None + finally: + if self.reserved: + self.budget.release(self.reserved) + self.reserved = 0 + self.capture = None + prompt, output, cached = values if state in ('known', 'partial') else (None, None, None) + try: + try: + self.recorder.submit((time.time(), self.source, max(0, time.perf_counter() - self.started), + self.status, state, prompt, output, cached, self.session_id, self.first), + self.request_id) + finally: + self.recorder.end(self.request_id) + except Exception: + try: + self.recorder.note_drop() + except Exception: + pass + + +def make_handler(base_handler, recorder, *, budget=None): + """Wrap one gateway handler; recorder failure never replaces its response.""" + budget = budget or BodyBudget(CAPTURE_BUDGET) + + class InstrumentedHandler(base_handler): + _llm_stats_wrapped = True + + def proxy(self): + try: + path = urlsplit(self.path).path + except ValueError: + return super().proxy() + if self.command != 'POST' or not path.startswith('/v1/'): + return super().proxy() + observer = None + try: + observer = _Observer(recorder, budget, self.headers, PROTOCOLS.get(path)) + except Exception: + try: + recorder.note_drop() + except Exception: + pass + self._stats_observer = observer + try: + return super().proxy() + finally: + if observer is not None: + try: + observer.finish() + except Exception: + observer.fail() + self._stats_observer = None + + def error(self, status, message): + self._observe('error', status) + return super().error(status, message) + + return InstrumentedHandler + + +def install(module, database, *, source=None): + """Enable capture on this gateway module; unavailable storage fails open.""" + if (getattr(module.Handler, '_llm_stats_wrapped', False) or + not callable(getattr(module.Handler, '_observe', None))): + return None + try: + recorder = Recorder(database, source=source) + except Exception: + return None + try: + module.Handler = make_handler(module.Handler, recorder) + except Exception: + recorder.close() + return None + return recorder + + +def serve(module, database, make_servers, *args, **kwargs): + """Run an existing two-server bridge with fail-open, in-process capture.""" + original_handler = module.Handler + recorder = install(module, database) + installed_handler = module.Handler if recorder is not None else None + inner = outer = None + inner_started = False + try: + inner, outer = make_servers(*args, **kwargs) + worker = threading.Thread(target=inner.serve_forever, daemon=True) + worker.start() + inner_started = True + outer.serve_forever() + finally: + try: + if outer is not None: + outer.server_close() + finally: + try: + if inner is not None: + try: + if inner_started: + inner.shutdown() + finally: + inner.server_close() + finally: + if recorder is not None: + if module.Handler is installed_handler: + module.Handler = original_handler + recorder.close() diff --git a/deploy_stats/motor_metrics.py b/deploy_stats/motor_metrics.py new file mode 100644 index 0000000..3f650ea --- /dev/null +++ b/deploy_stats/motor_metrics.py @@ -0,0 +1,669 @@ +"""Read Motor's registered vLLM counters without changing the inference path.""" +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +from copy import deepcopy +import hashlib +from http.client import HTTPConnection, HTTPException +import ipaddress +import json +import math +import os +from pathlib import Path +import re +import sqlite3 +import threading +import time + + +MAX_PAYLOAD = 1024 * 1024 +MAX_INSTANCES = 32 +MAX_ENDPOINTS = 256 +BUCKET_SECONDS = 600 +BUCKET_COUNT = 144 +HISTORY_METHOD = 'vllm-counter-rate-v1' +APPLICATION_ID = 0x4C444D48 +SCHEMA_VERSION = 1 +MTP_COUNTERS = ('vllm:spec_decode_num_drafts_total', 'vllm:spec_decode_num_draft_tokens_total', + 'vllm:spec_decode_num_accepted_tokens_total') +METRICS = frozenset({ + 'process_start_time_seconds', 'vllm:prompt_tokens_by_source_total', + 'vllm:generation_tokens_total', 'vllm:request_prefill_time_seconds_sum', + 'vllm:request_prefill_time_seconds_count', 'vllm:request_decode_time_seconds_sum', + 'vllm:request_decode_time_seconds_count', 'vllm:num_requests_running', + 'vllm:num_requests_waiting', 'vllm:prompt_tokens_total', + 'vllm:prefix_cache_hits_total', 'vllm:prefix_cache_queries_total', +}) | frozenset(MTP_COUNTERS) +SAMPLE = re.compile(r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{(.*)\})?\s+(\S+)(?:\s+\d+)?\s*$') +LABEL = re.compile(r'\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*("(?:[^"\\]|\\["\\n])*")\s*(,|$)') + + +def parse_metrics(payload): + """Parse only the bounded counter subset; duplicate series are ambiguous.""" + if not isinstance(payload, str) or len(payload.encode('utf-8')) > MAX_PAYLOAD: + raise ValueError('INVALID_METRICS') + result = {} + invalid_mtp = False + for line in payload.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + name = line.split('{', 1)[0].split(None, 1)[0] + if name not in METRICS: + continue + try: + sample = SAMPLE.fullmatch(line) + if not sample: + raise ValueError('INVALID_SAMPLE') + labels = {} + raw = sample.group(2) + if raw is not None: + offset = 0 + while offset < len(raw): + match = LABEL.match(raw, offset) + if not match or match.group(1) in labels: + raise ValueError('INVALID_LABELS') + labels[match.group(1)] = json.loads(match.group(2)) + offset = match.end() + if match.group(3) == '' and offset != len(raw): + raise ValueError('INVALID_LABELS') + if match.group(3) == ',' and offset == len(raw): + raise ValueError('INVALID_LABELS') + value = float(sample.group(3)) + if not math.isfinite(value) or value < 0: + raise ValueError('INVALID_VALUE') + if name.endswith('_count') and not value.is_integer(): + raise ValueError('INVALID_COUNT') + if name in MTP_COUNTERS and (not value.is_integer() or value > 2**53 - 1): + raise ValueError('INVALID_MTP_COUNTER') + key = tuple(sorted(labels.items())) + samples = result.setdefault(name, {}) + if key in samples: + raise ValueError('DUPLICATE_SAMPLE') + samples[key] = value + except ValueError: + if name not in MTP_COUNTERS: + raise + invalid_mtp = True + if invalid_mtp: + for name in MTP_COUNTERS: + result.pop(name, None) + starts = result.get('process_start_time_seconds', {}) + if set(starts) != {()} or starts[()] <= 0: + raise ValueError('MISSING_PROCESS_IDENTITY') + return result + + +def _get(host, port, path): + connection = HTTPConnection(host, port, timeout=2) + try: + connection.request('GET', path, headers={'Accept-Encoding': 'identity', 'Connection': 'close'}) + response = connection.getresponse() + if response.status != 200: + raise ValueError('SOURCE_UNAVAILABLE') + body = response.read(MAX_PAYLOAD + 1) + if len(body) > MAX_PAYLOAD: + raise ValueError('OVERSIZED_SOURCE') + return body.decode('utf-8') + finally: + connection.close() + + +def _fingerprint(value): + return hashlib.sha256(value.encode('utf-8')).hexdigest() + + +def _topology(payload): + value = json.loads(payload) + instances = value.get('instances') if isinstance(value, dict) else None + if not isinstance(instances, list) or len(instances) > MAX_INSTANCES: + raise ValueError('INVALID_TOPOLOGY') + result = {} + count = 0 + for entry in instances: + if not isinstance(entry, dict): + raise ValueError('INVALID_INSTANCE') + identity, role = entry.get('id'), entry.get('role') + if type(identity) is int and 0 <= identity < 2**63: + identity = str(identity) + if not isinstance(identity, str) or not identity or len(identity) > 512 or identity in result: + raise ValueError('INVALID_IDENTITY') + if role not in ('prefill', 'decode'): + continue + endpoints = entry.get('endpoints') + if not isinstance(endpoints, list): + raise ValueError('INVALID_ENDPOINTS') + targets = [] + for endpoint in endpoints: + if not isinstance(endpoint, dict): + raise ValueError('INVALID_ENDPOINT') + address = str(ipaddress.ip_address(endpoint.get('ip'))) + port = endpoint.get('business_port') + if type(port) is int: + parsed_port = port + elif isinstance(port, str) and port.isascii() and port.isdigit() and len(port) <= 5: + parsed_port = int(port) + else: + raise ValueError('INVALID_PORT') + if not 1 <= parsed_port <= 65535: + raise ValueError('INVALID_PORT') + targets.append((address, parsed_port)) + count += len(targets) + if count > MAX_ENDPOINTS: + raise ValueError('TOO_MANY_ENDPOINTS') + model = entry.get('model_name') + if model is not None and (not isinstance(model, str) or not model or len(model) > 512): + raise ValueError('INVALID_MODEL') + result[identity] = {'role': role, 'targets': tuple(sorted(targets)), 'model_name': model, + 'ready': entry.get('status') == 'active' and entry.get('healthy') is True} + return result + + +def _selected(values, name, source=None): + series = values.get(name, {}) + if source is not None: + series = {labels: value for labels, value in series.items() if dict(labels).get('source') == source} + return series + + +def _delta(current, previous, name, source=None): + after, before = _selected(current, name, source), _selected(previous, name, source) + if not after or set(after) != set(before): + return None + diffs = [after[key] - before[key] for key in after] + if any(value < 0 or not math.isfinite(value) for value in diffs): + return None + value = sum(diffs) + return value if math.isfinite(value) else None + + +def _sum_metric(values, name): + series = values.get(name, {}) + if not series: + return None + result = sum(series.values()) + return result if math.isfinite(result) else None + + +def _sum_values(values): + if not values: + return None + result = sum(values) + return result if math.isfinite(result) else None + + +def _mtp_counts(values, model): + """Use only consistent engine/model series; return counts in counter order.""" + if not isinstance(model, str) or not model: + return None + series = [values.get(name, {}) for name in MTP_COUNTERS] + if not series[0] or any(set(item) != set(series[0]) for item in series[1:]): + return None + if any(dict(labels).get('model_name') != model for labels in series[0]): + return None + if any(series[2][labels] > series[1][labels] for labels in series[0]): + return None + result = tuple(sum(int(value) for value in item.values()) for item in series) + return result if all(value <= 2**53 - 1 for value in result) else None + + +def _mtp_change(current, previous, model): + if _mtp_counts(current, model) is None or _mtp_counts(previous, model) is None: + return None + series = [current[name] for name in MTP_COUNTERS] + old = [previous[name] for name in MTP_COUNTERS] + if any(set(new) != set(before) for new, before in zip(series, old)): + return None + changes = [{key: new[key] - before[key] for key in new} for new, before in zip(series, old)] + if any(value < 0 or not math.isfinite(value) for item in changes for value in item.values()): + return None + if any(changes[2][labels] > changes[1][labels] for labels in changes[0]): + return None + result = tuple(sum(int(value) for value in item.values()) for item in changes) + return result if all(value <= 2**53 - 1 for value in result) else None + + +def _mtp_summary(items, expected, *, partial=False): + if not items: + return None + steps, drafted, accepted = (sum(row[index] for row in items) for index in range(3)) + if any(value > 2**53 - 1 for value in (steps, drafted, accepted)) or accepted > drafted: + return None + return {'steps': steps, 'drafted': drafted, 'accepted': accepted, + 'acceptance_rate': accepted / drafted if drafted else None, + 'mean_acceptance_length': 1 + accepted / steps if steps else None, + 'valid_endpoints': len(items), 'expected_endpoints': expected, + 'partial': partial or len(items) < expected} + + +def _agreed(topologies): + available = [top for top in topologies if top is not None] + groups, ambiguous = {}, False + if available: + identities = set().union(*(set(top) for top in available)) + for identity in identities: + entries = [top.get(identity) for top in available] + if any(entry is None for entry in entries): + ambiguous = True + continue + shape = {(entry['role'], entry['targets'], entry['model_name']) for entry in entries} + if len(shape) != 1: + ambiguous = True + continue + groups[identity] = {**entries[0], 'ready': all(entry['ready'] for entry in entries)} + return groups, ambiguous + + +class MotorHistory: + """Persist sampled rates in ten-minute buckets in a dedicated identified file.""" + def __init__(self, path): + self.path = Path(path).resolve() + try: + fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) + except FileExistsError: + created = False + else: + os.close(fd) + created = True + with closing(self._connect()) as connection, connection: + if created: + connection.execute(f'PRAGMA application_id = {APPLICATION_ID}') + connection.execute(f'PRAGMA user_version = {SCHEMA_VERSION}') + connection.execute('''CREATE TABLE rate_buckets ( + group_key TEXT NOT NULL, phase TEXT NOT NULL, bucket INTEGER NOT NULL, + rate_sum REAL NOT NULL, samples INTEGER NOT NULL, partial_samples INTEGER NOT NULL, + PRIMARY KEY (group_key, phase, bucket)) WITHOUT ROWID''') + elif (connection.execute('PRAGMA application_id').fetchone()[0] != APPLICATION_ID or + connection.execute('PRAGMA user_version').fetchone()[0] != SCHEMA_VERSION): + raise ValueError('INVALID_MOTOR_HISTORY') + # Additive table: older readers continue to use rate_buckets unchanged. + connection.execute('''CREATE TABLE IF NOT EXISTS mtp_buckets ( + group_key TEXT NOT NULL, bucket INTEGER NOT NULL, + steps INTEGER NOT NULL, drafted INTEGER NOT NULL, accepted INTEGER NOT NULL, + samples INTEGER NOT NULL, partial_samples INTEGER NOT NULL, + PRIMARY KEY (group_key, bucket)) WITHOUT ROWID''') + columns = [(row[1], row[2].upper(), row[5]) for row in connection.execute('PRAGMA table_info(mtp_buckets)')] + if columns != [('group_key', 'TEXT', 1), ('bucket', 'INTEGER', 2), ('steps', 'INTEGER', 0), + ('drafted', 'INTEGER', 0), ('accepted', 'INTEGER', 0), ('samples', 'INTEGER', 0), + ('partial_samples', 'INTEGER', 0)]: + raise ValueError('INVALID_MTP_HISTORY') + connection.execute('PRAGMA journal_mode = WAL') + + def _connect(self): + return sqlite3.connect(self.path.as_uri() + '?mode=rw', uri=True, timeout=1) + + def record(self, group_key, phase, at, rate, *, partial=False): + if (not isinstance(group_key, str) or not group_key or phase not in ('prefill', 'decode') or + not isinstance(at, (int, float)) or not math.isfinite(at) or at < 0 or + not isinstance(rate, (int, float)) or not math.isfinite(rate) or rate < 0 or + type(partial) is not bool): + raise ValueError('INVALID_RATE_SAMPLE') + bucket = int(at // BUCKET_SECONDS) * BUCKET_SECONDS + with closing(self._connect()) as connection, connection: + connection.execute('''INSERT INTO rate_buckets VALUES (?, ?, ?, ?, 1, ?) + ON CONFLICT(group_key, phase, bucket) DO UPDATE SET + rate_sum = rate_sum + excluded.rate_sum, samples = samples + 1, + partial_samples = partial_samples + excluded.partial_samples''', + (group_key, phase, bucket, rate, int(partial))) + + def series(self, group_key, now): + end = int(now // BUCKET_SECONDS) * BUCKET_SECONDS + start = end - BUCKET_COUNT * BUCKET_SECONDS + if start < 0: + return None + with closing(self._connect()) as connection: + rows = connection.execute('''SELECT bucket, phase, rate_sum, samples, partial_samples FROM rate_buckets + WHERE group_key = ? AND bucket >= ? AND bucket < ?''', (group_key, start, end)).fetchall() + grouped = {(bucket, phase): (total, samples, partial) for bucket, phase, total, samples, partial in rows} + points = [] + for bucket in range(start, end, BUCKET_SECONDS): + point = {'start': bucket, 'end': bucket + BUCKET_SECONDS} + for phase in ('prefill', 'decode'): + total, samples, partial = grouped.get((bucket, phase), (0, 0, 0)) + point[phase + '_tps'] = total / samples if samples else None + point[phase + '_samples'] = samples + point[phase + '_partial'] = partial > 0 + points.append(point) + return {'method': HISTORY_METHOD, 'bucket_seconds': BUCKET_SECONDS, 'points': points} + + def record_mtp(self, group_key, at, steps, drafted, accepted, *, partial=False): + if (not isinstance(group_key, str) or not group_key or type(at) not in (int, float) or + not math.isfinite(at) or at < 0 or any(type(v) is not int or not 0 <= v <= 2**53 - 1 + for v in (steps, drafted, accepted)) or + accepted > drafted or type(partial) is not bool): + raise ValueError('INVALID_MTP_SAMPLE') + bucket = int(at // BUCKET_SECONDS) * BUCKET_SECONDS + with closing(self._connect()) as connection, connection: + connection.execute('''INSERT INTO mtp_buckets VALUES (?, ?, ?, ?, ?, 1, ?) + ON CONFLICT(group_key, bucket) DO UPDATE SET + steps=steps+excluded.steps, drafted=drafted+excluded.drafted, + accepted=accepted+excluded.accepted, samples=samples+1, + partial_samples=partial_samples+excluded.partial_samples''', + (group_key, bucket, steps, drafted, accepted, int(partial))) + + def mtp_series(self, group_key, now): + end = int(now // BUCKET_SECONDS) * BUCKET_SECONDS + start = end - BUCKET_COUNT * BUCKET_SECONDS + if start < 0: + return None + with closing(self._connect()) as connection: + rows = connection.execute('''SELECT bucket, steps, drafted, accepted, samples, partial_samples + FROM mtp_buckets WHERE group_key=? AND bucket>=? AND bucket 0 else None, + # A sampled delta cannot establish coverage of the whole bucket. + 'samples': samples, 'partial': samples > 0, + 'time_coverage': 'unverified' if samples else None, + 'partial_workers': partial > 0}) + return {'method': 'vllm-mtp-counter-delta-v1', 'bucket_seconds': BUCKET_SECONDS, 'points': points} + + +class MotorCollector: + """Bounded passive scrapes; only same-process counter deltas produce rates.""" + def __init__(self, management_ports, *, fetch_topology=None, fetch_endpoint=None, + wall_time=time.time, monotonic=time.monotonic, interval=30, history=None): + ports = tuple(dict.fromkeys(management_ports)) + if (not ports or len(ports) > 8 or any(type(port) is not int or not 1 <= port <= 65535 for port in ports) + or not 5 <= interval <= 300): + raise ValueError('INVALID_MOTOR_CONFIG') + self.ports = ports + self.fetch_topology = fetch_topology or (lambda port: _get('127.0.0.1', port, '/instances')) + self.fetch_endpoint = fetch_endpoint or (lambda host, port: _get(host, port, '/metrics')) + self.wall_time, self.monotonic, self.interval = wall_time, monotonic, interval + self.history = history + self.previous = {} + self.sequence = 0 + self.lock = threading.Lock() + self.collect_lock = threading.Lock() + self.stop_event = threading.Event() + self.value = {'schema': 1, 'kind': 'vllm-motor-telemetry', 'sampled_at': 0, + 'nodes': [], 'history_available': False} + self.worker = None + + def _read_topology(self, port): + try: + return _topology(self.fetch_topology(port)) + except Exception: + return None + + def _read_endpoint(self, target): + started = self.monotonic() + try: + values = parse_metrics(self.fetch_endpoint(*target)) + ended = self.monotonic() + return values, ended, self.wall_time(), max(0, ended - started) + except Exception: + return None + + def collect(self): + with self.collect_lock: + return self._collect() + + def _collect(self): + with ThreadPoolExecutor(max_workers=min(4, len(self.ports))) as executor: + before = list(executor.map(self._read_topology, self.ports)) + before_groups, ambiguous = _agreed(before) + before_addresses = Counter(target for group in before_groups.values() for target in group['targets']) + targets = [target for target, count in before_addresses.items() if count == 1] + with ThreadPoolExecutor(max_workers=4) as executor: + readings = dict(zip(targets, executor.map(self._read_endpoint, targets))) + with ThreadPoolExecutor(max_workers=min(4, len(self.ports))) as executor: + after = list(executor.map(self._read_topology, self.ports)) + after_groups, after_ambiguous = _agreed(after) + common_sources = sum(left is not None and right is not None for left, right in zip(before, after)) + ambiguous = ambiguous or after_ambiguous or not common_sources + changed = {identity for identity in set(before_groups) | set(after_groups) + if before_groups.get(identity) != after_groups.get(identity)} + if changed: + ambiguous = True + groups = {identity: value for identity, value in before_groups.items() + if common_sources and identity not in changed} + after_addresses = Counter(target for group in after_groups.values() for target in group['targets']) + ambiguous_targets = {target for target in set(before_addresses) | set(after_addresses) + if before_addresses[target] > 1 or after_addresses[target] > 1} + for topology in before + after: + if topology is not None: + counts = Counter(target for group in topology.values() for target in group['targets']) + ambiguous_targets.update(target for target, count in counts.items() if count > 1) + for identity in changed: + for group in (before_groups.get(identity), after_groups.get(identity)): + if group is not None: + ambiguous_targets.update(group['targets']) + if ambiguous_targets: + ambiguous = True + self.sequence += 1 + now = self.wall_time() + now_monotonic = self.monotonic() + new_previous, nodes = {}, [] + history_ok = self.history is not None + for identity, group in sorted(groups.items()): + role = group['role'] + # Previous histories did not identify a model and cannot safely be attributed. + group_key = _fingerprint(json.dumps(['motor-v2', identity, role, group['targets'], group['model_name']], + separators=(',', ':'))) + # 48 bits stay exactly representable as an integer in the browser. + public_id = int(group_key[:12], 16) + node = {'label': ('Prefill ' if role == 'prefill' else 'Decode ') + group_key[:8], + 'backend_port': public_id, 'role': role, 'received_at': now, + 'source_ok': False, 'backend_ready': group['ready'], 'clock_ok': True, + 'clock_uncertainty_seconds': 0, 'driver_ok': False, + 'expected_endpoints': len(group['targets']), + 'prefill': None, 'decode': None, 'hbm': None, 'phase_timers': None, 'history': None, + 'mtp': None, 'mtp_history': None} + rates, window_starts, window_ends, deadlines, observed_times, count_delta, seconds_delta = [], [], [], [], [], 0, 0.0 + mtp_lifetime, mtp_interval, mtp_times, mtp_window = [], [], [], [] + mtp_key = _fingerprint(json.dumps(['mtp-v1', identity, role, group['targets'], group['model_name']], + separators=(',', ':'))) + metrics = {'running': [], 'waiting': [], 'input_total': [], 'hit_pairs': []} + observed_endpoints = 0 + valid_timer_workers = 0 + for target in group['targets']: + if target in ambiguous_targets or not group['ready']: + continue + reading = readings.get(target) + if reading is None: + continue + values, stamp, sampled_at, uncertainty = reading + # Motor's registered model must match every series used in the display. + model = group['model_name'] + values = {name: ({labels: value for labels, value in series.items() + if model is not None and dict(labels).get('model_name') == model} + if name.startswith('vllm:') else series) + for name, series in values.items()} + age = now_monotonic - stamp + earliest = sampled_at - uncertainty + if (not 0 <= age + uncertainty < self.interval * 2 + 5 or + not 0 <= now - earliest < self.interval * 2 + 5 or uncertainty > 20): + continue + observed_endpoints += 1 + observed_times.append(earliest) + node['source_ok'] = True + node['clock_uncertainty_seconds'] = max(node['clock_uncertainty_seconds'], uncertainty) + mtp = _mtp_counts(values, group['model_name']) if role == 'decode' else None + if mtp is not None: + mtp_lifetime.append(mtp) + mtp_times.append(earliest) + for key, name in (('running', 'vllm:num_requests_running'), + ('waiting', 'vllm:num_requests_waiting')): + value = _sum_metric(values, name) + if value is not None: + metrics[key].append(value) + if role == 'prefill': + total = _sum_metric(values, 'vllm:prompt_tokens_total') + hits = _sum_metric(values, 'vllm:prefix_cache_hits_total') + queries = _sum_metric(values, 'vllm:prefix_cache_queries_total') + if total is not None: + metrics['input_total'].append(total) + if hits is not None and queries is not None and hits <= queries: + metrics['hit_pairs'].append((hits, queries)) + endpoint_key = (group_key, target) + previous = self.previous.get(endpoint_key) + new_previous[endpoint_key] = (values, stamp, sampled_at, uncertainty) + if previous is None: + continue + old, old_stamp, old_sampled_at, old_uncertainty = previous + duration = stamp - old_stamp + if (not 0 < duration <= self.interval * 1.5 + 5 or + sampled_at < old_sampled_at or + abs((sampled_at - old_sampled_at) - duration) > 5 + uncertainty + old_uncertainty or + values['process_start_time_seconds'] != old['process_start_time_seconds']): + continue + mtp_change = _mtp_change(values, old, group['model_name']) if role == 'decode' else None + if mtp_change is not None: + mtp_interval.append(mtp_change) + mtp_window.append((old_sampled_at - old_uncertainty, sampled_at, earliest)) + name = 'vllm:prompt_tokens_by_source_total' if role == 'prefill' else 'vllm:generation_tokens_total' + change = _delta(values, old, name, 'local_compute' if role == 'prefill' else None) + if change is None: + continue + rates.append(change / duration) + window_starts.append(old_sampled_at - old_uncertainty) + window_ends.append(sampled_at) + deadlines.append(earliest + self.interval * 2 + 5) + prefix = 'vllm:request_' + role + '_time_seconds_' + count = _delta(values, old, prefix + 'count') + seconds = _delta(values, old, prefix + 'sum') + same_series = (set(_selected(values, prefix + 'count')) == set(_selected(values, prefix + 'sum')) + and set(_selected(old, prefix + 'count')) == set(_selected(old, prefix + 'sum'))) + if count is not None and seconds is not None and count > 0 and same_series: + count_delta += count + seconds_delta += seconds + valid_timer_workers += 1 + if rates: + rate = sum(rates) + span = max(window_ends) - min(window_starts) + valid_until = min(deadlines) + if math.isfinite(rate) and 0 <= span <= 60 and valid_until > now: + part = {'tokens_per_second': rate, 'log_at': max(window_ends), 'log_skew_seconds': span, + 'valid_until': valid_until, 'step': self.sequence, + 'valid_endpoints': len(rates), 'expected_endpoints': len(group['targets']), + 'partial': len(rates) < len(group['targets']) or common_sources < len(self.ports) or ambiguous, + 'mfu_estimate': None} + node[role] = part + if valid_timer_workers and count_delta and math.isfinite(count_delta) and math.isfinite(seconds_delta): + node['phase_timers'] = {'step': self.sequence, role: { + 'mean_ms': seconds_delta * 1000 / count_delta, + 'samples': int(count_delta), 'ranks': valid_timer_workers, + 'expected_ranks': len(group['targets'])}} + if self.history is not None: + try: + self.history.record(group_key, role, part['log_at'], rate, partial=part['partial']) + except (OSError, sqlite3.Error, ValueError, OverflowError): + history_ok = False + mtp_partial = common_sources < len(self.ports) or ambiguous + lifetime = _mtp_summary(mtp_lifetime, len(group['targets']), partial=mtp_partial) + interval = _mtp_summary(mtp_interval, len(group['targets']), partial=mtp_partial) + if lifetime is not None: + lifetime.update(sampled_at=min(mtp_times), valid_until=min(mtp_times) + self.interval * 2 + 5) + if interval is not None: + span = max(end for _, end, _ in mtp_window) - min(start for start, _, _ in mtp_window) + if span > 60: + interval = None + else: + interval.update(sampled_at=max(end for _, end, _ in mtp_window), + valid_until=min(earliest for _, _, earliest in mtp_window) + self.interval * 2 + 5, + span_seconds=span) + if self.history is not None: + try: + self.history.record_mtp(mtp_key, interval['sampled_at'], interval['steps'], + interval['drafted'], interval['accepted'], + partial=interval['partial']) + except (OSError, sqlite3.Error, ValueError, OverflowError): + history_ok = False + if lifetime is not None or interval is not None: + node['mtp'] = {'lifetime': lifetime, 'interval': interval} + if self.history is not None and role == 'decode' and group['model_name']: + try: + node['mtp_history'] = self.history.mtp_series(mtp_key, now) + except (OSError, sqlite3.Error, ValueError, OverflowError): + history_ok = False + pairs = metrics['hit_pairs'] + hit_total = _sum_values([hit for hit, _ in pairs]) + query_total = _sum_values([query for _, query in pairs]) + node['received_at'] = min(observed_times) if observed_times else None + node['engine_metrics'] = { + 'running': _sum_values(metrics['running']), + 'waiting': _sum_values(metrics['waiting']), + 'input_total': _sum_values(metrics['input_total']), + 'hit_rate': hit_total / query_total if hit_total is not None and query_total else None, + 'observed_endpoints': observed_endpoints, 'expected_endpoints': len(group['targets']), + 'sampled_at': min(observed_times), 'valid_until': min(observed_times) + self.interval * 2 + 5, + 'partial': (observed_endpoints < len(group['targets']) or common_sources < len(self.ports) + or ambiguous or len(metrics['running']) < observed_endpoints + or len(metrics['waiting']) < observed_endpoints or + (role == 'prefill' and (len(metrics['input_total']) < observed_endpoints or + len(metrics['hit_pairs']) < observed_endpoints))), + } if observed_times else None + if self.history is not None: + try: + node['history'] = self.history.series(group_key, now) + except (OSError, sqlite3.Error, ValueError, OverflowError): + history_ok = False + nodes.append(node) + self.previous = new_previous + if len({node['backend_port'] for node in nodes}) != len(nodes): + nodes, ambiguous = [], True + result = {'schema': 1, 'kind': 'vllm-motor-telemetry', 'sampled_at': now, 'nodes': nodes, + 'topology_sources': {'available': common_sources, 'total': len(self.ports)}, + 'topology_ambiguous': ambiguous, 'prefill_tps': None, 'decode_tps': None, + 'prefill_mfu': None, 'hbm_percent': None, 'history_available': history_ok, + 'history_dropped': 0} + with self.lock: + self.value = result + return deepcopy(result) + + def snapshot(self): + with self.lock: + value = deepcopy(self.value) + now = self.wall_time() + age = now - value['sampled_at'] + if not 0 <= age <= self.interval * 2 + 20: + return {'schema': 1, 'kind': 'vllm-motor-telemetry', 'sampled_at': 0, + 'nodes': [], 'history_available': False} + for node in value['nodes']: + for phase in ('prefill', 'decode'): + part = node.get(phase) + if part is not None and (not 0 <= now - part['log_at'] or now >= part['valid_until']): + node[phase] = None + timers = node.get('phase_timers') + if isinstance(timers, dict) and not any(node.get(phase) and node[phase]['step'] == timers.get('step') + for phase in ('prefill', 'decode')): + node['phase_timers'] = None + metrics = node.get('engine_metrics') + if metrics is not None and (not 0 <= now - metrics['sampled_at'] or now >= metrics['valid_until']): + node['engine_metrics'] = None + mtp = node.get('mtp') + if mtp is not None: + for key in ('lifetime', 'interval'): + part = mtp.get(key) + if part is not None and (not 0 <= now - part['sampled_at'] or now >= part['valid_until']): + mtp[key] = None + if not any(mtp.values()): + node['mtp'] = None + return value + + def start(self): + if self.worker is not None: + raise RuntimeError('COLLECTOR_ALREADY_STARTED') + def run(): + while not self.stop_event.is_set(): + try: + self.collect() + except Exception: + pass + if self.stop_event.wait(self.interval): + break + self.worker = threading.Thread(target=run, name='motor-metrics', daemon=True) + self.worker.start() + + def close(self): + self.stop_event.set() + if self.worker is not None: + self.worker.join(timeout=5) diff --git a/deploy_stats/proxy.py b/deploy_stats/proxy.py new file mode 100644 index 0000000..8042892 --- /dev/null +++ b/deploy_stats/proxy.py @@ -0,0 +1,551 @@ +"""A loopback-only, byte-transparent accounting hop before the existing gateway.""" +import asyncio +from contextlib import suppress +from email.parser import Parser +from http.client import HTTPMessage +import ipaddress +import os +import socket +import threading +import time +import uuid +from urllib.parse import urlsplit + +from .capture import SafeCapture +from .session import MAX_BODY, SessionBodyCapture, explicit_session, key_digest +from .store import Recorder + +CLIENT_HEADER = 'x-llmdeploy-client-ip' +PROTOCOLS = {'/v1/chat/completions': 'chat', '/v1/responses': 'responses', + '/v1/messages': 'messages'} +# Allow a large incoming header plus additional fields from the front proxy. +MAX_HEADER = 1024 * 1024 + 65536 +MAX_RESPONSE_HEADER = 10 * 1024 * 1024 +WINDOWS_WORKERS = 3 +WINDOWS_CONNECTIONS_PER_WORKER = 160 +MAX_CONNECTIONS = WINDOWS_WORKERS * WINDOWS_CONNECTIONS_PER_WORKER + + +def client_ip(headers, *, header=CLIENT_HEADER): + values = headers.get_all(header, []) + if len(values) != 1 or not isinstance(values[0], str): + return None + try: + address = ipaddress.ip_address(values[0].strip()) + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped: + address = address.ipv4_mapped + return str(address) + except ValueError: + return None + + +def session_fingerprint(headers): + return explicit_session(headers)[1] + + +def _headers(raw): + # Match http.client.parse_headers, used by the existing gateway: header + # octets are decoded as ISO-8859-1 before the email parser sees them. + return Parser(_class=HTTPMessage).parsestr((raw + b'\r\n').decode('iso-8859-1')) + + +async def _read(reader, size, timeout): + return await asyncio.wait_for(reader.read(size), timeout) + + +async def _readexactly(reader, size, timeout): + return await asyncio.wait_for(reader.readexactly(size), timeout) + + +async def _line(reader, timeout): + return await asyncio.wait_for(reader.readuntil(b'\r\n'), timeout) + + +async def _response_body(reader, writer, headers, method, status, capture, timeout): + async def send(raw, observe=False): + if observe and capture: + capture.feed(raw) + writer.write(raw) + await writer.drain() + + if method == 'HEAD' or status in (204, 304) or 100 <= status < 200: + return + encoding = (headers.get('Transfer-Encoding') or '').lower() + lengths = headers.get_all('Content-Length', []) + if encoding: + if encoding.strip() != 'chunked' or lengths: + raise ValueError('INVALID_RESPONSE_FRAMING') + while True: + line = await _line(reader, timeout) + size = int(line[:-2].split(b';', 1)[0], 16) + if size < 0: + raise ValueError('INVALID_CHUNK_SIZE') + await send(line) + if size == 0: + while True: + trailer = await _line(reader, timeout) + await send(trailer) + if trailer == b'\r\n': + return + left = size + while left: + data = await _read(reader, min(left, 65536), timeout) + if not data: + raise asyncio.IncompleteReadError(b'', left) + await send(data, True) + left -= len(data) + suffix = await _readexactly(reader, 2, timeout) + if suffix != b'\r\n': + raise ValueError('INVALID_CHUNK_END') + await send(suffix) + elif lengths: + if len(lengths) != 1: + raise ValueError('INVALID_RESPONSE_LENGTH') + left = int(lengths[0]) + if left < 0: + raise ValueError('INVALID_RESPONSE_LENGTH') + while left: + data = await _read(reader, min(left, 65536), timeout) + if not data: + raise asyncio.IncompleteReadError(b'', left) + await send(data, True) + left -= len(data) + else: + while True: + data = await _read(reader, 65536, timeout) + if not data: + return + await send(data, True) + + +class Proxy: + """Async transfer keeps cancellation independent from slow upstream reads.""" + + def __init__(self, port, upstream_port, database, *, timeout=10800): + if type(port) is not int or type(upstream_port) is not int or not 0 <= port <= 65535 or not 1 <= upstream_port <= 65535: + raise ValueError('INVALID_PORT') + if not 0 < timeout <= 86400: + raise ValueError('INVALID_TIMEOUT') + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) + self.socket.bind(('127.0.0.1', port)) + self.socket.listen(MAX_CONNECTIONS) + self.socket.setblocking(False) + self.server_port = self.socket.getsockname()[1] + if self.server_port == upstream_port: + raise ValueError('UPSTREAM_EQUALS_PROXY') + self.recorder = Recorder(database) + except Exception: + self.socket.close() + raise + self.upstream_port = upstream_port + self.timeout = timeout + self.active_lock = threading.Lock() + self.active = 0 + self.loop = None + self.stop = None + self.finished = threading.Event() + self.ready = threading.Event() + self.tasks = set() + self.tasks_lock = threading.Lock() + self.connections_lock = threading.Lock() + self.connections = 0 + self.shutdown_requested = threading.Event() + + def serve_forever(self, poll_interval=None): + try: + if os.name == 'nt': + self._serve_windows() + else: + asyncio.run(self._serve()) + finally: + self.finished.set() + + async def _serve(self): + self.loop = asyncio.get_running_loop() + self.stop = asyncio.Event() + server = await asyncio.start_server(self._handle, sock=self.socket, limit=MAX_HEADER) + self.ready.set() + async with server: + await self.stop.wait() + with self.tasks_lock: + tasks = list(self.tasks) + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + def shutdown(self): + if self.ready.wait(3): + self.shutdown_requested.set() + if self.loop and self.stop: + self.loop.call_soon_threadsafe(self.stop.set) + self.finished.wait(5) + + def _serve_windows(self): + # One Windows selector can watch only 512 sockets. Dispatch accepted + # connections across independent loops, each with room for both the + # client and upstream sockets, instead of using the reset-prone proactor. + workers = [] + worker_lock = threading.Lock() + started = [threading.Event() for _ in range(WINDOWS_WORKERS)] + loops = [None] * WINDOWS_WORKERS + stops = [None] * WINDOWS_WORKERS + assigned = [0] * WINDOWS_WORKERS + errors = [] + + async def worker(index): + loop = asyncio.get_running_loop() + loops[index], stops[index] = loop, asyncio.Event() + started[index].set() + await stops[index].wait() + with self.tasks_lock: + tasks = [task for task in self.tasks if task.get_loop() is loop] + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + def run_worker(index): + try: + with asyncio.Runner(loop_factory=asyncio.SelectorEventLoop) as runner: + runner.run(worker(index)) + except Exception as exc: + errors.append(exc) + started[index].set() + self.shutdown_requested.set() + + async def handle_socket(conn, index): + try: + loop = asyncio.get_running_loop() + reader = asyncio.StreamReader(limit=MAX_HEADER) + protocol = asyncio.StreamReaderProtocol(reader) + transport, _ = await loop.connect_accepted_socket(lambda: protocol, conn) + writer = asyncio.StreamWriter(transport, protocol, reader, loop) + await self._handle(reader, writer) + finally: + conn.close() + with worker_lock: + assigned[index] -= 1 + + def dispatch(conn, index): + task = asyncio.create_task(handle_socket(conn, index)) + # Track handoff as well as the handler so shutdown cannot leave an + # accepted but not-yet-attached socket behind. + with self.tasks_lock: + self.tasks.add(task) + task.add_done_callback(lambda done: self._discard_task(done)) + + try: + for index in range(WINDOWS_WORKERS): + thread = threading.Thread(target=run_worker, args=(index,), daemon=True) + workers.append(thread) + thread.start() + if not all(event.wait(3) for event in started) or errors: + raise RuntimeError('PROXY_WORKER_START_FAILED') + self.socket.settimeout(.2) + self.ready.set() + while not self.shutdown_requested.is_set(): + try: + conn, _ = self.socket.accept() + except socket.timeout: + continue + conn.setblocking(False) + with worker_lock: + index = min(range(WINDOWS_WORKERS), key=assigned.__getitem__) + if assigned[index] < WINDOWS_CONNECTIONS_PER_WORKER: + assigned[index] += 1 + else: + index = None + if index is None: + self.recorder.note_drop() + with suppress(OSError): + # Give an already-arriving head a short chance to drain; + # closing with unread bytes otherwise resets Windows peers. + conn.settimeout(.05) + conn.recv(MAX_HEADER) + with suppress(OSError): + conn.sendall(b'HTTP/1.1 429 Too Many Requests\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') + conn.close() + else: + loops[index].call_soon_threadsafe(dispatch, conn, index) + finally: + for loop, stop in zip(loops, stops): + if loop and stop: + with suppress(RuntimeError): + loop.call_soon_threadsafe(stop.set) + for thread in workers: + thread.join(5) + + def _discard_task(self, task): + with self.tasks_lock: + self.tasks.discard(task) + + def server_close(self): + if self.ready.is_set() and not self.finished.is_set(): + self.shutdown() + if not self.finished.is_set(): + raise RuntimeError('PROXY_SHUTDOWN_TIMEOUT') + self.socket.close() + with self.active_lock: + with self.recorder.lock: + self.recorder.dropped += self.active + self.recorder.close() + + async def _handle(self, client_reader, client_writer): + task = asyncio.current_task() + with self.tasks_lock: + self.tasks.add(task) + with self.connections_lock: + accepted = self.connections < MAX_CONNECTIONS + if accepted: + self.connections += 1 + if not accepted: + self.recorder.note_drop() + # Read the request head before closing so Windows does not reset a + # small request while the caller is trying to receive the 429. + with suppress(OSError, ConnectionError, asyncio.TimeoutError, asyncio.LimitOverrunError, + asyncio.IncompleteReadError): + await asyncio.wait_for(client_reader.readuntil(b'\r\n\r\n'), 1) + with suppress(OSError, ConnectionError): + client_writer.write(b'HTTP/1.1 429 Too Many Requests\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') + await client_writer.drain() + client_writer.close() + with suppress(OSError, ConnectionError, asyncio.TimeoutError): + await asyncio.wait_for(client_writer.wait_closed(), 1) + self._discard_task(task) + return + started = time.perf_counter() + started_wall = time.time() + counted = False + source = None + session = None + has_session_header = False + request_id = None + first_visible = None + status, state, values = None, 'interrupted', None + sent_headers = False + upstream_writer = None + body_task = watch_task = copy_task = head_task = None + body_done = asyncio.Event() + abort_send = asyncio.Event() + disconnected = asyncio.Event() + try: + raw = await asyncio.wait_for(client_reader.readuntil(b'\r\n\r\n'), 30) + request_line, _, header_bytes = raw[:-4].partition(b'\r\n') + method_bytes, target_bytes, version = request_line.split(b' ', 2) + method = method_bytes.decode('ascii') + target = target_bytes.decode('ascii') + if version not in (b'HTTP/1.0', b'HTTP/1.1'): + raise ValueError('INVALID_HTTP_VERSION') + path = urlsplit(target).path + counted = method == 'POST' and path.startswith('/v1/') + protocol = PROTOCOLS.get(path) + headers = _headers(header_bytes) + source = client_ip(headers) + if counted: + has_session_header, session = explicit_session(headers) + request_id = uuid.uuid4().hex + with self.active_lock: + self.active += 1 + try: + self.recorder.begin(request_id, started_wall, source, session) + except Exception: + self.recorder.note_drop() + lines = header_bytes.split(b'\r\n') + if any(line.startswith((b' ', b'\t')) for line in lines): + raise ValueError('FOLDED_HEADER_UNSUPPORTED') + forward = [line for line in lines if line.split(b':', 1)[0].strip().lower() + not in (CLIENT_HEADER.encode(), b'connection')] + forward.append(b'Connection: close') + upstream_reader, upstream_writer = await asyncio.wait_for( + asyncio.open_connection('127.0.0.1', self.upstream_port, limit=MAX_RESPONSE_HEADER), 5) + upstream_writer.write(request_line + b'\r\n' + b'\r\n'.join(forward) + b'\r\n\r\n') + await upstream_writer.drain() + lengths = headers.get_all('Content-Length', []) + try: + length = int(lengths[0]) if len(lengths) == 1 else 0 + except ValueError: + length = 0 + if headers.get('Transfer-Encoding') or len(lengths) > 1 or not 0 <= length <= 64 * 1024 * 1024: + length = 0 # Existing gateway rejects the unchanged headers. + body_capture = None + if counted and not has_session_header and 0 < length <= MAX_BODY: + try: + body_capture = SessionBodyCapture(key_digest(headers)) + except Exception: + self.recorder.note_drop() + + async def send_body(): + nonlocal session, body_capture + left = length + try: + while left: + chunk = await _read(client_reader, min(left, 65536), 30) + if not chunk: + disconnected.set() + upstream_writer.close() + return + left -= len(chunk) + if not abort_send.is_set(): + if body_capture: + try: + body_capture.feed(chunk) + except Exception: + with suppress(Exception): + body_capture.close() + body_capture = None + self.recorder.note_drop() + try: + upstream_writer.write(chunk) + await upstream_writer.drain() + await asyncio.sleep(0) + except (OSError, ConnectionError): + abort_send.set() + elif body_capture: + with suppress(Exception): + body_capture.close() + body_capture = None + except Exception: + disconnected.set() + upstream_writer.close() + finally: + try: + if body_capture: + if left == 0 and not disconnected.is_set(): + session = body_capture.finish() + if session is not None: + self.recorder.set_session(request_id, session) + else: + body_capture.close() + except Exception: + if body_capture: + with suppress(Exception): + body_capture.close() + self.recorder.note_drop() + finally: + body_done.set() + + async def watch_client(): + await body_done.wait() + try: + while await client_reader.read(65536): + pass + except Exception: + pass + disconnected.set() + upstream_writer.close() + + head_task = asyncio.create_task(asyncio.wait_for( + upstream_reader.readuntil(b'\r\n\r\n'), self.timeout)) + if length: + # Let an immediate auth/admission rejection arrive before a + # large upload can reset an upstream that does not read it. + deadline = asyncio.get_running_loop().time() + (.1 if length >= 4096 else .01) + while True: + remaining = max(0, deadline - asyncio.get_running_loop().time()) + done, _ = await asyncio.wait((head_task,), timeout=remaining) + if not done: + break + preliminary = head_task.result() + parts = preliminary.split(b'\r\n', 1)[0].split(b' ', 2) + if len(parts) < 2 or not parts[1].isdigit(): + break + code = int(parts[1]) + if code < 100 or code > 599: + raise ValueError('INVALID_RESPONSE_STATUS') + if code >= 200: + abort_send.set() + break + client_writer.write(preliminary) + await client_writer.drain() + head_task = asyncio.create_task(asyncio.wait_for( + upstream_reader.readuntil(b'\r\n\r\n'), self.timeout)) + body_task = asyncio.create_task(send_body()) + watch_task = asyncio.create_task(watch_client()) + while True: + response_head = await head_task + status_line, _, response_fields = response_head[:-4].partition(b'\r\n') + status_parts = status_line.split(b' ', 2) + status = int(status_parts[1]) + if status < 100 or status > 599: + raise ValueError('INVALID_RESPONSE_STATUS') + if status >= 200: + fields = [line for line in response_fields.split(b'\r\n') if line and + line.split(b':', 1)[0].strip().lower() != b'connection'] + response_head = status_line + b'\r\n' + b'\r\n'.join(fields + [b'Connection: close']) + b'\r\n\r\n' + client_writer.write(response_head) + await client_writer.drain() + if status >= 200: + sent_headers = True + break + head_task = asyncio.create_task(asyncio.wait_for( + upstream_reader.readuntil(b'\r\n\r\n'), self.timeout)) + if not body_done.is_set(): + abort_send.set() + response_headers = _headers(response_fields) + def first_output(): + nonlocal first_visible + if first_visible is None: + first_visible = max(0, time.perf_counter() - started) + if request_id is not None: + self.recorder.first_output(request_id, first_visible) + + capture = SafeCapture(protocol, response_headers.get('Content-Type'), + response_headers.get('Content-Encoding'), first_output) if protocol else None + copy_task = asyncio.create_task(_response_body(upstream_reader, client_writer, response_headers, + method, status, capture, self.timeout)) + done, _ = await asyncio.wait((copy_task, watch_task), return_when=asyncio.FIRST_COMPLETED) + error_response_completed = (status >= 400 and copy_task in done and + not copy_task.cancelled() and copy_task.exception() is None) + if disconnected.is_set() and not error_response_completed: + copy_task.cancel() + await asyncio.gather(copy_task, return_exceptions=True) + else: + await copy_task + if status >= 400: + state = 'failed' + elif capture and (values := capture.finish()) is not None: + state = 'known' + else: + state = 'missing' + except Exception: + if not sent_headers and not disconnected.is_set(): + with suppress(OSError, ConnectionError): + status = 502 + client_writer.write(b'HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n') + await client_writer.drain() + finally: + abort_send.set() + if body_task and not body_task.done(): + with suppress(asyncio.TimeoutError, OSError, ConnectionError): + await asyncio.wait_for(asyncio.shield(body_task), 2) + pending_tasks = [pending for pending in (body_task, watch_task, copy_task, head_task) if pending] + for pending in pending_tasks: + if pending and not pending.done(): + pending.cancel() + if pending_tasks: + await asyncio.gather(*pending_tasks, return_exceptions=True) + if upstream_writer: + upstream_writer.close() + with suppress(OSError, ConnectionError, asyncio.TimeoutError): + await asyncio.wait_for(upstream_writer.wait_closed(), 1) + client_writer.close() + with suppress(OSError, ConnectionError, asyncio.TimeoutError): + await asyncio.wait_for(client_writer.wait_closed(), 1) + if counted: + prompt, output, cached = values if state == 'known' else (None, None, None) + with self.active_lock: + try: + self.recorder.end(request_id) + self.recorder.submit((time.time(), source, max(0, time.perf_counter() - started), + status, state, prompt, output, cached, session, first_visible), request_id) + except Exception: + self.recorder.note_drop() + finally: + self.active -= 1 + with self.connections_lock: + self.connections -= 1 + self._discard_task(task) diff --git a/deploy_stats/remote_observe.py b/deploy_stats/remote_observe.py new file mode 100644 index 0000000..d4432f8 --- /dev/null +++ b/deploy_stats/remote_observe.py @@ -0,0 +1,248 @@ +"""Read bounded DeepInfer log summaries and Ascend device metrics on a host. + +This file is streamed to the remote Python interpreter; raw container logs and +container configuration are not returned or written by the probe. +""" +import ctypes +from datetime import datetime, timezone +import hashlib +import json +import math +import os +import re +import selectors +import subprocess +import sys +import time + + +SOURCES = {'stats': '/opt/deepinfer-build/source/deepinfer/engine/stats_logger.py', + 'backend': '/opt/deepinfer-build/source/examples/recipe_api/backend.py', + 'parallel': '/opt/deepinfer-build/source/deepinfer/config.py', + 'engine': '/opt/deepinfer-build/source/deepinfer/engine/llm_engine.py', + 'sampler': '/opt/deepinfer-build/source/deepinfer/model_executor/layers/sampler.py', + 'native': '/opt/deepinfer-build/source/deepinfer/_so/ascend-aarch64/_C.cpython-312-aarch64-linux-gnu.so'} +LOG_LIMIT = 8 * 1024 * 1024 +DRIVER = '/usr/local/Ascend/driver/lib64/driver/libdcmi.so' +CONTAINER = re.compile(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z') +HEADER = re.compile(r'RANK\s+(\d+),\s*GPU\s+(\d+),\s*step\s+(\d+),\s*\[(stats|cpu)\]') +STAMP = re.compile(r'^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z)\s') +NUMBER = r'(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?' +TIMER = re.compile(r'engine_step:(prefill|decode)\s+cur(' + NUMBER + r')ms/mean(' + NUMBER + + r')ms/max(' + NUMBER + r')ms/(\d+)calls/') + + +def _number(line, name): + match = re.search(r'(?:^|\|\s*)' + re.escape(name) + r'\s+(' + NUMBER + r')(?=\s|\||\))', line) + if not match: + return None + value = float(match.group(1)) + return value if math.isfinite(value) and value >= 0 else None + + +def parse_log(data): + """Extract numerical log fields only; invalid fields do not erase good ones.""" + rows = [] + for line in data.decode('utf-8', errors='replace').splitlines(): + stamp, head = STAMP.match(line), HEADER.search(line) + if not stamp or not head: + continue + try: + at = datetime.fromisoformat(stamp.group(1).replace('Z', '+00:00')).timestamp() + rank, device, step = (int(head.group(i)) for i in (1, 2, 3)) + except (ValueError, OverflowError): + continue + if rank > 255 or device > 255 or step > 2**63 - 1: + continue + row = {'at': at, 'rank': rank, 'device': device, 'step': step, 'kind': head.group(4)} + if row['kind'] == 'stats': + phase = re.search(r'\bis_prompt\s+(True|False)(?=\s|\||\))', line) + row.update(is_prompt=(phase.group(1) == 'True') if phase else None, + current_tps=_number(line, 'cur_TPS'), current_sps=_number(line, 'cur_SPS'), + prompt_tps=_number(line, 'prompt_TPS'), decode_tps=_number(line, 'completion_TPS'), + mixed_mfu=_number(line, 'MFU')) + share = re.search(r'\bprefill/decode\s+(' + NUMBER + r')%/(' + NUMBER + r')%', line) + if share: + p, d = (float(share.group(i)) for i in (1, 2)) + row['phase_share'] = [p, d] if math.isfinite(p) and math.isfinite(d) and p <= 100 and d <= 100 else None + else: + row['phase_share'] = None + else: + timers = {} + for match in TIMER.finditer(line): + phase = match.group(1) + if phase in timers: + timers[phase] = None + continue + cur, mean, maximum = (float(match.group(i)) for i in (2, 3, 4)) + calls = int(match.group(5)) + timers[phase] = ({'mean_ms': mean, 'calls': calls} if calls > 0 and calls < 10**9 and + all(math.isfinite(x) and x >= 0 for x in (cur, mean, maximum)) else None) + row['timers'] = timers + rows.append(row) + return rows + + +def bounded_command(command, timeout, limit): + """Collect finite output without leaving a blocked or unbounded log reader.""" + if command and command[0] == 'docker' and not docker_ready(): + raise ValueError('docker is not safely available') + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + output = bytearray() + selector = selectors.DefaultSelector() + deadline = time.monotonic() + timeout + try: + os.set_blocking(proc.stdout.fileno(), False) + selector.register(proc.stdout, selectors.EVENT_READ) + while selector.get_map(): + left = deadline - time.monotonic() + if left <= 0: + raise TimeoutError('probe timeout') + for key, _ in selector.select(min(left, .25)): + chunk = os.read(key.fd, min(65536, limit + 1 - len(output))) + if not chunk: + selector.unregister(key.fileobj) + break + output.extend(chunk) + if len(output) > limit: + raise ValueError('probe output too large') + if proc.wait(timeout=max(.01, deadline - time.monotonic())) != 0: + raise ValueError('probe command failed') + return bytes(output) + finally: + selector.close() + if proc.poll() is None: + proc.kill() + proc.wait(timeout=2) + proc.stdout.close() + + +def docker_ready(): + """Do not trigger Docker socket activation while observing an idle host.""" + try: + service = subprocess.run(['systemctl', 'is-active', 'docker.service'], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, timeout=3, check=False) + socket = subprocess.run(['systemctl', 'is-active', 'docker.socket'], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, timeout=3, check=False) + return service.returncode == 0 and service.stdout.strip() == b'active' and socket.stdout.strip() == b'inactive' + except (OSError, subprocess.SubprocessError): + return False + + +class HbmInfo(ctypes.Structure): + _fields_ = [('memory_size', ctypes.c_ulonglong), ('freq', ctypes.c_uint), + ('memory_usage', ctypes.c_ulonglong), ('temp', ctypes.c_int), + ('bandwidth', ctypes.c_uint)] + + +def sample_hbm(): + """Read device-wide driver percentages; failures remain unknown, never zero.""" + lib = ctypes.CDLL(DRIVER) + lib.dcmiv2_init.argtypes = [] + lib.dcmiv2_init.restype = ctypes.c_int + lib.dcmiv2_get_device_list.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int), ctypes.c_int] + lib.dcmiv2_get_device_list.restype = ctypes.c_int + lib.dcmiv2_get_device_hbm_info.argtypes = [ctypes.c_int, ctypes.POINTER(HbmInfo)] + lib.dcmiv2_get_device_hbm_info.restype = ctypes.c_int + if lib.dcmiv2_init() != 0: + return None + ids, count = (ctypes.c_int * 64)(), ctypes.c_int() + if lib.dcmiv2_get_device_list(ids, ctypes.byref(count), 64) != 0 or not 0 < count.value <= 64: + return None + results = [] + for dev in list(ids)[:count.value]: + started = time.time() + value = HbmInfo() + status = lib.dcmiv2_get_device_hbm_info(dev, ctypes.byref(value)) + ended = time.time() + valid = status == 0 and 0 < value.memory_size and value.bandwidth <= 100 + results.append({'id': dev, 'started_at': started, 'finished_at': ended, + 'hbm_bandwidth_percent': value.bandwidth if valid else None}) + return results + + +def source_predates(mtimes, started_at): + return (len(mtimes) == len(SOURCES) and math.isfinite(started_at) and all( + part.isascii() and part.isdigit() and int(part) < math.floor(started_at) for part in mtimes)) + + +def driver_digest(): + if not 0 < os.stat(DRIVER).st_size <= 16 * 1024 * 1024: + raise ValueError('driver size invalid') + digest = hashlib.sha256() + with open(DRIVER, 'rb') as source: + for chunk in iter(lambda: source.read(65536), b''): + digest.update(chunk) + return digest.hexdigest() + + +def probe(container): + if not CONTAINER.fullmatch(container): + raise ValueError('invalid container') + cpu_start = time.process_time() + try: + import resource + child_start = resource.getrusage(resource.RUSAGE_CHILDREN) + except ImportError: + resource, child_start = None, None + result = {'schema': 1, 'started_at': time.time(), 'source_id': None, 'host_fingerprint': None, + 'image_fingerprint': None, 'driver_sha256': None, 'source_started_at': None, + 'source_before_start': False, 'source_sha256': {}, + 'engine_rows': [], 'hbm': None} + identity = None + identity_cmd = ['docker', 'inspect', '--format', + '{{.Id}} {{.Image}} {{.State.StartedAt}} {{.State.Running}}', container] + try: + identity = bounded_command(identity_cmd, 5, 1024).decode().strip() + fields = identity.split() + if len(fields) != 4 or fields[-1] != 'true': + raise ValueError('container unavailable') + machine_id = open('/etc/machine-id', encoding='ascii').read().strip() + if not re.fullmatch(r'[0-9a-f]{32}', machine_id) or machine_id == '0' * 32: + raise ValueError('host identity unavailable') + result['host_fingerprint'] = hashlib.sha256(machine_id.encode()).hexdigest() + result['image_fingerprint'] = hashlib.sha256(fields[1].encode()).hexdigest() + result['source_id'] = hashlib.sha256(identity.encode()).hexdigest() + result['source_started_at'] = datetime.fromisoformat(fields[2].replace('Z', '+00:00')).timestamp() + for key, path in SOURCES.items(): + digest = bounded_command(['docker', 'exec', container, 'sha256sum', path], 5, 1024).decode().split()[0] + if not re.fullmatch(r'[0-9a-f]{64}', digest): + raise ValueError('source unavailable') + result['source_sha256'][key] = digest + mtimes = bounded_command(['docker', 'exec', container, 'stat', '-L', '-c', '%Y', *SOURCES.values()], + 5, 1024).decode().split() + result['source_before_start'] = source_predates(mtimes, result['source_started_at']) + if not result['source_before_start']: + raise ValueError('source changed after container start') + raw = bounded_command(['docker', 'logs', '--timestamps', '--since', '120s', '--tail', '3000', container], + 8, LOG_LIMIT) + result['engine_rows'] = parse_log(raw)[-1024:] + except (OSError, ValueError, TimeoutError, UnicodeError, IndexError, subprocess.SubprocessError): + pass + try: + result['driver_sha256'] = driver_digest() + result['hbm'] = sample_hbm() + if driver_digest() != result['driver_sha256']: + result['driver_sha256'], result['hbm'] = None, None + except (OSError, ValueError, AttributeError): + result['driver_sha256'], result['hbm'] = None, None + if identity is not None: + try: + if bounded_command(identity_cmd, 5, 1024).decode().strip() != identity: + raise ValueError('container changed') + except (OSError, ValueError, TimeoutError, UnicodeError, subprocess.SubprocessError): + result.update(source_id=None, host_fingerprint=None, image_fingerprint=None, driver_sha256=None, + source_before_start=False, source_sha256={}, engine_rows=[], hbm=None) + result['finished_at'] = time.time() + result['probe_cpu_seconds'] = time.process_time() - cpu_start + if resource is not None: + child_end = resource.getrusage(resource.RUSAGE_CHILDREN) + result['probe_cpu_seconds'] += (child_end.ru_utime + child_end.ru_stime - + child_start.ru_utime - child_start.ru_stime) + return result + + +if __name__ == '__main__': + if len(sys.argv) != 2 or not CONTAINER.fullmatch(sys.argv[1]): + raise SystemExit(2) + print(json.dumps(probe(sys.argv[1]), allow_nan=False, separators=(',', ':'))) diff --git a/deploy_stats/report.py b/deploy_stats/report.py new file mode 100644 index 0000000..040bd12 --- /dev/null +++ b/deploy_stats/report.py @@ -0,0 +1,388 @@ +"""Read-only windows, per-IP aggregates, and completion-time buckets.""" +from contextlib import closing +from datetime import datetime, timedelta, timezone +import ipaddress +import math +from pathlib import Path +import sqlite3 +import time + +from .store import IDENTITY, SCHEMA, partial_schema_available + +TZ = timezone(timedelta(hours=8)) +BUCKET_SECONDS = 600 +COVERAGE_EPSILON = 1e-6 +REQUEST_SORTS = frozenset(('finished', 'ip', 'session', 'state', 'input', 'output', 'cached', 'uncached_input', 'hit_rate', + 'first_visible', 'after_first', 'input_per_wait', 'output_per_after_first', 'first_share')) +SESSION_SORTS = frozenset(('last_finished', 'session', 'ips', 'requests', 'input', 'output', 'cached', 'uncached_input', 'hit_rate', + 'first_visible', 'after_first', 'input_per_wait', 'output_per_after_first', 'first_share')) + + +def _coverage(runs, start, end, now): + intervals = [] + dropped = 0 + unclean = 0 + for run in runs: + left, seen, ended, lost = run + live = ended is None and 0 <= now - seen < 15 + right = ended if ended is not None else (now if live else seen) + if left < end and right > start: + intervals.append((max(start, left), min(end, right))) + dropped += lost + if ended is None and not live: + # Events queued before the last heartbeat may never have been + # committed when a process exited without a clean drain. + unclean += 1 + intervals.sort() + covered, edge = 0.0, start + for left, right in intervals: + covered += max(0, right - max(left, edge)) + edge = max(edge, right) + return min(max(0.0, end - start), covered), dropped, unclean + + +def _counts(rows): + out = {'requests': 0, 'known': 0, 'partial': 0, 'missing': 0, 'failed': 0, 'interrupted': 0, + 'input': 0, 'output': 0, 'cached': 0, 'uncached_input': 0, 'elapsed': 0.0} + hit_input = 0 + for row in rows: + state, prompt, output, cached, elapsed = row + out['requests'] += 1 + if state in ('known', 'partial'): + out[state] += 1 + out['input'] += prompt + out['output'] += output + if state == 'known': + out['cached'] += cached + out['uncached_input'] += prompt - cached + hit_input += prompt + out['elapsed'] += elapsed + else: + out[state if state in ('missing', 'failed', 'interrupted') else 'missing'] += 1 + out['uncached_input'] = out['uncached_input'] if out['known'] else None + out['hit_rate'] = out['cached'] / hit_input if hit_input else None + out['request_tps'] = out['output'] / out['elapsed'] if out['elapsed'] else None + return out + + +def _percentile(values, fraction): + if not values: + return None + values = sorted(values) + position = (len(values) - 1) * fraction + left = int(position) + right = min(left + 1, len(values) - 1) + return values[left] + (values[right] - values[left]) * (position - left) + + +def _request_view(row): + finished, ip, state, prompt, output, cached, elapsed, session, first, status, record_id = row + observed = (state in ('known', 'partial') and first is not None and math.isfinite(first) and + 0 <= first <= elapsed and elapsed > 0) + tail = elapsed - first if observed else None + known = state == 'known' + measured = state in ('known', 'partial') + return {'id': record_id, 'finished': finished, 'ip': ip, 'session': session, 'state': state, + 'status': status, 'input': prompt, 'output': output, 'cached': cached, + 'uncached_input': prompt - cached if known else None, + 'hit_rate': cached / prompt if known and prompt else None, 'elapsed': elapsed, + 'first_visible': first if observed else None, 'after_first': tail, + 'first_share': first / elapsed if observed else None, + 'after_first_share': tail / elapsed if observed else None, + # Estimates use gateway-observed intervals, not engine phase times. + 'input_per_wait': (prompt - cached) / first if known and observed and first > 0 else None, + 'output_per_after_first': (output - 1) / tail if measured and observed and output > 1 and tail > 0 else None} + + +def _session_views(details): + groups = {} + for item in details: + if item['session']: + groups.setdefault(item['session'], []).append(item) + result = [] + for session, items in groups.items(): + known = [item for item in items if item['state'] == 'known'] + partial = [item for item in items if item['state'] == 'partial'] + measured = known + partial + timed = [item for item in measured if item['first_visible'] is not None] + rate_items = [item for item in known if item['first_visible'] is not None and item['first_visible'] > 0] + wait = sum(item['first_visible'] for item in timed) + after = sum(item['after_first'] for item in timed) + total_input = sum(item['input'] for item in measured) + total_output = sum(item['output'] for item in measured) + hit_input = sum(item['input'] for item in known) + total_cached = sum(item['cached'] for item in known) + in_wait = sum(item['first_visible'] for item in rate_items) + out_items = [item for item in timed if item['output'] > 1 and item['after_first'] > 0] + out_time = sum(item['after_first'] for item in out_items) + result.append({'session': session, 'ips': sorted({item['ip'] or '未知来源' for item in items}), + 'requests': len(items), 'known': len(known), 'partial': len(partial), 'timed': len(timed), + 'input': total_input if measured else None, 'output': total_output if measured else None, + 'cached': total_cached if known else None, + 'uncached_input': sum(item['uncached_input'] for item in known) if known else None, + 'hit_rate': total_cached / hit_input if hit_input else None, + 'first_visible': wait if timed else None, 'after_first': after if timed else None, + 'first_share': wait / (wait + after) if timed and wait + after > 0 else None, + 'input_per_wait': sum(item['input'] - item['cached'] for item in rate_items) / in_wait if in_wait else None, + 'output_per_after_first': sum(item['output'] - 1 for item in out_items) / out_time if out_time else None, + 'last_finished': max(item['finished'] for item in items)}) + return sorted(result, key=lambda item: item['last_finished'], reverse=True) + + +def _ordered_page(items, field, direction, offset, limit, identity): + def value(item): + result = item.get(field) + if field == 'ips': + known = [address for address in result if address != '未知来源'] if result else [] + result = ', '.join(known) if known else None + elif field == 'ip' and result is not None: + try: + address = ipaddress.ip_address(result) + return address.version, int(address) + except ValueError: + return None + elif field == 'state': + labels = {'known': '已计量', 'partial': '部分用量', 'missing': '未计量', 'failed': '失败', 'interrupted': '中断'} + return labels.get(result, '未知'), item.get('status') or 0 + if isinstance(result, str): + result = result.casefold() + return result + + present = [item for item in items if value(item) is not None] + missing = [item for item in items if value(item) is None] + present.sort(key=lambda item: (value(item), item[identity]), reverse=direction == 'desc') + missing.sort(key=lambda item: item[identity]) + ordered = present + missing + offset = min(offset, ((len(ordered) - 1) // limit) * limit) if ordered else 0 + return ordered[offset:offset + limit], offset + + +def _check_database(db): + if db.execute('PRAGMA user_version').fetchone()[0] != SCHEMA: + raise ValueError('STATS_DATABASE_SCHEMA_MISMATCH') + try: + marker = db.execute("SELECT value FROM stats_meta WHERE key='application'").fetchone() + except sqlite3.Error as error: + raise ValueError('STATS_DATABASE_SCHEMA_MISMATCH') from error + if marker != (IDENTITY,): + raise ValueError('STATS_DATABASE_SCHEMA_MISMATCH') + + +def _active_snapshot(db, now, source=None): + source_filter = (" AND EXISTS (SELECT 1 FROM stats_meta m WHERE m.key='source:' || r.id AND m.value=?)" + if source is not None else '') + params = (now - 15, now, source) if source is not None else (now - 15, now) + samples = list(db.execute('''SELECT r.last_seen,s.seen FROM runs r + LEFT JOIN active_samples s ON s.run_id=r.id WHERE r.ended IS NULL AND r.last_seen>? AND r.last_seen<=?''' + source_filter, + params)) + ready = bool(samples) and all(seen is not None and 0 <= now - seen < 3 for _, seen in samples) + rows = list(db.execute('''SELECT a.client_ip,a.session_id,a.started,a.first_visible + FROM active_requests a JOIN runs r ON a.run_id=r.id + WHERE r.ended IS NULL AND r.last_seen>? AND r.last_seen<=?''' + source_filter + ' ORDER BY a.started', params)) if ready else [] + current = [{'ip': ip, 'session': session, 'started': started, 'elapsed': max(0, now - started), + 'first_visible': first, 'input': None, 'output': None, 'cached': None} + for ip, session, started, first in rows] + return {'as_of': now, 'collector_live': bool(samples), 'active': current, 'active_ready': ready, + 'active_sampled_at': min((seen for _, seen in samples), default=None) if ready else None} + + +def active_report(path, *, now=None, source=None): + now = time.time() if now is None else float(now) + uri = Path(path).resolve().as_uri() + '?mode=ro' + with closing(sqlite3.connect(uri, uri=True, timeout=1)) as db: + db.execute('PRAGMA query_only=ON') + db.execute('BEGIN') + _check_database(db) + return _active_snapshot(db, now, source) + + +def report(path, window='24h', *, now=None, request_offset=0, request_limit=100, session_offset=0, + session_limit=None, request_sort='finished', request_dir='desc', + session_sort='last_finished', session_dir='desc', snapshot_at=None, snapshot_id=None, source=None): + if window not in ('today', '24h', '7d', 'total'): + raise ValueError('INVALID_STATS_WINDOW') + if session_limit is None: + session_limit = request_limit + if (type(request_offset) is not int or request_offset < 0 or type(session_offset) is not int or session_offset < 0 or + type(request_limit) is not int or not 1 <= request_limit <= 200 or + type(session_limit) is not int or not 1 <= session_limit <= 200 or + not isinstance(request_sort, str) or request_sort not in REQUEST_SORTS or + not isinstance(session_sort, str) or session_sort not in SESSION_SORTS or + request_dir not in ('asc', 'desc') or session_dir not in ('asc', 'desc')): + raise ValueError('INVALID_STATS_PAGE') + now = time.time() if now is None else float(now) + if ((snapshot_at is None) != (snapshot_id is None) or + snapshot_id is not None and (type(snapshot_id) is not int or snapshot_id < 0) or + snapshot_at is not None and (type(snapshot_at) not in (int, float) or + not math.isfinite(snapshot_at) or not 0 < snapshot_at <= now)): + raise ValueError('INVALID_STATS_SNAPSHOT') + uri = Path(path).resolve().as_uri() + '?mode=ro' + with closing(sqlite3.connect(uri, uri=True, timeout=1)) as db: + db.execute('PRAGMA query_only=ON') + db.execute('BEGIN') + _check_database(db) + all_runs = list(db.execute('SELECT started,last_seen,ended,dropped FROM runs')) + runs = (all_runs if source is None else list(db.execute(''' + SELECT started,last_seen,ended,dropped FROM runs r + WHERE EXISTS (SELECT 1 FROM stats_meta m WHERE m.key='source:' || r.id AND m.value=?)''', (source,)))) + overload_runs = list(db.execute('''SELECT runs.started,runs.last_seen,runs.ended,stats_meta.value + FROM runs LEFT JOIN stats_meta ON stats_meta.key='overload:' || runs.id''')) + since = min((r[0] for r in all_runs), default=now) + if snapshot_at is None: + snapshot_at = now + snapshot_id = db.execute('SELECT COALESCE(MAX(id),0) FROM requests').fetchone()[0] + midnight = datetime.fromtimestamp(now, TZ).replace(hour=0, minute=0, second=0, microsecond=0).timestamp() + start = {'today': midnight, '24h': now - 86400, '7d': now - 7 * 86400, + 'total': since}[window] + start = min(start, now) + snapshot_midnight = datetime.fromtimestamp(snapshot_at, TZ).replace(hour=0, minute=0, second=0, microsecond=0).timestamp() + snapshot_start = min(snapshot_at, {'today': snapshot_midnight, '24h': snapshot_at - 86400, + '7d': snapshot_at - 7 * 86400, 'total': since}[window]) + if partial_schema_available(db): + query = '''SELECT r.finished,r.client_ip,CASE WHEN p.request_id IS NULL THEN r.state ELSE 'partial' END, + COALESCE(p.input_tokens,r.input_tokens),COALESCE(p.output_tokens,r.output_tokens),r.cached_tokens,r.elapsed, + r.session_id,r.first_visible,r.http_status,r.id FROM requests r + LEFT JOIN stats_partial_usage p ON p.request_id=r.id WHERE r.finished>=? AND r.finished=? AND finished= start: + selected.append(row[2:7]) + grouped.setdefault(row[1], []).append(row[2:7]) + view = _request_view(row) + details.append(view) + if snapshot_start <= stamp < snapshot_at and row[-1] <= snapshot_id: + snapshot_details.append(view if view is not None else _request_view(row)) + if stamp >= now - 86400: + bucket = int(stamp // BUCKET_SECONDS) * BUCKET_SECONDS + series_rows.setdefault(bucket, []).append(row) + if stamp >= midnight - 13 * 86400: + day = int((stamp + 8 * 3600) // 86400) * 86400 - 8 * 3600 + day_rows.setdefault(day, []).append(row[2:7]) + totals = _counts(selected) + seconds = max(0.0, now - start) + covered, dropped, unclean = _coverage(runs, start, now, now) + totals['window_tps'] = totals['output'] / seconds if seconds else None + totals['unknown_source'] = len(grouped.get(None, [])) + totals['covered_seconds'] = covered + totals['seconds'] = seconds + totals['dropped'] = dropped + totals['unclean_runs'] = unclean + overload_count, overload_tracked, overload_untracked = 0, 0, 0 + for left, seen, ended, value in overload_runs: + live = ended is None and 0 <= now - seen < 15 + right = ended if ended is not None else (now if live else seen) + if left < now and right > start: + try: + if value is None or not value.isascii() or not value.isdigit(): + raise ValueError() + overload_count += int(value) + overload_tracked += 1 + except (ValueError, TypeError, OverflowError): + overload_untracked += 1 + totals['overload_rejected'] = overload_count + totals['overload_tracked_runs'] = overload_tracked + totals['overload_untracked_runs'] = overload_untracked + classified = [item for item in details if item['state'] in ('known', 'partial', 'failed', 'interrupted')] + successes = [item for item in classified if item['state'] in ('known', 'partial') and item['status'] is not None + and 200 <= item['status'] < 300] + timed = [item for item in successes if item['first_visible'] is not None] + first_values = [item['first_visible'] for item in timed] + prefill_items = [item for item in timed if item['state'] == 'known' and item['first_visible'] > 0] + decode_items = [item for item in timed if item['output'] > 1 and item['after_first'] > 0] + prefill_time = sum(item['first_visible'] for item in prefill_items) + decode_time = sum(item['after_first'] for item in decode_items) + duration_values = [item['elapsed'] for item in successes] + totals.update(successes=len(successes), classified=len(classified), + success_rate=len(successes) / len(classified) if classified else None, + requests_per_second=len(details) / seconds if seconds else None, + first_p50=_percentile(first_values, .50), first_p95=_percentile(first_values, .95), + first_mean=sum(first_values) / len(first_values) if first_values else None, + first_samples=len(first_values), + prefill_estimate_tps=sum(item['input'] - item['cached'] for item in prefill_items) / prefill_time if prefill_time else None, + prefill_samples=len(prefill_items), + decode_estimate_tps=sum(item['output'] - 1 for item in decode_items) / decode_time if decode_time else None, + decode_samples=len(decode_items), elapsed_p50=_percentile(duration_values, .50), + elapsed_p95=_percentile(duration_values, .95), elapsed_samples=len(duration_values), + identified_sessions=len({item['session'] for item in details if item['session']}), + unknown_session=sum(1 for item in details if not item['session'])) + totals['complete'] = (bool(runs) and seconds > 0 and covered >= seconds - COVERAGE_EPSILON and dropped == 0 and unclean == 0 and + totals['partial'] == 0 and totals['missing'] == 0 and totals['interrupted'] == 0 and totals['failed'] == 0) + if source is not None: + # A recorder heartbeat does not establish that every public entry routes through it. + totals['complete'] = False + clients = [] + for ip, items in grouped.items(): + item = _counts(items) + item['ip'] = ip + item['window_tps'] = item['output'] / seconds if seconds else None + clients.append(item) + clients.sort(key=lambda x: (-x['output'], x['ip'] or '')) + first_bucket = int((now - 86400) // BUCKET_SECONDS) * BUCKET_SECONDS + last_bucket = int(now // BUCKET_SECONDS) * BUCKET_SECONDS + series = [] + for bucket in range(first_bucket, last_bucket + 1, BUCKET_SECONDS): + a, b = max(bucket, now - 86400), min(bucket + BUCKET_SECONDS, now) + bucket_rows = series_rows.get(bucket, []) + counts = _counts([row[2:7] for row in bucket_rows]) + bucket_requests = [_request_view(row) for row in bucket_rows] + hits = [item['hit_rate'] for item in bucket_requests if item['hit_rate'] is not None] + timed = [item for item in bucket_requests if item['state'] in ('known', 'partial') and + item['status'] is not None and 200 <= item['status'] < 300 and + item['first_visible'] is not None] + prefill = [item for item in timed if item['state'] == 'known' and item['first_visible'] > 0] + decode = [item for item in timed if item['output'] > 1 and item['after_first'] > 0] + prefill_tokens = sum(item['input'] - item['cached'] for item in prefill) + prefill_seconds = sum(item['first_visible'] for item in prefill) + decode_tokens = sum(item['output'] - 1 for item in decode) + decode_seconds = sum(item['after_first'] for item in decode) + coverage, lost, unclean = _coverage(runs, a, b, now) + counts.update(start=a, end=b, covered_seconds=coverage, dropped=lost, unclean_runs=unclean, + mean_request_hit_rate=sum(hits) / len(hits) if hits else None, + request_hit_sum=sum(hits), request_hit_samples=len(hits), + prefill_estimate_tps=prefill_tokens / prefill_seconds if prefill_seconds else None, + prefill_tokens=prefill_tokens, prefill_seconds=prefill_seconds, prefill_samples=len(prefill), + decode_estimate_tps=decode_tokens / decode_seconds if decode_seconds else None, + decode_tokens=decode_tokens, decode_seconds=decode_seconds, decode_samples=len(decode), + complete=bool(runs) and b > a and coverage >= b - a - COVERAGE_EPSILON and lost == 0 and unclean == 0 and + counts['partial'] == 0 and counts['missing'] == 0 and counts['interrupted'] == 0 and counts['failed'] == 0) + if source is not None: + counts['complete'] = False + series.append(counts) + days = [] + for day in range(13, -1, -1): + a, b = midnight - day * 86400, min(midnight - (day - 1) * 86400, now) + counts = _counts(day_rows.get(a, [])) + coverage, lost, unclean = _coverage(runs, a, b, now) + counts.update(date=datetime.fromtimestamp(a, TZ).strftime('%Y-%m-%d'), start=a, end=b, + covered_seconds=coverage, dropped=lost, unclean_runs=unclean, + complete=bool(runs) and b > a and coverage >= b - a - COVERAGE_EPSILON and lost == 0 and unclean == 0 and + counts['partial'] == 0 and counts['missing'] == 0 and counts['interrupted'] == 0 and counts['failed'] == 0) + if source is not None: + counts['complete'] = False + days.append(counts) + snapshot_covered, snapshot_dropped, snapshot_unclean = _coverage(runs, snapshot_start, snapshot_at, now) + snapshot_complete = (bool(runs) and snapshot_at > snapshot_start and + snapshot_covered >= snapshot_at - snapshot_start - COVERAGE_EPSILON and + snapshot_dropped == 0 and snapshot_unclean == 0 and + all(item['state'] == 'known' for item in snapshot_details)) + if source is not None: + snapshot_complete = False + sessions = _session_views(snapshot_details) + recent, request_offset = _ordered_page(snapshot_details, request_sort, request_dir, request_offset, request_limit, 'id') + session_page, session_offset = _ordered_page(sessions, session_sort, session_dir, session_offset, session_limit, 'session') + return {'schema': SCHEMA, 'as_of': now, 'since': since if all_runs else None, + 'window': window, 'collector_live': live_active['collector_live'], 'totals': totals, + 'clients': clients, 'series_24h': series, 'days': days, 'active': live_active['active'], + 'active_ready': live_active['active_ready'], 'active_sampled_at': live_active['active_sampled_at'], + 'snapshot_at': snapshot_at, 'snapshot_id': snapshot_id, 'snapshot_complete': snapshot_complete, + 'requests': recent, 'request_count': len(snapshot_details), 'request_offset': request_offset, 'request_limit': request_limit, + 'sessions': session_page, 'session_count': len(sessions), 'session_offset': session_offset, 'session_limit': session_limit, + 'session_unknown_count': sum(1 for item in snapshot_details if not item['session'])} diff --git a/deploy_stats/session.py b/deploy_stats/session.py new file mode 100644 index 0000000..aa45b4a --- /dev/null +++ b/deploy_stats/session.py @@ -0,0 +1,116 @@ +"""Derive the gateway's session fingerprint without persisting request bodies.""" +import hashlib +import json +import threading + +MAX_BODY = 1024 * 1024 +MAX_PENDING = 32 * 1024 * 1024 + + +class BodyBudget: + def __init__(self, limit=MAX_PENDING): + self.limit = limit + self.used = 0 + self.lock = threading.Lock() + + def reserve(self, size): + with self.lock: + if size < 0 or self.used + size > self.limit: + return False + self.used += size + return True + + def release(self, size): + with self.lock: + self.used -= size + + +BUDGET = BodyBudget() + + +def key_digest(headers): + auth = headers.get_all('Authorization', []) + if len(auth) != 1 or not isinstance(auth[0], str): + return None + fields = auth[0].split() + if len(fields) != 2 or fields[0].lower() != 'bearer' or len(fields[1]) > 256: + return None + try: + return hashlib.sha256(fields[1].encode()).hexdigest() + except UnicodeError: + return None + + +def fingerprint(digest, seed): + try: + return hashlib.sha256((digest + '\0' + seed).encode()).hexdigest() + except UnicodeError: + return None + + +def explicit_session(headers): + """Return whether an explicit header exists and its fingerprint if valid.""" + digest = key_digest(headers) + for name in ('x-session-affinity', 'x-deepseek-harness-session-id', 'x-session-id'): + values = headers.get_all(name, []) + if values: + if not digest or len(values) != 1 or not isinstance(values[0], str) or not 0 < len(values[0]) <= 256: + return True, None + return True, fingerprint(digest, 'session:' + values[0]) + return False, None + + +class SessionBodyCapture: + """Bounded temporary buffering for the gateway's JSON fallback identity.""" + + def __init__(self, digest, *, budget=BUDGET, max_body=MAX_BODY): + self.digest = digest + self.budget = budget + self.max_body = max_body + self.buffer = bytearray() + self.reserved = 0 + self.disabled = not bool(digest) + + def feed(self, data): + if self.disabled: + return + if len(self.buffer) + len(data) > self.max_body or not self.budget.reserve(len(data)): + self.close() + return + self.reserved += len(data) + try: + self.buffer.extend(data) + except Exception: + self.close() + + def finish(self): + if self.disabled: + return None + try: + value = json.loads(self.buffer) + if not isinstance(value, dict): + return None + cache_key = value.get('prompt_cache_key') + if isinstance(cache_key, str) and cache_key: + seed = 'session:' + cache_key + elif isinstance(value.get('messages'), list): + first = next((item for item in value['messages'] + if isinstance(item, dict) and item.get('role') == 'user'), None) + if first is None or not isinstance(first.get('content'), (str, list)): + return None + seed = 'first-user:' + json.dumps(first['content'], sort_keys=True, ensure_ascii=False, + separators=(',', ':')) + else: + return None + return fingerprint(self.digest, seed) + except (ValueError, TypeError, UnicodeError, RecursionError, MemoryError): + return None + finally: + self.close() + + def close(self): + if self.reserved: + self.budget.release(self.reserved) + self.reserved = 0 + self.buffer.clear() + self.disabled = True diff --git a/deploy_stats/static/app.css b/deploy_stats/static/app.css new file mode 100644 index 0000000..48eacad --- /dev/null +++ b/deploy_stats/static/app.css @@ -0,0 +1,16 @@ +:root{font-family:Inter,"Segoe UI","Microsoft YaHei",sans-serif;color:#1a2841;background:#f5f7fb;font-synthesis:none}*{box-sizing:border-box}body{margin:0}.app-shell{min-height:100vh;display:grid;grid-template-columns:216px minmax(0,1fr)}.sidebar{background:#17243c;color:#c4cde0;display:flex;flex-direction:column;padding:26px 14px 20px}.brand{display:flex;align-items:center;gap:10px;padding:0 8px 40px}.brand-mark{width:36px;height:36px;display:grid;place-items:center;color:#e7fbf8;background:#2d8f89;border-radius:10px;font-size:23px}.brand strong{display:block;color:#fff;font-size:16px;letter-spacing:.2px}.brand small,.sidebar-label{display:block;font-size:9px;letter-spacing:1.45px;color:#8a9bb6;margin-top:4px}.sidebar-label{padding:0 11px;margin-bottom:12px}.sidebar nav{display:grid;gap:5px}.sidebar nav a{display:flex;gap:12px;align-items:center;color:#b5c2d8;font-size:13px;font-weight:600;text-decoration:none;padding:11px 12px;border-radius:7px}.sidebar nav a span{font-size:17px;width:18px;text-align:center}.sidebar nav a:hover,.sidebar nav .nav-active{color:#fff;background:#293a55}.sidebar-bottom{display:flex;align-items:center;gap:10px;margin-top:auto;padding:14px 8px;border-top:1px solid #33435a}.sidebar-bottom .shield{font-size:22px;color:#73c8b8}.sidebar-bottom strong,.sidebar-bottom small{display:block;font-size:12px}.sidebar-bottom small{font-size:10px;color:#8fa0b9;margin-top:4px}.topbar{height:62px;background:#fff;border-bottom:1px solid #e7ebf2;display:flex;align-items:center;justify-content:space-between;padding:0 42px}.breadcrumb{font-size:12px;color:#8b97a9}.breadcrumb span{padding:0 10px;color:#ccd3df}.top-right{display:flex;align-items:center;gap:16px}.local-pill,.outline-pill{font-size:10px;font-weight:700;letter-spacing:.7px;border:1px solid #e1e7ef;border-radius:6px;padding:7px 9px;color:#8290a2;white-space:nowrap}.local-pill i{display:inline-block;width:6px;height:6px;border-radius:50%;background:#2aab87;margin-right:5px}.refresh{border:1px solid #e1e7ef;background:#fff;color:#586b85;width:30px;height:30px;border-radius:7px;font-size:19px;cursor:pointer}.refresh:hover{background:#f5f8fb}.content{max-width:1680px;margin:0 auto;padding:32px 42px 18px}.page-title{display:flex;align-items:center;justify-content:space-between;gap:20px;margin-bottom:34px}.eyebrow,.section-kicker{font-size:9px;font-weight:800;letter-spacing:1.55px;color:#8190a4}.page-title h1{font-size:26px;letter-spacing:-.4px;margin:8px 0 7px;color:#17253d}.page-title p,.panel-head p{font-size:12px;color:#8a97aa;margin:0}.status{display:flex;gap:10px;align-items:center;border:1px solid #e0e8ee;background:white;border-radius:9px;padding:10px 13px;min-width:160px}.status strong,.status small{display:block}.status strong{font-size:11px;color:#516378}.status small{font-size:10px;color:#9aa6b7;margin-top:3px}.status-dot{width:8px;height:8px;border-radius:50%;background:#e2ad50;box-shadow:0 0 0 4px #fbf2df}.status-dot.live{background:#36ad8e;box-shadow:0 0 0 4px #e5f6f0}.toolbar{display:flex;justify-content:space-between;align-items:end;gap:15px;margin-bottom:15px}.toolbar h2,.panel-head h2{font-size:16px;margin:5px 0 0;letter-spacing:-.1px}.segmented{background:#e9eef5;padding:3px;border-radius:8px;display:flex;gap:2px}.segmented button{border:0;background:transparent;padding:7px 12px;font-size:11px;font-weight:600;color:#8090a5;border-radius:6px;cursor:pointer;white-space:nowrap}.segmented button.is-active{background:white;color:#293c57;box-shadow:0 1px 4px #233a5a19}.notice{border:1px solid #d8e8ef;background:#eff7f9;color:#4a6e7a;border-radius:7px;padding:10px 13px;font-size:11px;line-height:1.6;margin-bottom:14px}.notice.warning{background:#fff9ed;border-color:#f5e6c4;color:#8a6a2e}.cards{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px;margin-bottom:17px}.metric,.panel{background:white;border:1px solid #e6ebf1;border-radius:10px;box-shadow:0 3px 12px #2c476407}.metric{min-height:120px;padding:16px 18px}.metric-top{display:flex;align-items:center;justify-content:space-between;color:#708099;font-size:12px;font-weight:600}.metric-icon{width:28px;height:28px;display:grid;place-items:center;border-radius:8px;font-size:16px}.blue{color:#477bdb;background:#edf3ff}.violet{color:#8562cd;background:#f2edfb}.green{color:#299c89;background:#e6f6f2}.orange{color:#d79243;background:#fff3e3}.metric strong{display:block;font-size:24px;letter-spacing:-.7px;color:#1a2c45;margin:9px 0 5px}.metric small{color:#96a2b2;font-size:10px}.panel{margin-bottom:17px;min-width:0}.panel-head{display:flex;justify-content:space-between;align-items:flex-start;gap:15px;padding:20px 21px 15px}.panel-head p{margin-top:7px;line-height:1.5}.chart-lanes{padding:0 21px}.lane{display:grid;grid-template-columns:100px minmax(0,1fr);align-items:center;border-top:1px solid #f0f3f7;min-height:72px}.lane-label{display:grid;grid-template-columns:8px 1fr;align-items:center;gap:5px 7px;color:#53647d;font-size:11px;font-weight:700}.lane-label small{grid-column:2;font-size:10px;color:#96a3b3;font-weight:500}.dot{width:7px;height:7px;border-radius:50%;display:inline-block}.dot.input{background:#5388e7}.dot.output{background:#9c77df}.dot.cached{background:#31aa99}.lane svg{width:100%;height:68px;overflow:visible}.lane svg .grid{stroke:#eff2f7;stroke-width:1}.lane svg .input{stroke:#5388e7;fill:none;stroke-width:2.2;vector-effect:non-scaling-stroke}.lane svg .output{stroke:#9c77df;fill:none;stroke-width:2.2;vector-effect:non-scaling-stroke}.lane svg .cached{stroke:#31aa99;fill:none;stroke-width:2.2;vector-effect:non-scaling-stroke}.lane svg circle.input{fill:#5388e7}.lane svg circle.output{fill:#9c77df}.lane svg circle.cached{fill:#31aa99}.chart-axis{display:flex;justify-content:space-between;color:#a7b1bf;font-size:10px;padding:5px 21px 13px 121px}.panel-foot{border-top:1px solid #f0f3f7;padding:11px 21px;color:#9aa6b6;font-size:10px;line-height:1.5}.lower-grid{display:grid;grid-template-columns:minmax(0,1.4fr) minmax(370px,1fr);gap:17px}.table-wrap{overflow-x:auto}.table-panel table{width:100%;border-collapse:collapse;text-align:right;white-space:nowrap}.table-panel th{padding:10px 12px;color:#9ba8b9;background:#f8fafc;font-size:10px;font-weight:600}.table-panel td{padding:12px;color:#53657e;font-size:11px;border-bottom:1px solid #f1f4f7}.table-panel th:first-child,.table-panel td:first-child{text-align:left;padding-left:21px}.table-panel td:first-child{font-weight:650;color:#293c57}.table-panel th:last-child,.table-panel td:last-child{padding-right:21px}.table-panel .empty{text-align:center!important;color:#a0acba!important;font-weight:400!important;padding:28px!important}.daily .table-wrap{max-height:340px;overflow:auto}footer{text-align:center;color:#aab4c2;font-size:10px;padding:7px 0 15px}button:focus-visible,a:focus-visible{outline:2px solid #3a8cca;outline-offset:2px}@media(max-width:1100px){.lower-grid{grid-template-columns:1fr}.daily .table-wrap{max-height:300px}}@media(max-width:850px){.app-shell{grid-template-columns:1fr}.sidebar{display:none}.topbar{padding:0 20px}.content{padding:26px 20px}.cards{grid-template-columns:repeat(2,minmax(0,1fr))}}@media(max-width:520px){.topbar{height:54px}.page-title{align-items:flex-start;flex-direction:column;margin-bottom:24px}.status{min-width:0}.toolbar{align-items:flex-start;flex-direction:column}.segmented{width:100%}.segmented button{flex:1;padding:8px 4px}.metric{padding:14px;min-height:110px}.metric strong{font-size:20px}.panel-head{padding:18px 15px 13px}.chart-lanes{padding:0 15px}.lane{grid-template-columns:72px minmax(0,1fr)}.chart-axis{padding-left:87px;padding-right:15px}.outline-pill{font-size:9px}} +.cards{grid-template-columns:repeat(5,minmax(0,1fr))}.slate{color:#56738f;background:#edf2f7}@media(max-width:1100px){.cards{grid-template-columns:repeat(3,minmax(0,1fr))}}@media(max-width:850px){.cards{grid-template-columns:repeat(2,minmax(0,1fr))}} +.pager{display:flex;align-items:center;justify-content:flex-end;gap:10px;margin-top:12px}.pager button{border:1px solid #dfe7ef;background:#fff;border-radius:6px;padding:5px 9px;color:#51667d;cursor:pointer}.pager button:disabled{opacity:.45;cursor:default}.pager span{white-space:nowrap}.service-cards .metric{min-height:110px}.table-panel td{font-variant-numeric:tabular-nums} +.app-shell>main{min-width:0}@media(max-width:850px){.app-shell{grid-template-columns:minmax(0,1fr)}} +.table-panel th[aria-sort]{cursor:pointer}.sort-control{border:0;background:transparent;color:inherit;font:inherit;padding:0;white-space:nowrap;cursor:pointer}.sort-control:after{content:' ↕';color:#aab7c8}.table-panel th[aria-sort=ascending] .sort-control:after{content:' ↑';color:#4978b9}.table-panel th[aria-sort=descending] .sort-control:after{content:' ↓';color:#4978b9}.pager label{display:flex;align-items:center;gap:5px;white-space:nowrap}.pager select{border:1px solid #dfe7ef;border-radius:6px;background:#fff;color:#51667d;padding:4px 6px;font:inherit}.pager select:focus-visible{outline:2px solid #3a8cca;outline-offset:2px}@media(max-width:520px){.pager{flex-wrap:wrap;gap:8px}} +[data-view][hidden],.segmented[hidden]{display:none!important}.lower-grid{display:block}.sidebar{position:sticky;top:0;align-self:start;height:100vh;overflow-y:auto}@media(max-width:850px){.sidebar{display:block;position:sticky;top:0;z-index:10;height:auto;overflow-y:visible;padding:6px 10px}.sidebar .brand,.sidebar-label,.sidebar-bottom{display:none}.sidebar nav{display:flex;gap:4px;overflow-x:auto}.sidebar nav a{flex:0 0 auto;white-space:nowrap;padding:8px 10px}} +.dot.prefill{background:#d99036}.dot.decode{background:#c26589}.lane svg .prefill{stroke:#d99036;fill:none;stroke-width:2.2;vector-effect:non-scaling-stroke}.lane svg .decode{stroke:#c26589;fill:none;stroke-width:2.2;vector-effect:non-scaling-stroke}.lane svg circle.prefill{fill:#d99036}.lane svg circle.decode{fill:#c26589}.lane-label small{overflow-wrap:anywhere} +.lane{grid-template-columns:155px minmax(0,1fr)}.chart-axis{padding-left:176px}@media(max-width:520px){.lane{grid-template-columns:125px minmax(0,1fr)}.chart-axis{padding-left:140px}} +.lane svg .partial{opacity:.58;stroke-dasharray:4 3}.lane svg circle.partial{fill:#fff} +.pager span{white-space:normal}.snapshot-hint{display:block;margin-top:6px} +.cards.phase-cards{grid-template-columns:repeat(4,minmax(0,1fr))}.phase-cards .metric{min-width:0;padding:14px}.phase-cards .metric-top{font-size:11px}.phase-cards .metric strong{font-size:18px;overflow-wrap:anywhere}.phase-note{font-size:11px;line-height:1.5;color:#8796a9;margin:-7px 0 18px}@media(max-width:520px){.cards.phase-cards{grid-template-columns:repeat(2,minmax(0,1fr))}} +.phase-heading{margin:2px 0 8px;color:#8190a4;font-size:10px;font-weight:800;letter-spacing:1.25px}.cards.phase-cards.decode-cards{grid-template-columns:repeat(3,minmax(0,1fr))}@media(max-width:850px){.cards.phase-cards.prefill-cards,.cards.phase-cards.decode-cards{grid-template-columns:repeat(2,minmax(0,1fr))}} +.telemetry-scope{display:flex;align-items:center;gap:10px;margin:0 0 14px;color:#8190a4;font-size:12px;flex-wrap:wrap}.telemetry-scope label{font-weight:700;color:#475569}.telemetry-scope select{border:1px solid #dce3ec;border-radius:7px;background:#fff;padding:6px 9px;color:#253449;font:inherit} +.phase-cards .metric small{color:#657891;line-height:1.4} +.trend-tooltip{position:fixed;z-index:50;max-width:min(320px,calc(100vw - 16px));padding:8px 10px;background:#17243c;color:#fff;border-radius:6px;box-shadow:0 5px 18px #17243c35;font-size:12px;line-height:1.5;white-space:pre-line;pointer-events:none}.trend-tooltip[hidden]{display:none}.trend-hover-overlay{cursor:crosshair} +.cards.phase-cards.decode-cards{grid-template-columns:repeat(4,minmax(0,1fr))}.dot.mtp{background:#4f7de3}.lane svg .mtp{stroke:#4f7de3;fill:none;stroke-width:2.2;vector-effect:non-scaling-stroke}.lane svg circle.mtp{fill:#4f7de3}.lane svg circle.mtp.partial{fill:#fff}@media(max-width:850px){.cards.phase-cards.decode-cards{grid-template-columns:repeat(2,minmax(0,1fr))}} diff --git a/deploy_stats/static/app.js b/deploy_stats/static/app.js new file mode 100644 index 0000000..d8a55a5 --- /dev/null +++ b/deploy_stats/static/app.js @@ -0,0 +1,1118 @@ +'use strict'; + +const $ = (id) => document.getElementById(id); +const number = new Intl.NumberFormat('zh-CN', {maximumFractionDigits: 2}); +const integer = new Intl.NumberFormat('zh-CN'); +const clock = new Intl.DateTimeFormat('zh-CN', {timeZone: 'Asia/Shanghai', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false}); +const dateTime = new Intl.DateTimeFormat('zh-CN', {timeZone: 'Asia/Shanghai', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false}); +const snapshotTime = new Intl.DateTimeFormat('zh-CN', {timeZone: 'Asia/Shanghai', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false}); +let selected = '24h'; +let revision = 0; +let lastSnapshot = null; +let pendingRefresh = null; +let activeExpiry = null; +let pendingActive = null; +let pendingTelemetry = null; +let telemetrySnapshot = null; +let telemetryExpiry = null; +let backendExpiry = null; +let telemetryNodePort = 'all'; +let telemetryOptionsKey = null; +let latestActiveAsOf = 0; +let currentView = 'overview'; +const tables = { + active: {selector: '#active', sort: 'elapsed', direction: 'desc', columns: ['ip', 'session', 'elapsed', 'phase', 'input', 'output', 'hit_rate', 'tps']}, + backends: {selector: '#backends', sort: 'port', direction: 'asc', columns: ['port', 'running', 'waiting', 'workers_alive', 'admission_ready', 'input_total', 'hit_rate'], noPager: true}, + sessions: {selector: '#sessions', sort: 'last_finished', direction: 'desc', columns: ['session', 'ips', 'requests', 'input', 'output', 'cached', 'uncached_input', 'hit_rate', 'first_visible', 'after_first', 'input_per_wait', 'output_per_after_first', 'first_share']}, + requests: {selector: '[aria-labelledby="sessions-heading"]', sort: 'finished', direction: 'desc', columns: ['finished', 'ip', 'session', 'state', 'input', 'output', 'cached', 'uncached_input', 'hit_rate', 'first_visible', 'after_first', 'input_per_wait', 'output_per_after_first', 'first_share']}, + clients: {selector: '#clients', sort: 'output', direction: 'desc', columns: ['ip', 'requests', 'total', 'input', 'output', 'cached', 'request_tps', 'window_tps']}, + days: {selector: '#daily', sort: 'date', direction: 'desc', columns: ['date', 'input', 'output', 'cached']}, +}; +const textSorts = new Set(['ip', 'ips', 'session', 'state', 'date']); +Object.values(tables).forEach((table) => { table.offset = 0; table.size = 20; }); +const views = { + overview: ['总览', 'OVERVIEW'], active: ['当前连接', 'LIVE REQUESTS'], + sessions: ['会话记录', 'SESSIONS'], clients: ['请求来源', 'CLIENTS'], + daily: ['每日用量', 'DAILY'], backends: ['后端状态', 'BACKENDS'], +}; +const seconds = (value) => value === null || value === undefined ? '—' : `${number.format(value)} s`; +const rate = (value) => value === null || value === undefined ? '—' : number.format(value); +const speed = (value) => value === null || value === undefined ? '—' : `${rate(value)} tok/s`; +const percent = (value) => value === null || value === undefined ? '—' : `${number.format(value * 100)}%`; +const count = (value) => value === null || value === undefined ? '—' : integer.format(value); +const fingerprint = (value) => value ? value.slice(0, 12) : '未知'; +const setTextIfPresent = (id, value) => { const element = $(id); if (element) element.textContent = value; }; + +function freshTelemetry(part, now) { + return part && typeof part.valid_until === 'number' && Number.isFinite(part.valid_until) && now < part.valid_until ? part : null; +} + +function validMtp(part, now) { + if (!freshTelemetry(part, now) || typeof part.sampled_at !== 'number' || !Number.isFinite(part.sampled_at) || + part.sampled_at > now || !['steps', 'drafted', 'accepted', 'valid_endpoints', 'expected_endpoints'].every( + (key) => Number.isSafeInteger(part[key]) && part[key] >= 0) || + part.accepted > part.drafted || part.valid_endpoints < 1 || part.valid_endpoints > part.expected_endpoints) return null; + return part; +} + +function mtpSummary(data, nodes, port, now, kind) { + if (!data || data.available !== true || data.kind !== 'vllm-motor-telemetry') return null; + const groups = nodes.filter((node) => node && node.role === 'decode' && (port === 'all' || node.backend_port === port)); + if (!groups.length || new Set(groups.map((node) => node.backend_port)).size !== groups.length) return null; + const parts = groups.flatMap((node) => { + const part = node.source_ok === true && node.backend_ready === true && node.mtp && validMtp(node.mtp[kind], now); + return part ? [part] : []; + }); + if (!parts.length) return null; + const steps = parts.reduce((sum, part) => sum + part.steps, 0); + const drafted = parts.reduce((sum, part) => sum + part.drafted, 0); + const accepted = parts.reduce((sum, part) => sum + part.accepted, 0); + if (![steps, drafted, accepted].every(Number.isSafeInteger) || accepted > drafted) return null; + return {steps, drafted, accepted, rate: drafted ? accepted / drafted : null, + groups: parts.length, group_total: groups.length, + workers: parts.reduce((sum, part) => sum + part.valid_endpoints, 0), + workers_total: groups.reduce((sum, node) => sum + (node.expected_endpoints || 0), 0), + partial: parts.length < groups.length || parts.some((part) => part.partial === true)}; +} + +function renderMtp(data, nodes, port, now) { + const lifetime = mtpSummary(data, nodes, port, now, 'lifetime'); + const interval = mtpSummary(data, nodes, port, now, 'interval'); + setTextIfPresent('mtp-acceptance', lifetime ? percent(lifetime.rate) : '—'); + const coverage = lifetime ? `${lifetime.groups}/${lifetime.group_total} 注册组 · ${lifetime.workers}/${lifetime.workers_total} 端点${lifetime.partial ? ' · 部分数据' : ''}` : '所选范围暂无推测解码指标'; + const recent = interval && interval.drafted ? ` · 最近采样 ${percent(interval.rate)}（${interval.groups}/${interval.group_total} 组，${interval.workers}/${interval.workers_total} 端点${interval.partial ? '，部分数据' : ''}）` : ' · 最近采样无有效草稿'; + setTextIfPresent('mtp-detail', `${coverage}${lifetime ? recent : ''}`); + renderMtpHistory(data, nodes, port); +} + +function telemetryValues(data, port, now) { + const nodes = data && data.available === true && Array.isArray(data.nodes) ? data.nodes : []; + const read = (value, max = Infinity) => typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= max ? value : null; + const motor = data && data.kind === 'vllm-motor-telemetry'; + if (port === 'all') { + const coverage = Object.fromEntries(['prefill', 'decode', 'mfu', 'hbm'].map((key) => + [key, {valid: 0, total: nodes.filter((node) => node && (!motor || key === 'hbm' || + node.role === (key === 'mfu' ? 'prefill' : key))).length, span_seconds: null, + workers_valid: 0, workers_total: nodes.filter((node) => node && (!motor || key === 'hbm' || + node.role === (key === 'mfu' ? 'prefill' : key))).reduce((sum, node) => sum + (node.expected_endpoints || 0), 0), partial: false}])); + const empty = {prefill: null, decode: null, mfu: null, hbm: null, coverage}; + if (!nodes.length || new Set(nodes.map((node) => node && node.backend_port)).size !== nodes.length) return empty; + const parts = (name, timestamp, duration, requireReady = true) => { + return nodes.flatMap((node) => { + if (motor && name !== 'hbm' && node && node.role !== name) return []; + if (!node || node.source_ok !== true || (requireReady && node.backend_ready !== true) || node.clock_ok !== true || + read(node.clock_uncertainty_seconds, 20) === null) return []; + const part = freshTelemetry(node[name], now); + if (!part || read(part[timestamp]) === null || read(part[duration], 60) === null) return []; + return [{part, first: part[timestamp] - part[duration] - node.clock_uncertainty_seconds, + last: part[timestamp] + node.clock_uncertainty_seconds}]; + }); + }; + const prefill = parts('prefill', 'log_at', 'log_skew_seconds'); + const decode = parts('decode', 'log_at', 'log_skew_seconds'); + const hbm = parts('hbm', 'sample_at', 'span_seconds', false); + const recordCoverage = (key, values) => { + coverage[key].valid = values.length; + coverage[key].workers_valid = values.reduce((sum, {part}) => sum + (part.valid_endpoints || 0), 0); + coverage[key].partial = values.some(({part}) => part.partial === true); + if (values.length) coverage[key].span_seconds = Math.max(...values.map((x) => x.last)) - Math.min(...values.map((x) => x.first)); + }; + const sumRate = (key, values) => { + const valid = values.filter(({part}) => read(part.tokens_per_second) !== null); + recordCoverage(key, valid); + const total = valid.reduce((sum, {part}) => sum + part.tokens_per_second, 0); + return valid.length && Number.isFinite(total) ? total : null; + }; + const weighted = (key, values, value, weight, limit = Infinity) => { + const valid = values.filter(({part}) => read(part[value], limit) !== null && read(part[weight]) > 0 && + (weight !== 'device_count' || Number.isSafeInteger(part[weight]))); + recordCoverage(key, valid); + const denominator = valid.reduce((sum, {part}) => sum + part[weight], 0); + const numerator = valid.reduce((sum, {part}) => sum + part[value] * part[weight], 0); + return denominator > 0 && Number.isFinite(denominator) && Number.isFinite(numerator) ? numerator / denominator : null; + }; + return {prefill: sumRate('prefill', prefill), decode: sumRate('decode', decode), + mfu: weighted('mfu', prefill, 'mfu_estimate', 'peak_flops'), + hbm: weighted('hbm', hbm, 'mean_percent', 'device_count', 100), coverage}; + } + const item = nodes.find((node) => node && node.backend_port === port && node.source_ok === true); + const usable = item && item.clock_ok === true && read(item.clock_uncertainty_seconds, 20) !== null; + const engineReady = usable && item.backend_ready === true; + const prefill = engineReady && freshTelemetry(item.prefill, now); + const decode = engineReady && freshTelemetry(item.decode, now); + const hbm = usable && freshTelemetry(item.hbm, now); + return {prefill: read(prefill && prefill.tokens_per_second), decode: read(decode && decode.tokens_per_second), + mfu: read(prefill && prefill.mfu_estimate), hbm: read(hbm && hbm.mean_percent, 100)}; +} + +function latestHistoricalSample(history, phase, now, bucketStart = null) { + if (!history || !['prefill', 'decode'].includes(phase) || !Number.isFinite(now) || + !['deepinfer-log-step-approx-v1', 'vllm-counter-rate-v1'].includes(history.method) || history.bucket_seconds !== 600 || + !Array.isArray(history.points) || history.points.length !== 144) return null; + const points = history.points; + if (points.some((point, i) => !point || !Number.isSafeInteger(point.start) || point.start < 0 || + point.end !== point.start + 600 || point.end > now || (i && point.start !== points[i - 1].start + 600))) return null; + if (history.aggregate && (!Number.isSafeInteger(history.group_count) || history.group_count <= 0)) return null; + for (let i = points.length - 1; i >= 0; i--) { + const point = points[i], value = point[`${phase}_tps`], samples = point[`${phase}_samples`]; + if (bucketStart !== null && point.start !== bucketStart) continue; + const groups = history.aggregate ? point[`${phase}_groups`] : 1; + const total = history.aggregate ? (history[`${phase}_group_count`] ?? history.group_count) : 1; + if (typeof value === 'number' && Number.isFinite(value) && value >= 0 && + Number.isSafeInteger(samples) && samples > 0 && Number.isSafeInteger(groups) && groups > 0 && groups <= total) { + return {value, samples, start: point.start, end: point.end, groups, total, partial: point[`${phase}_partial`] === true}; + } + } + return null; +} + +function latestGroupHistory(data, nodes, port, phase, now) { + if (!data || data.available !== true || data.history_available !== true || !Array.isArray(nodes) || !nodes.length || + new Set(nodes.map((node) => node && node.backend_port)).size !== nodes.length) return null; + const selected = (port === 'all' ? nodes : nodes.filter((node) => node && node.backend_port === port)) + .filter((node) => data.kind !== 'vllm-motor-telemetry' || node.role === phase); + if (!selected.length || (port !== 'all' && selected.length !== 1)) return null; + const candidates = selected.flatMap((node) => { + if (!node || node.source_ok !== true || !node.history || node.history.aggregate) return []; + const value = latestHistoricalSample(node.history, phase, now); + return value ? [value] : []; + }); + if (!candidates.length) return null; + const newest = Math.max(...candidates.map((value) => value.start)); + const matching = candidates.filter((value) => value.start === newest); + const value = matching.reduce((sum, point) => sum + point.value, 0); + if (!Number.isFinite(value)) return null; + return {value, start: newest, end: newest + 600, groups: matching.length, + total: selected.length, samples: port === 'all' ? 1 : matching[0].samples, + partial: matching.some((point) => point.partial)}; +} + +function historicalPrefillMfu(nodes, port, sample, now) { + if (!sample || !Array.isArray(nodes)) return null; + let work = 0, peak = 0, groups = 0; + nodes.forEach((node) => { + if (!node || (port !== 'all' && node.backend_port !== port) || node.source_ok !== true || + node.clock_ok !== true || typeof node.clock_uncertainty_seconds !== 'number' || + !Number.isFinite(node.clock_uncertainty_seconds) || node.clock_uncertainty_seconds < 0 || + node.clock_uncertainty_seconds > 20) return; + const point = latestHistoricalSample(node.history, 'prefill', now, sample.start); + const hbm = freshTelemetry(node.hbm, now), assumptions = node.assumptions; + if (!point || point.end !== sample.end || !hbm || !Number.isSafeInteger(hbm.device_count) || + hbm.device_count < 1 || hbm.device_count > 64 || !assumptions || + typeof assumptions.prefill_flops_per_token !== 'number' || !Number.isFinite(assumptions.prefill_flops_per_token) || + assumptions.prefill_flops_per_token <= 0 || typeof assumptions.peak_flops_per_device !== 'number' || + !Number.isFinite(assumptions.peak_flops_per_device) || assumptions.peak_flops_per_device <= 0) return; + const numerator = point.value * assumptions.prefill_flops_per_token; + const denominator = hbm.device_count * assumptions.peak_flops_per_device; + if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator <= 0) return; + work += numerator; peak += denominator; groups++; + }); + const value = work / peak; + return groups && Number.isFinite(value) && value >= 0 ? {...sample, value, groups} : null; +} + +function renderTelemetry(data) { + telemetrySnapshot = data; + if (telemetryExpiry) clearTimeout(telemetryExpiry); + telemetryExpiry = null; + const now = Date.now() / 1000; + const motor = data && data.kind === 'vllm-motor-telemetry'; + const nodes = data && data.available === true && Array.isArray(data.nodes) ? data.nodes.filter((item) => item && typeof item === 'object') : []; + if (telemetryNodePort !== 'all' && !nodes.some((item) => item.backend_port === telemetryNodePort)) telemetryNodePort = 'all'; + const select = $('telemetry-node'); + if (select) { + const key = JSON.stringify(nodes.map((item) => [item.backend_port, item.label])); + if (key !== telemetryOptionsKey) { + select.replaceChildren(); + const all = document.createElement('option'); all.value = 'all'; all.textContent = '跨设备组（全部）'; select.appendChild(all); + nodes.forEach((item) => { + const option = document.createElement('option'); option.value = String(item.backend_port); + option.textContent = item.label || '未知实例'; select.appendChild(option); + }); + telemetryOptionsKey = key; + } + select.value = String(telemetryNodePort); + } + const values = telemetryValues(data, telemetryNodePort, now); + renderMtp(data, nodes, telemetryNodePort, now); + const selectedNode = telemetryNodePort === 'all' ? combinedDeviceHistory(data, nodes) : nodes.find((item) => item.backend_port === telemetryNodePort); + renderDeviceHistory(data, selectedNode); + const prefillSample = selectedNode && freshTelemetry(selectedNode.prefill, now); + const decodeSample = selectedNode && freshTelemetry(selectedNode.decode, now); + const measured = (value) => value === null || value === undefined ? '—' : speed(value); + const showMfu = (value) => value !== null && value !== undefined && value > 0 && value < .0001 ? '<0.01%' : percent(value); + const previousPrefill = latestGroupHistory(data, nodes, telemetryNodePort, 'prefill', now); + const previousDecode = latestGroupHistory(data, nodes, telemetryNodePort, 'decode', now); + const previousMfu = historicalPrefillMfu(nodes, telemetryNodePort, previousPrefill, now); + // A browser can receive old HTML and new JS across a local restart. Never + // place a historical value in the primary position unless both its title + // and its non-real-time detail can be shown by that HTML version. + const usePrefillHistory = values.prefill === null && previousPrefill && $('live-prefill-title') && $('live-prefill-samples'); + const useDecodeHistory = values.decode === null && previousDecode && $('live-decode-title') && $('live-decode-samples'); + const useMfuHistory = values.mfu === null && previousMfu && $('live-mfu-title') && $('live-mfu-detail'); + setTextIfPresent('live-prefill-title', usePrefillHistory ? 'Prefill 并发吞吐（历史均值）' : 'Prefill 并发吞吐（采样）'); + setTextIfPresent('live-decode-title', useDecodeHistory ? 'Decode 并发吞吐（历史均值）' : 'Decode 并发吞吐（采样）'); + setTextIfPresent('live-mfu-title', useMfuHistory ? 'Prefill-MFU（历史估算）' : 'Prefill-MFU（单步估算）'); + setTextIfPresent('live-prefill-tps', measured(usePrefillHistory ? previousPrefill.value : values.prefill)); + setTextIfPresent('live-decode-tps', measured(useDecodeHistory ? previousDecode.value : values.decode)); + setTextIfPresent('live-mfu', showMfu(useMfuHistory ? previousMfu.value : values.mfu)); + setTextIfPresent('live-mbu', '—'); + setTextIfPresent('live-hbm', values.hbm === null ? '—' : `${number.format(values.hbm)}%`); + const historicalDetail = (sample, method) => { + if (!sample) return ''; + const period = `${dateTime.format(new Date(sample.start * 1000))}–${clock.format(new Date(sample.end * 1000))}`; + const scope = `${sample.groups}/${sample.total} 组` + + (telemetryNodePort === 'all' ? '' : ` · ${count(sample.samples)} 个采样`); + return `${method} · ${period} · 非实时 · ${scope}${sample.partial ? ' · 部分数据' : ''}`; + }; + const groupDetail = (key, method) => { + const item = values.coverage[key]; + if (!item.total) return '采集未连接 · 组数未知'; + if (!item.valid) return `0/${item.total} 组有样本 · 暂无有效数据`; + const partial = item.valid < item.total || item.partial ? ' · 部分数据' : ''; + const span = item.span_seconds > 30 ? ` · 时间跨度约 ${number.format(item.span_seconds)} s` : ''; + const workers = motor && (key === 'prefill' || key === 'decode') ? ` · ${item.workers_valid}/${item.workers_total} worker` : ''; + return `${item.valid}/${item.total} 组有样本${partial}${workers} · ${method}${span}`; + }; + const detail = (key, value, sample) => telemetryNodePort === 'all' ? groupDetail(key, motor ? '计数差 / 间隔' : '采样和') : + value === null ? '所选设备组暂无有效样本' : motor ? + `${sample.valid_endpoints}/${sample.expected_endpoints} worker · 计数差 / 间隔${sample.partial ? ' · 部分数据' : ''}` : + `日志 ${clock.format(new Date(sample.log_at * 1000))} 写出 · rank 近似和`; + setTextIfPresent('live-prefill-samples', usePrefillHistory ? historicalDetail(previousPrefill, '历史采样均值') : + detail('prefill', values.prefill, prefillSample)); + setTextIfPresent('live-decode-samples', useDecodeHistory ? historicalDetail(previousDecode, '历史采样均值') : + detail('decode', values.decode, decodeSample)); + setTextIfPresent('live-mfu-detail', useMfuHistory ? historicalDetail(previousMfu, '按当前假设估算') : + telemetryNodePort === 'all' ? groupDetail('mfu', '按峰值算力加权') : + values.mfu === null ? '所选设备组暂无有效样本' : '引擎吞吐与模型假设换算'); + setTextIfPresent('live-hbm-detail', telemetryNodePort === 'all' ? groupDetail('hbm', '按卡数加权') : + values.hbm === null ? '所选设备组暂无有效样本' : '所选设备组驱动读数的平均'); + const topology = motor && data.topology_sources ? ` · 管理源 ${data.topology_sources.available}/${data.topology_sources.total}${data.topology_ambiguous ? ' · 拓扑待确认' : ''}` : ''; + setTextIfPresent('telemetry-status', nodes.length ? `最近读取于 ${clock.format(new Date(data.sampled_at * 1000))}${topology}` : '采样未连接'); + const tbody = $('telemetry-body'); + if (!tbody) return; + tbody.replaceChildren(); + nodes.forEach((item) => { + const row = document.createElement('tr'); + const usable = item.source_ok === true && item.clock_ok === true && + typeof item.clock_uncertainty_seconds === 'number' && Number.isFinite(item.clock_uncertainty_seconds) && + item.clock_uncertainty_seconds >= 0 && item.clock_uncertainty_seconds <= 20; + const prefill = usable && item.backend_ready === true ? freshTelemetry(item.prefill, now) : null; + const decode = usable && item.backend_ready === true ? freshTelemetry(item.decode, now) : null; + const hbm = usable ? freshTelemetry(item.hbm, now) : null; + const timerStep = item.phase_timers && item.phase_timers.step; + const timersFresh = prefill && prefill.step === timerStep || decode && decode.step === timerStep; + const timer = (phase) => { + const value = timersFresh && item.phase_timers && item.phase_timers[phase]; + return value && value.mean_ms !== null ? motor ? `${number.format(value.mean_ms)} ms · ${count(value.samples)} 请求 · ${value.ranks}/${value.expected_ranks} worker` : + `${number.format(value.mean_ms)} ms · ${value.ranks}/${value.expected_ranks} rank` : '—'; + }; + cell(row, item.label || '未知实例'); + cell(row, prefill ? `${measured(prefill.tokens_per_second)} · ${motor ? '采样' : '日志'} ${clock.format(new Date(prefill.log_at * 1000))}` : '—'); + cell(row, decode ? `${measured(decode.tokens_per_second)} · ${motor ? '采样' : '日志'} ${clock.format(new Date(decode.log_at * 1000))}` : '—'); + cell(row, prefill ? showMfu(prefill.mfu_estimate) : '—'); + cell(row, hbm ? `${number.format(hbm.mean_percent)}% · ${hbm.device_count} 卡` : '—'); + cell(row, timer('prefill')); + cell(row, timer('decode')); + const mtp = usable && item.backend_ready === true && item.mtp && validMtp(item.mtp.lifetime, now); + cell(row, mtp ? `${percent(mtp.drafted ? mtp.accepted / mtp.drafted : null)} · ${mtp.valid_endpoints}/${mtp.expected_endpoints} 端点${mtp.partial ? ' · 部分数据' : ''}` : '—'); + const engine = Boolean(prefill || decode); + const device = Boolean(hbm); + cell(row, item.duplicate_source ? '设备组重复，待核验' : !item.clock_ok ? '时钟不可用' : + !item.source_ok ? (motor ? '指标不可达或实例未就绪' : '引擎版本待核验') : item.backend_ready === false ? '后端未就绪' : + item.driver_ok === false && !motor ? '驱动版本待核验' : + engine && device ? '引擎与设备有样本' : engine ? '仅引擎有样本' : device ? '仅设备有样本' : '来源已核验，暂无样本'); + tbody.appendChild(row); + }); + if (!nodes.length) { + const row = document.createElement('tr'); + const value = document.createElement('td'); value.colSpan = 9; value.className = 'empty'; + value.textContent = '真实采样尚未连接'; row.appendChild(value); tbody.appendChild(row); + } + const profile = selectedNode && selectedNode.assumptions; + const assumption = profile ? `当前假设 ${number.format(profile.prefill_flops_per_token / 1e9)} GFLOP/输入 Token、${number.format(profile.peak_flops_per_device / 1e12)} TFLOP/s/卡；` : ''; + setTextIfPresent('live-efficiency-note', motor ? '当前 vLLM 指标未提供可用的真实阶段算力或 HBM 带宽归因；Prefill-MFU、Decode-MBU 和 HBM 利用率暂显示未知。' : + `${assumption}Prefill-MFU 按引擎吞吐和模型假设估算，未计上下文 Attention 等，并非硬件算力实测。HBM 为设备级混合阶段采样，无法直接归因 Decode。`); + setTextIfPresent('phase-source-note', motor ? + '吞吐为各 worker 在连续两次采样间的计数增量除以墙钟间隔并相加；Prefill 只计本地计算的输入，Decode 只计输出。采样前后核对拓扑，但无法保证两次读取之间没有变化；通过进程标识和计数回退检测重启，仍可能漏掉内部重启。缺失或冲突的样本不作为零。' : + '并发吞吐是相同日志 step 的各 DP rank 单步速率近似和。跨组视图对有效采样求和，缺失组不参与并标注覆盖数；利用率只对有效组加权。各组并非同时测量，两个阶段也可能来自不同时刻。'); + setTextIfPresent('telemetry-source-note', motor ? + '阶段耗时为引擎请求阶段直方图的增量均值，不能还原单个会话；Prefill 和 Decode 分别取对应角色。HBM 阶段带宽尚未接入。' : + '步耗时来自引擎 CPU 计时日志，包含调度等开销；括号内为有有效计时的 rank 数。HBM 为驱动设备级百分比，不能可靠分离纯 Decode 阶段。'); + setTextIfPresent('device-history-note', motor ? + '速度为每个 10 分钟区间内的 vLLM 计数差速率采样均值；跨组对当前设备组的有效采样求和。空心点表示部分采集；采集前或不可用期间留空。' : + '速度是各组日志步骤速率的区间采样均值之和；空心点表示部分设备组有样本。采集前或不可用期间留空。'); + const deadlines = nodes.flatMap((item) => [item.prefill, item.decode, item.hbm, item.mtp && item.mtp.lifetime, item.mtp && item.mtp.interval]) + .map((part) => part && part.valid_until).filter((value) => typeof value === 'number' && Number.isFinite(value) && value > now); + if (deadlines.length) telemetryExpiry = setTimeout(() => renderTelemetry(telemetrySnapshot), + Math.min(2147483647, Math.max(1, (Math.min(...deadlines) - now) * 1000 + 20))); +} + +async function refreshTelemetry() { + if (pendingTelemetry) return pendingTelemetry; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 4000); + pendingTelemetry = (async () => { + try { + const response = await fetch('/api/v5/telemetry', {cache: 'no-store', signal: controller.signal}); + if (!response.ok) throw new Error('telemetry unavailable'); + renderTelemetry(await response.json()); + } catch (_) { + renderTelemetry({available: false, nodes: []}); + } finally { + clearTimeout(timeout); + pendingTelemetry = null; + } + })(); + return pendingTelemetry; +} + +function activateView(name) { + const view = Object.prototype.hasOwnProperty.call(views, name) ? name : 'overview'; + currentView = view; + document.querySelectorAll('[data-view]').forEach((element) => { element.hidden = element.dataset.view !== view; }); + document.querySelectorAll('.sidebar nav a').forEach((link) => { + const active = link.getAttribute('href') === `#${view}`; + link.classList.toggle('nav-active', active); + if (active) link.setAttribute('aria-current', 'page'); else link.removeAttribute('aria-current'); + }); + $('view-title').textContent = views[view][0]; + $('view-kicker').textContent = views[view][1]; + document.querySelector('.segmented').hidden = view === 'active' || view === 'backends'; +} + +function setupNavigation() { + const navigate = () => { + activateView(window.location.hash.slice(1)); + if (currentView === 'overview' || currentView === 'active') refreshActive(); + }; + document.querySelectorAll('.sidebar nav a').forEach((link) => link.addEventListener('click', (event) => { + event.preventDefault(); + const view = link.getAttribute('href').slice(1); + if (window.location.hash !== `#${view}`) history.pushState(null, '', `#${view}`); + activateView(view); + if (view === 'overview' || view === 'active') refreshActive(); + window.scrollTo(0, 0); + })); + window.addEventListener('popstate', navigate); + window.addEventListener('hashchange', navigate); + navigate(); +} + +function display(value, stats, digits = false, includePartial = true) { + if (!stats.known && !(includePartial && stats.partial) && !stats.complete) return '—'; + const prefix = stats.complete ? '' : '≥'; + return prefix + (digits ? number.format(value) : integer.format(value)); +} + +function cell(row, text) { + const node = document.createElement('td'); + node.textContent = text; + row.appendChild(node); +} + +function empty(body, columns, text) { + const row = document.createElement('tr'); + const item = document.createElement('td'); + item.colSpan = columns; + item.className = 'empty'; + item.textContent = text; + row.appendChild(item); + body.appendChild(row); +} + +function setupTables() { + Object.values(tables).forEach((table) => { + const section = document.querySelector(table.selector); + const title = section.querySelector('h2').textContent; + table.headers = [...section.querySelectorAll('thead th')]; + table.headers.forEach((header, index) => { + const key = table.columns[index]; + const label = header.textContent; + const button = document.createElement('button'); + button.type = 'button'; button.className = 'sort-control'; button.textContent = label; + button.setAttribute('aria-label', `${label}，点击排序`); + header.replaceChildren(button); + header.addEventListener('click', () => { + if (table.sort === key) table.direction = table.direction === 'asc' ? 'desc' : 'asc'; + else { table.sort = key; table.direction = textSorts.has(key) ? 'asc' : 'desc'; } + table.offset = 0; + refresh(); + }); + }); + if (table.noPager) return; + let foot = section.querySelector('.panel-foot'); + if (!foot) { foot = document.createElement('div'); foot.className = 'panel-foot'; section.appendChild(foot); } + let pager = foot.querySelector('.pager'); + if (!pager) { pager = document.createElement('div'); pager.className = 'pager'; foot.appendChild(pager); } + pager.replaceChildren(); + if (table === tables.sessions || table === tables.requests) { + const hint = document.createElement('span'); hint.className = 'snapshot-hint'; + hint.textContent = '翻页快照只包含建立时已入账的记录；将会话和请求都返回第一页，或点击右上角刷新，可查看后续入账。'; + foot.insertBefore(hint, pager); + } + if (table === tables.active) { + const hint = document.createElement('span'); hint.className = 'snapshot-hint'; + hint.textContent = '当前连接随采样更新，翻页结果可能变化。'; + foot.insertBefore(hint, pager); + } + const label = document.createElement('label'); + label.textContent = '每页 '; + const select = document.createElement('select'); + select.setAttribute('aria-label', `${title}每页显示条数`); + [10, 20, 50].forEach((size) => { + const option = document.createElement('option'); option.value = String(size); option.textContent = String(size); + select.appendChild(option); + }); + select.value = String(table.size); + select.addEventListener('change', () => { table.size = Number(select.value); table.offset = 0; refresh(); }); + label.appendChild(select); + const previous = document.createElement('button'); previous.type = 'button'; previous.textContent = '上一页'; previous.disabled = true; + const page = document.createElement('span'); + const next = document.createElement('button'); next.type = 'button'; next.textContent = '下一页'; next.disabled = true; + previous.setAttribute('aria-label', `${title}上一页`); next.setAttribute('aria-label', `${title}下一页`); + previous.addEventListener('click', () => { table.offset = Math.max(0, table.offset - table.size); refresh(); }); + next.addEventListener('click', () => { table.offset += table.size; refresh(); }); + pager.append(label, previous, page, next); + table.pager = {previous, page, next}; + }); +} + +function updateHeaders(table) { + table.headers.forEach((header, index) => { + const direction = table.columns[index] === table.sort ? (table.direction === 'asc' ? 'ascending' : 'descending') : 'none'; + header.setAttribute('aria-sort', direction); + }); +} + +function updatePager(table, total, available = true) { + if (!table.pager) return; + const at = table === tables.requests || table === tables.sessions ? lastSnapshot?.at : + table.offset > 0 ? table.localData?.as_of : null; + const snapshot = at ? ` · 快照 ${snapshotTime.format(new Date(at * 1000))}` : ''; + table.pager.page.textContent = available ? `共 ${count(total)} 条 · ${total ? table.offset + 1 : 0}–${Math.min(table.offset + table.size, total)}${snapshot}` : '数据不可用'; + table.pager.previous.disabled = !available || table.offset === 0; + table.pager.next.disabled = !available || table.offset + table.size >= total; +} + +function localSource(name, data) { + const table = tables[name]; + if (table.offset > 0 && table.localData) return table.localData; + table.localData = data; + return data; +} + +function localValue(name, item, key, data) { + if (name === 'active' && key === 'phase') return item.first_visible === null ? 0 : 1; + if (name === 'active' && (key === 'hit_rate' || key === 'tps')) return null; + if (name === 'backends' && key !== 'port' && !item.available) return null; + if (name === 'clients' && key === 'total') return item.known || item.partial || data.totals.complete ? item.input + item.output : null; + if (name === 'clients' && key === 'cached' && !item.known && !data.totals.complete) return null; + if (name === 'clients' && ['input', 'output', 'window_tps'].includes(key) && !item.known && !item.partial && !data.totals.complete) return null; + if (name === 'days' && key === 'cached' && !item.known && !item.complete) return null; + if (name === 'days' && key !== 'date' && !item.known && !item.partial && !item.complete) return null; + return item[key] ?? null; +} + +function localPage(name, rows, data, available = true) { + const table = tables[name]; + const ordered = rows.map((item, index) => ({item, index})).sort((a, b) => { + const left = localValue(name, a.item, table.sort, data), right = localValue(name, b.item, table.sort, data); + if (left === null || right === null) return left === right ? a.index - b.index : left === null ? 1 : -1; + const delta = typeof left === 'number' && typeof right === 'number' ? left - right : + String(left).localeCompare(String(right), 'zh-CN', {numeric: true, sensitivity: 'base'}); + return (table.direction === 'asc' ? delta : -delta) || a.index - b.index; + }).map((entry) => entry.item); + if (table.noPager) return ordered; + table.offset = ordered.length ? Math.min(table.offset, Math.floor((ordered.length - 1) / table.size) * table.size) : 0; + updatePager(table, ordered.length, available); + return ordered.slice(table.offset, table.offset + table.size); +} + +function renderClients(data) { + data = localSource('clients', data); + const body = $('clients-body'); + body.replaceChildren(); + $('client-count').textContent = `${data.clients.length} 个来源`; + const clients = localPage('clients', data.clients, data); + if (!data.clients.length) { + empty(body, 8, '当前范围内暂无请求记录'); + return; + } + clients.forEach((client) => { + const row = document.createElement('tr'); + const complete = data.totals.complete && !client.partial && !client.missing && !client.failed && !client.interrupted; + const view = {...client, complete, covered_seconds: data.totals.covered_seconds}; + const unknown = client.missing + client.failed + client.interrupted; + cell(row, client.ip || '未知来源'); + const details = [client.partial ? `${integer.format(client.partial)} 部分` : '', unknown ? `${integer.format(unknown)} 未计量` : ''].filter(Boolean).join('，'); + cell(row, details ? `${integer.format(client.requests)}（${details}）` : integer.format(client.requests)); + cell(row, display(client.input + client.output, view)); + cell(row, display(client.input, view)); + cell(row, display(client.output, view)); + cell(row, display(client.cached, view, false, false)); + cell(row, rate(client.request_tps)); + cell(row, client.window_tps === null ? '—' : display(client.window_tps, view, true)); + body.appendChild(row); + }); +} + +function renderActive(data) { + if (data.as_of < latestActiveAsOf) return; + latestActiveAsOf = data.as_of; + const body = $('active-body'); + body.replaceChildren(); + updateHeaders(tables.active); + if (activeExpiry) clearTimeout(activeExpiry); + const age = (Date.now() / 1000) - data.active_sampled_at; + const ready = data.collector_live && data.active_ready && data.active_sampled_at !== null && age >= 0 && age < 3; + const unavailable = () => { + tables.active.offset = 0; + $('active-count').textContent = '—'; $('active-badge').textContent = '快照不可用'; + body.replaceChildren(); empty(body, 8, '采集快照过期或不可用'); + updatePager(tables.active, 0, false); + }; + if (!ready) { + unavailable(); + return; + } + $('active-count').textContent = count(data.active.length); + $('active-badge').textContent = `快照 ${clock.format(new Date(data.active_sampled_at * 1000))}`; + const active = localPage('active', data.active, data); + activeExpiry = setTimeout(unavailable, Math.max(0, (3 - age) * 1000)); + if (!data.active.length) { empty(body, 8, '当前采集快照中暂无在途推理请求'); return; } + active.forEach((item) => { + const row = document.createElement('tr'); + cell(row, item.ip || '未知来源'); + cell(row, fingerprint(item.session)); + cell(row, seconds(item.elapsed)); + cell(row, item.first_visible === null ? '等待可见增量' : '已有可见增量'); + ['input', 'output'].forEach((key) => cell(row, count(item[key]))); + cell(row, '—'); + cell(row, '—'); + body.appendChild(row); + }); +} + +async function refreshActive() { + if (pendingActive || (currentView !== 'overview' && currentView !== 'active')) return; + const controller = new AbortController(); + pendingActive = controller; + const timeout = setTimeout(() => controller.abort(), 3000); + try { + const response = await fetch('/api/v5/active', {cache: 'no-store', signal: controller.signal}); + if (!response.ok) throw new Error('unavailable'); + const data = await response.json(); + if (currentView === 'overview' || currentView === 'active') renderActive(data); + } catch (_) { + // The existing sample expires independently if the lightweight poll fails. + } finally { + clearTimeout(timeout); + if (pendingActive === controller) pendingActive = null; + } +} + +function renderBackends(data) { + if (backendExpiry) clearTimeout(backendExpiry); + backendExpiry = null; + const now = Date.now() / 1000; + const body = $('backends-body'); + body.replaceChildren(); + if (!data.backends.length) { empty(body, 7, '未配置后端原生指标'); return; } + localPage('backends', data.backends, data).forEach((backend) => { + const row = document.createElement('tr'); + cell(row, backend.label || String(backend.port)); + const hasDeadline = Object.prototype.hasOwnProperty.call(backend, 'valid_until'); + const current = backend.available && (!hasDeadline || typeof backend.valid_until === 'number' && + Number.isFinite(backend.valid_until) && now < backend.valid_until); + if (!current) { + cell(row, '不可用'); + for (let i = 0; i < 5; i++) cell(row, '—'); + } else { + const prefix = backend.partial ? '≥' : ''; + cell(row, backend.running === null || backend.running === undefined ? '—' : prefix + count(backend.running)); + cell(row, backend.waiting === null || backend.waiting === undefined ? '—' : prefix + count(backend.waiting)); + cell(row, backend.workers_expected === undefined ? count(backend.workers_alive) : `${count(backend.workers_alive)}/${count(backend.workers_expected)}`); + cell(row, backend.admission_ready === 1 ? '就绪' : '未就绪'); + cell(row, backend.input_total === null || backend.input_total === undefined ? '—' : prefix + count(backend.input_total)); + cell(row, `${percent(backend.hit_rate)}${backend.partial && backend.hit_rate !== null && backend.hit_rate !== undefined ? ' · 部分' : ''}`); + } + body.appendChild(row); + }); + const deadlines = data.backends.filter((backend) => backend.available && typeof backend.valid_until === 'number' && + Number.isFinite(backend.valid_until) && backend.valid_until > now).map((backend) => backend.valid_until); + if (deadlines.length) backendExpiry = setTimeout(() => renderBackends(data), + Math.min(2147483647, Math.max(1, (Math.min(...deadlines) - now) * 1000 + 20))); +} + +function renderSessions(data) { + const summary = $('session-summary-body'); + summary.replaceChildren(); + tables.sessions.offset = data.session_offset; + tables.requests.offset = data.request_offset; + updatePager(tables.sessions, data.session_count); + updatePager(tables.requests, data.request_count); + $('session-count').textContent = `${count(data.session_count)} 个可识别会话 · ${count(data.session_unknown_count)} 个请求未知`; + if (!data.sessions.length) empty(summary, 13, '当前页暂无可识别会话'); + data.sessions.forEach((item) => { + const row = document.createElement('tr'); + const view = {known: item.known, partial: item.partial, complete: data.snapshot_complete && item.known === item.requests}; + cell(row, fingerprint(item.session)); + cell(row, item.ips.length ? item.ips.join(', ') : '未知来源'); + cell(row, `${count(item.requests)} / ${count(item.known)} 完整 / ${count(item.partial || 0)} 部分`); + cell(row, item.input === null ? '—' : display(item.input, view)); + cell(row, item.output === null ? '—' : display(item.output, view)); + cell(row, item.cached === null || item.cached === undefined ? '—' : display(item.cached, view, false, false)); + cell(row, item.uncached_input === null || item.uncached_input === undefined ? '—' : display(item.uncached_input, view, false, false)); + cell(row, percent(item.hit_rate)); + cell(row, seconds(item.first_visible)); cell(row, seconds(item.after_first)); + cell(row, rate(item.input_per_wait)); cell(row, rate(item.output_per_after_first)); + cell(row, item.first_share === null ? '—' : `${percent(item.first_share)} / ${percent(1 - item.first_share)}`); + summary.appendChild(row); + }); + const body = $('sessions-body'); + body.replaceChildren(); + if (!data.requests.length) empty(body, 14, '当前页暂无请求记录'); + const states = {known: '已计量', partial: '部分用量', missing: '未计量', failed: '失败', interrupted: '中断'}; + data.requests.forEach((item) => { + const row = document.createElement('tr'); + cell(row, dateTime.format(new Date(item.finished * 1000))); + cell(row, item.ip || '未知来源'); + cell(row, fingerprint(item.session)); + cell(row, `${states[item.state] || '未知'}${item.status ? ` ${item.status}` : ''}`); + cell(row, count(item.input)); cell(row, count(item.output)); cell(row, count(item.cached)); + cell(row, count(item.uncached_input)); cell(row, percent(item.hit_rate)); + cell(row, seconds(item.first_visible)); cell(row, seconds(item.after_first)); + cell(row, rate(item.input_per_wait)); cell(row, rate(item.output_per_after_first)); + cell(row, item.first_share === null ? '—' : `${percent(item.first_share)} / ${percent(item.after_first_share)}`); + body.appendChild(row); + }); +} + +function renderDays(data) { + data = localSource('days', data); + const body = $('days-body'); + body.replaceChildren(); + localPage('days', data.days, data).forEach((day) => { + const row = document.createElement('tr'); + cell(row, day.date); + cell(row, display(day.input, day)); + cell(row, display(day.output, day)); + cell(row, display(day.cached, day, false, false)); + body.appendChild(row); + }); +} + +function svgElement(name, attributes) { + const el = document.createElementNS('http://www.w3.org/2000/svg', name); + Object.entries(attributes).forEach(([key, value]) => el.setAttribute(key, String(value))); + return el; +} + +function attachTrendTooltip(svg, descriptions) { + const tooltip = $('trend-tooltip'); + const previousPointer = tooltip && tooltip._trendActive === svg ? svg._trendPointer : null; + const previousKeyboard = tooltip && tooltip._trendActive === svg ? svg._trendKeyboardIndex : null; + const hide = () => { + if (tooltip && tooltip._trendOwner === svg) { tooltip.hidden = true; tooltip._trendOwner = null; } + }; + hide(); + svg.onpointermove = svg.onpointerleave = svg.onfocus = svg.onblur = svg.onkeydown = null; + svg.removeAttribute('tabindex'); + svg.removeAttribute('aria-describedby'); + if (!tooltip || !Array.isArray(descriptions) || !descriptions.some((text) => typeof text === 'string' && text)) { + svg._trendPointer = null; svg._trendKeyboardIndex = null; + if (tooltip && tooltip._trendActive === svg) tooltip._trendActive = null; + return; + } + svg.appendChild(svgElement('rect', {x: 0, y: 0, width: 800, height: 68, fill: 'transparent', class: 'trend-hover-overlay'})); + svg.setAttribute('tabindex', '0'); + svg.setAttribute('aria-describedby', 'trend-tooltip'); + let selected = Number.isSafeInteger(svg._trendKeyboardIndex) ? svg._trendKeyboardIndex : -1; + const activate = () => { + if (tooltip._trendOwner && tooltip._trendOwner !== svg) { tooltip.hidden = true; tooltip._trendOwner = null; } + tooltip._trendActive = svg; + }; + const show = (index, x, y) => { + const description = descriptions[index]; + selected = index; + if (typeof description !== 'string' || !description) { hide(); return; } + tooltip._trendOwner = svg; + tooltip.textContent = description; + tooltip.hidden = false; + const box = tooltip.getBoundingClientRect(); + const width = document.documentElement.clientWidth || window.innerWidth; + const height = document.documentElement.clientHeight || window.innerHeight; + tooltip.style.left = `${Math.max(8, Math.min(x + 12, width - box.width - 8))}px`; + tooltip.style.top = `${Math.max(8, Math.min(y + 12, height - box.height - 8))}px`; + }; + svg.onpointermove = (event) => { + const bounds = svg.getBoundingClientRect(); + if (!Number.isFinite(event.clientX) || !Number.isFinite(event.clientY) || bounds.width <= 0 || + event.clientX < bounds.left || event.clientX > bounds.left + bounds.width) { svg._trendPointer = null; hide(); return; } + activate(); + svg._trendPointer = {clientX: event.clientX, clientY: event.clientY}; + svg._trendKeyboardIndex = null; + const index = descriptions.length === 1 ? 0 : Math.round((event.clientX - bounds.left) / bounds.width * (descriptions.length - 1)); + show(index, event.clientX, event.clientY); + }; + const showSelected = (index) => { + const bounds = svg.getBoundingClientRect(); + const fraction = descriptions.length === 1 ? 0 : index / (descriptions.length - 1); + svg._trendKeyboardIndex = index; + svg._trendPointer = null; + show(index, bounds.left + fraction * bounds.width, bounds.top); + }; + svg.onpointerleave = () => { svg._trendPointer = null; hide(); if (tooltip._trendActive === svg) tooltip._trendActive = null; }; + svg.onfocus = () => { activate(); const first = descriptions.findIndex((value) => typeof value === 'string' && value); if (first >= 0) showSelected(first); }; + svg.onblur = () => { svg._trendKeyboardIndex = null; hide(); if (tooltip._trendActive === svg) tooltip._trendActive = null; }; + svg.onkeydown = (event) => { + if (!['ArrowLeft', 'ArrowRight'].includes(event.key)) return; + activate(); + const direction = event.key === 'ArrowRight' ? 1 : -1; + for (let index = selected + direction; index >= 0 && index < descriptions.length; index += direction) { + if (typeof descriptions[index] === 'string' && descriptions[index]) { event.preventDefault(); showSelected(index); break; } + } + }; + if (previousPointer) svg.onpointermove(previousPointer); + else if (Number.isSafeInteger(previousKeyboard) && previousKeyboard >= 0 && previousKeyboard < descriptions.length) showSelected(previousKeyboard); +} + +const charts = [ + {id: 'input', field: 'input', kind: 'tokens', label: '输入'}, + {id: 'output', field: 'output', kind: 'tokens', label: '输出'}, + {id: 'cache-rate', style: 'cached', field: 'mean_request_hit_rate', kind: 'hit', label: '平均请求命中率', samples: 'request_hit_samples'}, +]; + +function combinedDeviceHistory(data, nodes) { + if (!data || data.available !== true || data.history_available !== true || !nodes.length || + new Set(nodes.map((node) => node && node.backend_port)).size !== nodes.length) return null; + const method = data.kind === 'vllm-motor-telemetry' ? 'vllm-counter-rate-v1' : 'deepinfer-log-step-approx-v1'; + const usable = nodes.flatMap((node) => { + if (!node || node.source_ok !== true || !node.history || node.history.method !== method || + node.history.bucket_seconds !== 600 || !Array.isArray(node.history.points) || node.history.points.length !== 144) return []; + const buckets = new Map(); + for (const point of node.history.points) { + if (!point || !Number.isSafeInteger(point.start) || point.end !== point.start + 600 || buckets.has(point.start)) return []; + buckets.set(point.start, point); + } + return [{points: node.history.points, buckets}]; + }); + if (!usable.length) return null; + const points = usable[0].points.map((point, index) => { + if (point.start !== usable[0].points[0].start + index * 600) return null; + const all = usable.map((node) => node.buckets.get(point.start)).filter((item) => + item && item.start === point.start && item.end === point.end); + const combined = {start: point.start, end: point.end}; + ['prefill', 'decode'].forEach((phase) => { + const valid = all.filter((item) => typeof item[`${phase}_tps`] === 'number' && + Number.isFinite(item[`${phase}_tps`]) && item[`${phase}_tps`] >= 0 && + Number.isSafeInteger(item[`${phase}_samples`]) && item[`${phase}_samples`] > 0); + const total = valid.length ? valid.reduce((sum, item) => sum + item[`${phase}_tps`], 0) : null; + combined[`${phase}_tps`] = total !== null && Number.isFinite(total) ? total : null; + combined[`${phase}_samples`] = combined[`${phase}_tps`] === null ? 0 : 1; + combined[`${phase}_groups`] = combined[`${phase}_tps`] === null ? 0 : valid.length; + combined[`${phase}_partial`] = valid.some((item) => item[`${phase}_partial`] === true); + }); + return combined; + }); + if (points.some((point) => point === null)) return null; + return {history: {method, bucket_seconds: 600, aggregate: true, group_count: nodes.length, + prefill_group_count: nodes.filter((node) => data.kind !== 'vllm-motor-telemetry' || node.role === 'prefill').length, + decode_group_count: nodes.filter((node) => data.kind !== 'vllm-motor-telemetry' || node.role === 'decode').length, points}}; +} + +function renderMtpHistory(data, nodes, port) { + const svg = $('chart-mtp-svg'), label = $('chart-mtp'); + if (!svg || !label) return; + svg.replaceChildren(); + [16, 34, 52].forEach((y) => svg.appendChild(svgElement('line', {x1: 0, y1: y, x2: 800, y2: y, class: 'grid'}))); + const groups = data && data.available === true && data.history_available === true && data.kind === 'vllm-motor-telemetry' ? + nodes.filter((node) => node && node.role === 'decode' && (port === 'all' || node.backend_port === port)) : []; + const histories = groups.flatMap((node) => { + const history = node.mtp_history; + if (!history || history.method !== 'vllm-mtp-counter-delta-v1' || history.bucket_seconds !== 600 || + !Array.isArray(history.points) || history.points.length !== 144 || + history.points.some((point, i) => !point || !Number.isSafeInteger(point.start) || point.start < 0 || + point.end !== point.start + 600 || (i && point.start !== history.points[i - 1].start + 600))) return []; + return [new Map(history.points.map((point) => [point.start, point]))]; + }); + if (!groups.length || new Set(groups.map((node) => node.backend_port)).size !== groups.length || !histories.length) { + attachTrendTooltip(svg, []); label.textContent = '等待 MTP 历史采样'; return; + } + const starts = Array.from(histories[0].keys()); + const points = starts.map((start) => { + const valid = histories.map((history) => history.get(start)).filter((point) => point && + Number.isSafeInteger(point.samples) && point.samples > 0 && Number.isSafeInteger(point.drafted) && point.drafted >= 0 && + Number.isSafeInteger(point.accepted) && point.accepted >= 0 && point.accepted <= point.drafted); + const drafted = valid.reduce((sum, point) => sum + point.drafted, 0); + const accepted = valid.reduce((sum, point) => sum + point.accepted, 0); + if (!Number.isSafeInteger(drafted) || !Number.isSafeInteger(accepted) || !drafted) return null; + return {start, drafted, accepted, rate: accepted / drafted, groups: valid.length, + timeUnverified: valid.some((point) => point.time_coverage !== 'complete'), + partial: valid.length < groups.length || valid.some((point) => point.partial === true || point.time_coverage !== 'complete')}; + }); + const descriptions = Array(points.length).fill(null); + let segment = [], segmentPartial = false, drafted = 0, accepted = 0, buckets = 0, partialBuckets = 0; + const draw = () => { + if (segment.length > 1) svg.appendChild(svgElement('polyline', {points: segment.join(' '), class: `mtp${segmentPartial ? ' partial' : ''}`})); + segment = []; + }; + points.forEach((point, index) => { + if (!point) { draw(); return; } + if (segment.length && point.partial !== segmentPartial) draw(); + segmentPartial = point.partial; + const x = index * 800 / (points.length - 1), y = 59 - point.rate * 50; + segment.push(`${x.toFixed(2)},${y.toFixed(2)}`); + const marker = svgElement('circle', {cx: x, cy: y, r: 2.5, class: `mtp${point.partial ? ' partial' : ''}`}); + const period = `${dateTime.format(new Date(point.start * 1000))}–${dateTime.format(new Date((point.start + 600) * 1000))}`; + const description = `${period}\n推测解码草稿接受率：${percent(point.rate)}\n接受 ${count(point.accepted)} / 草稿 ${count(point.drafted)} · ${point.groups}/${groups.length} 注册组${point.partial ? ' · 部分数据' : ''}${point.timeUnverified ? ' · 时间覆盖未核验' : ''}`; + const title = svgElement('title', {}); title.textContent = description; + marker.appendChild(title); svg.appendChild(marker); descriptions[index] = description; + drafted += point.drafted; accepted += point.accepted; buckets++; if (point.partial) partialBuckets++; + }); + draw(); attachTrendTooltip(svg, descriptions); + label.textContent = drafted && Number.isSafeInteger(drafted) && Number.isSafeInteger(accepted) ? + `${percent(accepted / drafted)} · ${count(buckets)} 个有草稿区间${partialBuckets ? ` · ${count(partialBuckets)} 个部分区间` : ''}` : '暂无有草稿的区间'; +} + +function renderDeviceHistory(data, selectedNode) { + const dropped = data && Number.isSafeInteger(data.history_dropped) && data.history_dropped > 0 ? data.history_dropped : 0; + setTextIfPresent('device-history-health', dropped ? ` 历史队列已跳过 ${count(dropped)} 次采集。` : ''); + const history = data && data.available === true && data.history_available === true && selectedNode && selectedNode.history; + const points = history && ['deepinfer-log-step-approx-v1', 'vllm-counter-rate-v1'].includes(history.method) && history.bucket_seconds === 600 && + Array.isArray(history.points) && history.points.length === 144 ? history.points : null; + ['prefill', 'decode'].forEach((phase) => { + const svg = $(`chart-device-${phase}-svg`); + const label = $(`chart-device-${phase}`); + if (!svg || !label) return; + svg.replaceChildren(); + [16, 34, 52].forEach((y) => svg.appendChild(svgElement('line', {x1: 0, y1: y, x2: 800, y2: y, class: 'grid'}))); + if (!points) { attachTrendTooltip(svg, []); label.textContent = '设备历史采样未连接'; return; } + const valid = points.map((point) => { + const value = point && point[`${phase}_tps`]; + const samples = point && point[`${phase}_samples`]; + return typeof value === 'number' && Number.isFinite(value) && value >= 0 && Number.isSafeInteger(samples) && samples > 0 ? value : null; + }); + const peak = Math.max(1, ...valid.filter((value) => value !== null)); + const descriptions = Array(points.length).fill(null); + let segment = [], total = 0, samples = 0, segmentPartial = false, partialBuckets = 0; + const draw = () => { + if (segment.length > 1) svg.appendChild(svgElement('polyline', {points: segment.join(' '), class: `${phase}${segmentPartial ? ' partial' : ''}`})); + segment = []; + }; + valid.forEach((value, i) => { + if (value === null) { draw(); return; } + const groups = history[`${phase}_group_count`] ?? history.group_count; + const partial = points[i][`${phase}_partial`] === true || history.aggregate && points[i][`${phase}_groups`] < groups; + if (segment.length && partial !== segmentPartial) draw(); + segmentPartial = partial; + const x = i * 800 / (points.length - 1), y = 59 - value / peak * 50; + segment.push(`${x.toFixed(2)},${y.toFixed(2)}`); + const marker = svgElement('circle', {cx: x, cy: y, r: 2.5, class: `${phase}${partial ? ' partial' : ''}`}); + const title = svgElement('title', {}); + const scope = history.aggregate ? `${points[i][`${phase}_groups`]}/${groups} 组采样均值之和${partial ? ' · 部分数据' : ''}` : `${count(points[i][`${phase}_samples`])} 个采样${partial ? ' · 部分数据' : ''}`; + const period = `${dateTime.format(new Date(points[i].start * 1000))}–${dateTime.format(new Date(points[i].end * 1000))}`; + title.textContent = `${period}：${speed(value)} · ${scope}`; + descriptions[i] = `${period}\n${phase === 'prefill' ? 'Prefill' : 'Decode'} 并发吞吐：${speed(value)}\n${scope}`; + marker.appendChild(title); svg.appendChild(marker); + if (partial) partialBuckets++; + total += value * points[i][`${phase}_samples`]; samples += points[i][`${phase}_samples`]; + }); + draw(); + attachTrendTooltip(svg, descriptions); + label.textContent = samples ? `${speed(total / samples)} · ${count(samples)} ${history.aggregate ? `个有采样区间${partialBuckets ? ` · ${count(partialBuckets)} 个仅部分组` : ''}` : '个采样'}` : '暂无设备采样'; + }); +} + +function renderPlot(chart, points) { + const {id, field, kind} = chart; + const svg = $(`chart-${id}-svg`); + const label = $(`chart-${id}`); + if (!svg || !label) return; + svg.replaceChildren(); + [16, 34, 52].forEach((y) => svg.appendChild(svgElement('line', {x1: 0, y1: y, x2: 800, y2: y, class: 'grid'}))); + const required = kind === 'hit' ? [field, 'request_hit_sum', 'request_hit_samples'] : + kind === 'speed' ? [field, `${id}_tokens`, `${id}_seconds`, chart.samples] : []; + if (points.some((point) => required.some((key) => !Object.prototype.hasOwnProperty.call(point, key)))) { + attachTrendTooltip(svg, []); + label.textContent = '当前数据版本不支持'; + return; + } + const values = points.map((point) => point.end - point.start >= 599.99 && (point.known || point.partial || point.complete) && + typeof point[field] === 'number' && Number.isFinite(point[field]) ? point[field] : null); + const peak = Math.max(1, ...values.filter((value) => value !== null)); + const descriptions = Array(points.length).fill(null); + let segment = []; + const draw = () => { + if (segment.length > 1) svg.appendChild(svgElement('polyline', {points: segment.join(' '), class: chart.style || id})); + segment = []; + }; + values.forEach((value, i) => { + if (value === null) { draw(); return; } + const x = points.length > 1 ? i * 800 / (points.length - 1) : 0; + const y = 59 - value / peak * 50; + segment.push(`${x.toFixed(2)},${y.toFixed(2)}`); + const marker = svgElement('circle', {cx: x, cy: y, r: 2.5, class: chart.style || id}); + const title = svgElement('title', {}); + const formatted = kind === 'tokens' ? `${integer.format(value)} Token` : kind === 'hit' ? percent(value) : speed(value); + const boundary = points[i].complete ? '' : kind === 'tokens' ? '至少 ' : '已观测样本 '; + const samples = chart.samples ? ` · ${count(points[i][chart.samples])} 个样本` : ''; + const period = `${dateTime.format(new Date(points[i].start * 1000))}–${dateTime.format(new Date(points[i].end * 1000))}`; + title.textContent = `${period}：${boundary}${formatted}${samples}`; + descriptions[i] = `${period}\n${chart.label || id}：${boundary}${formatted}${samples}`; + marker.appendChild(title); + svg.appendChild(marker); + }); + draw(); + attachTrendTooltip(svg, descriptions); + const sum = (key) => points.reduce((total, point) => total + (point[key] || 0), 0); + if (kind === 'tokens') { + const any = points.some((point) => point.known || point.partial || point.complete); + const complete = points.filter((point) => point.end > point.start).every((point) => point.complete); + label.textContent = any ? `${complete ? '' : '≥'}${integer.format(sum(field))}` : '暂无数据'; + } else if (kind === 'hit') { + const samples = sum(chart.samples); + label.textContent = samples ? `${percent(sum('request_hit_sum') / samples)} · ${count(samples)} 请求` : '暂无样本'; + } else { + const samples = sum(chart.samples); + const duration = sum(`${id}_seconds`); + label.textContent = samples && duration ? `${speed(sum(`${id}_tokens`) / duration)} · ${count(samples)} 样本` : '暂无样本'; + } +} + +function render(data) { + const stats = data.totals; + $('status-dot').classList.toggle('live', data.collector_live); + $('status-label').textContent = data.collector_live ? '采集器心跳正常' : '入口采集未连接'; + $('updated').textContent = `数据查询于 ${clock.format(new Date(data.as_of * 1000))}`; + $('total-usage').textContent = display(stats.input + stats.output, stats); + $('total-input').textContent = display(stats.input, stats); + $('total-output').textContent = display(stats.output, stats); + $('total-cached').textContent = display(stats.cached, stats, false, false); + setTextIfPresent('total-uncached', stats.uncached_input === null || stats.uncached_input === undefined ? '—' : + display(stats.uncached_input, stats, false, false)); + $('total-tps').textContent = stats.window_tps === null ? '—' : display(stats.window_tps, stats, true); + $('cache-detail').textContent = stats.hit_rate === null ? '有缓存明细的请求命中率 —' : `有缓存明细的请求按输入 Token 加权 ${number.format(stats.hit_rate * 100)}%`; + setTextIfPresent('overview-prefill-wait', seconds(stats.first_mean)); + setTextIfPresent('overview-hit-rate', percent(stats.hit_rate)); + setTextIfPresent('overview-wait-samples', `${count(stats.first_samples)} 个可见增量样本`); + setTextIfPresent('overview-hit-samples', `按输入 Token 加权 · ${count(stats.known)} 个有缓存明细的请求`); + $('success-rate').textContent = percent(stats.success_rate); + $('success-detail').textContent = `${count(stats.successes)} / ${count(stats.classified)} 个可判定请求`; + $('requests-rate').textContent = !stats.covered_seconds || stats.requests_per_second === null ? '—' : + `${stats.complete ? '' : '≥'}${rate(stats.requests_per_second)}`; + $('first-p95').textContent = seconds(stats.first_p95); + $('first-detail').textContent = `${count(stats.first_samples)} 个已计量成功样本`; + $('elapsed-p95').textContent = seconds(stats.elapsed_p95); + $('elapsed-detail').textContent = `${count(stats.elapsed_samples)} 个已计量成功样本`; + const unknown = stats.missing + stats.interrupted + stats.failed; + const coverage = stats.seconds ? Math.min(100, stats.covered_seconds / stats.seconds * 100) : 0; + const notice = $('notice'); + notice.classList.toggle('warning', !stats.complete || !data.collector_live || stats.unknown_source > 0); + const collectorNotice = data.collector_live ? '' : '入口采集未连接不代表模型服务不可用；无法确认新请求已记录，后端状态仍可单独查看。'; + let overload = ''; + if (stats.overload_untracked_runs || stats.unclean_runs) { + const reasons = [stats.overload_untracked_runs ? '部分周期未细分' : '', + stats.unclean_runs ? '存在未正常结束周期' : ''].filter(Boolean).join('、'); + overload = stats.overload_tracked_runs ? + ` · 相关采集周期内代理超载降级已记录至少 ${integer.format(stats.overload_rejected)} 次（${reasons}）` : + ` · 相关采集周期内代理超载降级次数未知（${reasons}）`; + } else if (stats.overload_tracked_runs) { + overload = ` · 相关采集周期内代理超载降级已记录 ${integer.format(stats.overload_rejected)} 次`; + } + notice.textContent = `${collectorNotice}完整用量 ${integer.format(stats.known)} 个请求 · 仅输入/输出 ${integer.format(stats.partial || 0)} 个 · 未计量 ${integer.format(unknown)} 个 · 未知来源 ${integer.format(stats.unknown_source)} 个 · 未正常结束或失联 ${integer.format(stats.unclean_runs)} 次 · 相关采集周期未完整计入 ${integer.format(stats.dropped)} 个${overload} · 采集进程运行覆盖 ${number.format(coverage)}%。仅统计采集器已写盘的请求记录；缓存命中率仅使用有缓存明细的请求。入口回退、网关内部重试及取消时的消耗可能无法观测，因此不是后端物理总消耗。${stats.complete ? '' : ' ≥ 表示已观测下限；无覆盖不视为零。'}`; + charts.forEach((chart) => renderPlot(chart, data.series_24h)); + renderClients(data); + renderDays(data); + renderActive(data); + renderBackends(data); + renderSessions(data); + Object.values(tables).forEach(updateHeaders); +} + +async function refresh() { + const current = ++revision; + if (pendingRefresh) pendingRefresh.abort(); + const controller = new AbortController(); + pendingRefresh = controller; + const timeout = setTimeout(() => controller.abort(), 15000); + try { + const query = new URLSearchParams({window: selected, + offset: String(tables.requests.offset), limit: String(tables.requests.size), + sort: tables.requests.sort, direction: tables.requests.direction, + session_offset: String(tables.sessions.offset), session_limit: String(tables.sessions.size), + session_sort: tables.sessions.sort, session_direction: tables.sessions.direction}); + if (lastSnapshot && (tables.requests.offset > 0 || tables.sessions.offset > 0)) { + query.set('snapshot_at', String(lastSnapshot.at)); + query.set('snapshot_id', String(lastSnapshot.id)); + } + const response = await fetch(`/api/v5/stats?${query}`, {cache: 'no-store', signal: controller.signal}); + if (!response.ok) throw new Error('unavailable'); + const data = await response.json(); + if (current === revision) { + lastSnapshot = {at: data.snapshot_at, id: data.snapshot_id}; + render(data); + } + } catch (_) { + if (current !== revision) return; + if (activeExpiry) clearTimeout(activeExpiry); + activeExpiry = null; + if (backendExpiry) clearTimeout(backendExpiry); + backendExpiry = null; + $('status-dot').classList.remove('live'); + $('status-label').textContent = '统计不可用'; + $('updated').textContent = '请检查本机采集和账本'; + ['total-usage', 'total-input', 'total-output', 'total-cached', 'total-uncached', 'total-tps', + 'overview-prefill-wait', 'overview-hit-rate'].forEach((id) => setTextIfPresent(id, '—')); + ['overview-wait-samples', 'overview-hit-samples'].forEach((id) => setTextIfPresent(id, '统计不可用')); + charts.forEach(({id}) => { + const svg = $(`chart-${id}-svg`), label = $(`chart-${id}`); + if (svg) { svg.replaceChildren(); attachTrendTooltip(svg, []); } + if (label) label.textContent = '暂无数据'; + }); + $('clients-body').replaceChildren(); + $('days-body').replaceChildren(); + empty($('clients-body'), 8, '统计不可用'); + empty($('days-body'), 4, '统计不可用'); + ['active-count', 'success-rate', 'requests-rate', 'first-p95', 'elapsed-p95'].forEach((id) => { $(id).textContent = '—'; }); + [['active-body', 8], ['backends-body', 7], ['session-summary-body', 13], ['sessions-body', 14]].forEach(([id, columns]) => { + $(id).replaceChildren(); empty($(id), columns, '统计不可用'); + }); + $('notice').classList.add('warning'); + $('notice').textContent = '查询超时或无法读取统计。历史缺失与采集故障不会显示为零。'; + Object.values(tables).forEach((table) => updatePager(table, 0, false)); + } finally { + clearTimeout(timeout); + if (pendingRefresh === controller) pendingRefresh = null; + } +} + +document.querySelectorAll('[data-window]').forEach((button) => button.addEventListener('click', () => { + selected = button.dataset.window; + Object.values(tables).forEach((table) => { table.offset = 0; }); + lastSnapshot = null; + document.querySelectorAll('[data-window]').forEach((item) => item.classList.toggle('is-active', item === button)); + refresh(); +})); +$('refresh').addEventListener('click', () => { + Object.values(tables).forEach((table) => { table.offset = 0; table.localData = null; }); + lastSnapshot = null; refresh(); +}); +setupTables(); +setupNavigation(); +$('telemetry-node')?.addEventListener('change', (event) => { + telemetryNodePort = event.target.value === 'all' ? 'all' : Number(event.target.value); + if (telemetrySnapshot) renderTelemetry(telemetrySnapshot); +}); +document.addEventListener('visibilitychange', () => { + if (!document.hidden) { + if (telemetrySnapshot) renderTelemetry(telemetrySnapshot); + refreshTelemetry(); + } +}); +refresh(); +refreshTelemetry(); +setInterval(() => { if (!pendingRefresh) refresh(); }, 10000); +setInterval(refreshActive, 2000); +setInterval(refreshTelemetry, 15000); diff --git a/deploy_stats/static/index.html b/deploy_stats/static/index.html new file mode 100644 index 0000000..646515a --- /dev/null +++ b/deploy_stats/static/index.html @@ -0,0 +1,78 @@ + + + + + + 模型用量监控 + + + + +
+ +
+
服务监控 / 模型用量
LOCAL
+
+
MODEL OBSERVABILITY

模型用量监控

按最终响应统计请求来源、Token 用量与缓存复用情况。

正在连接等待统计数据
+
OVERVIEW

总览

+
正在读取采集状态…
+
默认汇总全部设备组，也可选择单个后端；入口请求用量不受此选择影响
+
PREFILL
+
+
Prefill 并发吞吐（采样）↘
—等待引擎采样
+
Prefill 平均等待（估算）◷
—等待可测样本
+
Prefill-MFU（单步估算）%
—等待引擎采样
+
缓存命中率◈
—等待已计量样本
+
+
DECODE
+
+
Decode 并发吞吐（采样）↗
—等待引擎采样
+
Decode-MBU（纯阶段）%
—尚未接入阶段带宽归因
+
HBM 带宽利用率（设备采样）%
—等待设备采样
+
推测解码接受率（MTP 等）%
—等待 Decode 指标
+
+
并发吞吐是相同日志 step 的各 DP rank 单步速率近似和。跨组视图对有效采样求和，缺失组不参与并标注覆盖数；利用率只对有效组加权。各组并非同时测量，两个阶段也可能来自不同时刻。
+
Prefill-MFU 按引擎吞吐和模型假设估算，并非硬件算力实测；HBM 为设备级混合阶段采样，无法直接归因 Decode。
+
ENGINE & NPU

引擎与设备采样

按实例分别展示，避免把不同设备组或未对齐的采样合并。

等待采样
实例Prefill 吞吐Decode 吞吐Prefill-MFUHBM 带宽Prefill 阶段耗时Decode 阶段耗时推测解码接受率状态
等待采样
步耗时来自引擎 CPU 计时日志，包含调度等开销；括号内为有有效计时的 rank 数。HBM 为驱动设备级百分比，不能可靠分离纯 Decode 阶段。
+
+
总 TokenΣ
—输入 + 输出，缓存不重复计算
+
输入 Token↘
—包含缓存命中的输入
+
输出 Token↗
—已完成并取得用量的请求
+
缓存命中◈
—命中率 —
+
未命中输入（估算）↘
—已知缓存的输入 − 命中，非引擎实际计算量
+
窗口输出速度⌁
—输出 Token / 窗口秒数
+
+
+
当前在途请求◉
—仅采集已识别请求
+
可判定成功率✓
—已排除结果未知的请求
+
已观测请求速率↗
—可见结束请求 / 窗口秒数
+
首可见增量 p95◷
—入口观测，非纯 prefill
+
入口内耗时 p95◷
—已计量成功请求
+
+ + + + +
TREND

最近 24 小时

用量按请求完成时间分桶；速度随设备组选择切换。

10 分钟分桶
+
+
输入—
+
输出—
+
平均请求命中率—
+
Prefill 设备组并发吞吐（采样）—
+
Decode 设备组并发吞吐（采样）—
+
推测解码接受率（MTP 等）—
+
24 小时前12 小时前现在
只画满 10 分钟的区间；用量汇总包含未满区间。缓存命中率是各已计量请求命中率的算术平均。推测解码接受率 = 接受的草稿 Token / 草稿 Token；后端指标未标明方法，无法仅凭指标区分 MTP 等算法。累计值为各进程启动以来的数据；曲线按采样增量加权，属于当前 Motor 注册的 Decode 组而非单个请求，也可能包含其他入口。未注册的组不计入分母。曲线时间覆盖尚未核验，空心点表示观测到的部分数据；重启、缺失或采集前的区间不会回填。速度是各组日志步骤速率的区间采样均值之和；空心点表示部分设备组有样本。采集前或不可用期间留空。
+
+
+
+ +
+
+
+ + diff --git a/deploy_stats/store.py b/deploy_stats/store.py new file mode 100644 index 0000000..52dff28 --- /dev/null +++ b/deploy_stats/store.py @@ -0,0 +1,244 @@ +"""Asynchronous, local-only storage of request accounting metadata.""" +from pathlib import Path +import queue +import sqlite3 +import threading +import time +import uuid + + +SCHEMA = 2 +IDENTITY = 'llmdeploy-gateway-stats' +PARTIAL_MARKER = 'partial_usage_schema' + + +def partial_schema_available(db): + marker = db.execute('SELECT value FROM stats_meta WHERE key=?', (PARTIAL_MARKER,)).fetchone() + exists = db.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='stats_partial_usage'").fetchone() + if marker is None and exists is None: + return False + if marker != ('1',) or exists is None: + raise ValueError('PARTIAL_USAGE_SCHEMA_MISMATCH') + columns = [(row[1], row[2].upper(), row[5]) for row in db.execute('PRAGMA table_info(stats_partial_usage)')] + if columns != [('request_id', 'INTEGER', 1), ('input_tokens', 'INTEGER', 0), ('output_tokens', 'INTEGER', 0)]: + raise ValueError('PARTIAL_USAGE_SCHEMA_MISMATCH') + return True + + +def _enable_partial_schema(db): + if partial_schema_available(db): + return + db.execute('''CREATE TABLE stats_partial_usage ( + request_id INTEGER PRIMARY KEY REFERENCES requests(id), + input_tokens INTEGER NOT NULL CHECK(input_tokens >= 0), + output_tokens INTEGER NOT NULL CHECK(output_tokens >= 0))''') + db.execute('INSERT INTO stats_meta(key,value) VALUES (?,?)', (PARTIAL_MARKER, '1')) + + +def _open(path): + path = Path(path) + if not path.parent.is_dir(): + raise ValueError('STATS_DIRECTORY_MISSING') + db = sqlite3.connect(path, timeout=1) + try: + version = db.execute('PRAGMA user_version').fetchone()[0] + tables = db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").fetchall() + if version not in (0, 1, SCHEMA) or version == 0 and tables: + raise ValueError('STATS_DATABASE_SCHEMA_MISMATCH') + if version in (1, SCHEMA): + try: + marker = db.execute("SELECT value FROM stats_meta WHERE key='application'").fetchone() + except sqlite3.Error as error: + raise ValueError('STATS_DATABASE_SCHEMA_MISMATCH') from error + if marker != (IDENTITY,): + raise ValueError('STATS_DATABASE_SCHEMA_MISMATCH') + db.execute('PRAGMA journal_mode=WAL') + db.execute('PRAGMA synchronous=NORMAL') + if version == 0: + db.executescript(''' + CREATE TABLE stats_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + INSERT INTO stats_meta VALUES ('application','llmdeploy-gateway-stats'); + CREATE TABLE runs ( + id TEXT PRIMARY KEY, started REAL NOT NULL, last_seen REAL NOT NULL, + ended REAL, dropped INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE requests ( + id INTEGER PRIMARY KEY, run_id TEXT NOT NULL, finished REAL NOT NULL, + client_ip TEXT, elapsed REAL NOT NULL, http_status INTEGER, + state TEXT NOT NULL CHECK(state IN ('known','missing','failed','interrupted')), + input_tokens INTEGER, output_tokens INTEGER, cached_tokens INTEGER, + session_id TEXT, first_visible REAL, + CHECK(state != 'known' OR (input_tokens IS NOT NULL AND output_tokens IS NOT NULL + AND cached_tokens IS NOT NULL AND input_tokens >= 0 AND output_tokens >= 0 + AND cached_tokens >= 0 AND cached_tokens <= input_tokens)), + CHECK(state = 'known' OR (input_tokens IS NULL AND output_tokens IS NULL AND cached_tokens IS NULL)) + ); + CREATE INDEX requests_finished ON requests(finished); + CREATE TABLE active_requests ( + id TEXT PRIMARY KEY, run_id TEXT NOT NULL, started REAL NOT NULL, + client_ip TEXT, session_id TEXT, first_visible REAL + ); + CREATE TABLE active_samples (run_id TEXT PRIMARY KEY, seen REAL NOT NULL); + PRAGMA user_version=2; + ''') + db.commit() + elif version == 1: + db.execute('BEGIN IMMEDIATE') + db.execute('ALTER TABLE requests ADD COLUMN session_id TEXT') + db.execute('ALTER TABLE requests ADD COLUMN first_visible REAL') + db.execute('''CREATE TABLE active_requests ( + id TEXT PRIMARY KEY, run_id TEXT NOT NULL, started REAL NOT NULL, + client_ip TEXT, session_id TEXT, first_visible REAL + )''') + db.execute('CREATE TABLE active_samples (run_id TEXT PRIMARY KEY, seen REAL NOT NULL)') + db.execute('PRAGMA user_version=2') + db.commit() + return db + except Exception: + db.close() + raise + + +class Recorder: + def __init__(self, path, *, queue_size=4096, heartbeat=5, source=None): + if source is not None and source != 'gateway-inprocess-v1': + raise ValueError('INVALID_STATS_SOURCE') + self.path = Path(path) + self.source = source + self.run = uuid.uuid4().hex + self.started = time.time() + self.heartbeat = heartbeat + self.pending = queue.Queue(maxsize=queue_size) + self.stop = threading.Event() + self.ready = threading.Event() + self.lock = threading.Lock() + self.live_lock = threading.Lock() + self.live = {} + self.dropped = 0 + self.failed = False + self.thread = threading.Thread(target=self._write, name='stats-writer', daemon=True) + self.thread.start() + if not self.ready.wait(3) or self.failed: + self.stop.set() + raise ValueError('STATS_DATABASE_UNAVAILABLE') + + def _enqueue(self, item): + if self.failed or self.stop.is_set(): + return False + try: + self.pending.put_nowait(item) + return True + except queue.Full: + self.note_drop() + return False + + def begin(self, request_id, started, client_ip, session_id): + with self.live_lock: + self.live[request_id] = (started, client_ip, session_id, None) + + def first_output(self, request_id, first_visible): + with self.live_lock: + item = self.live.get(request_id) + if item is not None: + self.live[request_id] = (*item[:3], first_visible) + + def set_session(self, request_id, session_id): + with self.live_lock: + item = self.live.get(request_id) + if item is not None: + self.live[request_id] = (item[0], item[1], session_id, item[3]) + + def end(self, request_id): + with self.live_lock: + self.live.pop(request_id, None) + + def submit(self, event, request_id=None): + return self._enqueue(('event', (request_id, event))) + + def note_drop(self, count=1): + with self.lock: + self.dropped += count + + def flush(self, timeout=3): + done = threading.Event() + try: + self.pending.put(('barrier', done), timeout=timeout) + except queue.Full: + return False + return done.wait(timeout) and not self.failed + + def close(self, timeout=3): + self.stop.set() + self.thread.join(timeout) + return not self.thread.is_alive() and not self.failed + + def _write(self): + db = None + try: + db = _open(self.path) + db.execute('INSERT INTO runs(id,started,last_seen) VALUES (?,?,?)', + (self.run, self.started, self.started)) + db.execute('INSERT INTO active_samples(run_id,seen) VALUES (?,?)', (self.run, self.started)) + if self.source is not None: + _enable_partial_schema(db) + db.execute('INSERT INTO stats_meta(key,value) VALUES (?,?)', ('source:' + self.run, self.source)) + db.commit() + self.ready.set() + last_tick = time.monotonic() + last_snapshot = last_tick + while not self.stop.is_set() or not self.pending.empty(): + try: + kind, item = self.pending.get(timeout=min(self.heartbeat, 0.2)) + except queue.Empty: + kind, item = None, None + if kind == 'event': + request_id, event = item + event = tuple(event) + (None,) * (10 - len(event)) + partial = event[4] == 'partial' + if partial: + if (self.source != 'gateway-inprocess-v1' or type(event[5]) is not int or + type(event[6]) is not int or event[5] < 0 or event[6] < 0 or event[7] is not None): + raise ValueError('INVALID_PARTIAL_USAGE') + input_tokens, output_tokens = event[5:7] + event = (*event[:4], 'missing', None, None, None, *event[8:]) + cursor = db.execute('''INSERT INTO requests + (run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) + VALUES (?,?,?,?,?,?,?,?,?,?,?)''', (self.run, *event)) + if partial: + db.execute('INSERT INTO stats_partial_usage VALUES (?,?,?)', + (cursor.lastrowid, input_tokens, output_tokens)) + db.commit() + elif kind == 'barrier': + item.set() + if time.monotonic() - last_tick >= self.heartbeat: + with self.lock: + dropped = self.dropped + db.execute('UPDATE runs SET last_seen=?,dropped=? WHERE id=?', + (time.time(), dropped, self.run)) + db.commit() + last_tick = time.monotonic() + if time.monotonic() - last_snapshot >= 1: + with self.live_lock: + snapshot = list(self.live.items()) + sampled_at = time.time() + db.execute('DELETE FROM active_requests WHERE run_id=?', (self.run,)) + db.executemany('''INSERT INTO active_requests + (id,run_id,started,client_ip,session_id,first_visible) VALUES (?,?,?,?,?,?)''', + [(request_id, self.run, *item) for request_id, item in snapshot]) + db.execute('UPDATE active_samples SET seen=? WHERE run_id=?', (sampled_at, self.run)) + db.commit() + last_snapshot = time.monotonic() + with self.lock: + dropped = self.dropped + now = time.time() + db.execute('DELETE FROM active_requests WHERE run_id=?', (self.run,)) + db.execute('UPDATE runs SET last_seen=?,ended=?,dropped=? WHERE id=?', + (now, now, dropped, self.run)) + db.commit() + except Exception: + self.failed = True + self.ready.set() + finally: + if db is not None: + db.close() diff --git a/deploy_stats/telemetry.py b/deploy_stats/telemetry.py new file mode 100644 index 0000000..97e551f --- /dev/null +++ b/deploy_stats/telemetry.py @@ -0,0 +1,502 @@ +"""Collect sanitized engine-step and device samples via read-only SSH probes.""" +import argparse +from copy import deepcopy +from concurrent.futures import ThreadPoolExecutor +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import math +from pathlib import Path +import queue +import re +import sqlite3 +import subprocess +import threading +import time + +from .backends import read_backend +from .telemetry_history import History + + +SOURCE_KEYS = frozenset(('stats', 'backend', 'parallel', 'engine', 'sampler', 'native')) +HOST = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,63}@[A-Za-z0-9][A-Za-z0-9.-]{0,252}\Z') +CONTAINER = re.compile(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z') +SHA256 = re.compile(r'[0-9a-f]{64}\Z') +MAX_AGE = 60.0 +MAX_LOG_SKEW = 5.0 +MAX_STEP_SPREAD = .10 +HBM_MAX_AGE = 90.0 + + +def finite(value, *, minimum=None, maximum=None): + try: + return (type(value) in (int, float) and math.isfinite(value) and + (minimum is None or value >= minimum) and (maximum is None or value <= maximum)) + except OverflowError: + return False + + +def validate_config(value): + if not isinstance(value, dict) or set(value) != {'interval_seconds', 'nodes'}: + raise ValueError('INVALID_TELEMETRY_CONFIG') + interval, nodes = value['interval_seconds'], value['nodes'] + if not finite(interval, minimum=15, maximum=60) or not isinstance(nodes, list) or not 1 <= len(nodes) <= 8: + raise ValueError('INVALID_TELEMETRY_CONFIG') + ports, labels, targets = set(), set(), set() + fields = {'label', 'backend_port', 'ssh_target', 'container', 'source_sha256', 'rank_devices', + 'image_fingerprint', 'driver_sha256', 'prefill_flops_per_token', 'peak_flops_per_device'} + for node in nodes: + if not isinstance(node, dict) or set(node) != fields: + raise ValueError('INVALID_TELEMETRY_NODE') + label, port = node['label'], node['backend_port'] + target, container, sources, devices = (node['ssh_target'], node['container'], + node['source_sha256'], node['rank_devices']) + if (not isinstance(label, str) or not 1 <= len(label) <= 32 or any(ord(c) < 32 for c in label) or + type(port) is not int or not 1 <= port <= 65535 or port in ports or label in labels or + not isinstance(target, str) or not HOST.fullmatch(target) or target in targets or + not isinstance(container, str) or not CONTAINER.fullmatch(container) or + not isinstance(node['image_fingerprint'], str) or not SHA256.fullmatch(node['image_fingerprint']) or + not isinstance(node['driver_sha256'], str) or not SHA256.fullmatch(node['driver_sha256']) or + not isinstance(sources, dict) or set(sources) != SOURCE_KEYS or + any(not isinstance(v, str) or not SHA256.fullmatch(v) for v in sources.values()) or + not isinstance(devices, list) or not 1 <= len(devices) <= 64 or + any(type(d) is not int or not 0 <= d <= 255 for d in devices) or len(set(devices)) != len(devices) or + not finite(node['prefill_flops_per_token'], minimum=1, maximum=1e30) or + not finite(node['peak_flops_per_device'], minimum=1, maximum=1e25)): + raise ValueError('INVALID_TELEMETRY_NODE') + ports.add(port) + labels.add(label) + targets.add(target) + return value + + +def _phase_groups(rows, devices, started, observed): + """Align unique DP ranks by logger step; the result remains a sampled approximation.""" + if not isinstance(rows, list) or len(rows) > 1024: + return {} + groups, duplicates = {}, set() + for row in rows: + if not isinstance(row, dict) or row.get('kind') != 'stats': + continue + rank, device, step, at = (row.get(k) for k in ('rank', 'device', 'step', 'at')) + if (type(rank) is not int or not 0 <= rank < len(devices) or type(device) is not int or device != devices[rank] or + type(step) is not int or step < 0 or not finite(at, minimum=started, maximum=observed + 2)): + continue + group = groups.setdefault(step, {}) + if rank in group: + duplicates.add(step) + group[rank] = row + latest = {} + for step, group in groups.items(): + if step in duplicates or set(group) != set(range(len(devices))): + continue + data = list(group.values()) + at = [r['at'] for r in data] + if max(at) - min(at) > MAX_LOG_SKEW or observed - min(at) > MAX_AGE: + continue + phases = {r.get('is_prompt') for r in data} + if len(phases) != 1 or next(iter(phases)) not in (True, False) or any(type(r.get('is_prompt')) is not bool for r in data): + continue + if any(not finite(r.get('current_tps'), minimum=0, maximum=1e15) or + not finite(r.get('current_sps'), minimum=.001, maximum=1e9) for r in data): + continue + rates = [r['current_sps'] for r in data] + # cur_SPS is printed to three decimals; include the rounding interval. + spread = (max(rates) + .0005) / (min(rates) - .0005) - 1 + if not finite(spread, maximum=MAX_STEP_SPREAD): + continue + phase = 'prefill' if data[0]['is_prompt'] else 'decode' + item = {'step': step, 'log_at': max(at), 'rank_count': len(data), + 'log_skew_seconds': max(at) - min(at), 'step_rate_spread': spread, + 'tokens_per_second': sum(r['current_tps'] for r in data), + '_rank_at': {rank: row['at'] for rank, row in group.items()}} + identity = [step, phase, sorted(item['_rank_at'].items())] + item['_observation_id'] = hashlib.sha256(json.dumps(identity, separators=(',', ':')).encode()).hexdigest() + if phase not in latest or item['log_at'] > latest[phase]['log_at']: + latest[phase] = item + return latest + + +def _phase_timers(rows, step, devices, source_started, rank_times): + groups, duplicates = {}, set() + for row in rows: + if not isinstance(row, dict) or row.get('kind') != 'cpu' or row.get('step') != step: + continue + rank, at = row.get('rank'), row.get('at') + if (type(rank) is not int or not 0 <= rank < len(devices) or row.get('device') != devices[rank] or + rank not in rank_times or not finite(at, minimum=max(source_started, rank_times[rank]), + maximum=rank_times[rank] + MAX_LOG_SKEW)): + continue + if rank in groups: + duplicates.add(rank) + groups[rank] = row.get('timers') + result = {} + for phase in ('prefill', 'decode'): + pairs = [] + for rank, timers in groups.items(): + if rank in duplicates or not isinstance(timers, dict): + continue + timer = timers.get(phase) + if (isinstance(timer, dict) and finite(timer.get('mean_ms'), minimum=0, maximum=1e9) and + type(timer.get('calls')) is int and 0 < timer['calls'] < 10**9): + pairs.append((timer['mean_ms'], timer['calls'])) + calls = sum(c for _, c in pairs) + result[phase] = {'mean_ms': sum(m*c for m, c in pairs) / calls if calls else None, + 'ranks': len(pairs), 'expected_ranks': len(devices)} + return result + + +def summarize_node(raw, node, local_start, local_end): + result = {'label': node['label'], 'backend_port': node['backend_port'], 'received_at': local_end, + 'clock_ok': False, 'clock_uncertainty_seconds': None, 'source_ok': False, 'driver_ok': False, + 'prefill': None, 'decode': None, + 'hbm': None, 'phase_timers': None} + if not isinstance(raw, dict) or raw.get('schema') != 1: + return result + start, end, born = (raw.get(k) for k in ('started_at', 'finished_at', 'source_started_at')) + if (not finite(start) or not finite(end, minimum=start) or not finite(local_start) or + not finite(local_end, minimum=local_start) or end - start > 20 or + end - start > local_end - local_start + .1): + return result + # Remote start follows local send, and remote finish precedes local receipt. + # These two inequalities bound the unknown host clock offset without changing clocks. + low, high = end - local_end, start - local_start + if high < low: + return result + offset, uncertainty = (low + high) / 2, (high - low) / 2 + result['clock_ok'] = True + result['clock_uncertainty_seconds'] = uncertainty + result['probe_wall_seconds'] = end - start + cost = raw.get('probe_cpu_seconds') + result['probe_cpu_seconds'] = cost if finite(cost, minimum=0, maximum=20) else None + identity, host, sources = raw.get('source_id'), raw.get('host_fingerprint'), raw.get('source_sha256') + if (not isinstance(identity, str) or not SHA256.fullmatch(identity) or + not isinstance(host, str) or not SHA256.fullmatch(host) or + raw.get('image_fingerprint') != node['image_fingerprint'] or + sources != node['source_sha256'] or raw.get('source_before_start') is not True or + not finite(born, maximum=start)): + return result + result['source_ok'] = True + result['host_fingerprint'] = host + result['source_id'] = identity + result['driver_ok'] = raw.get('driver_sha256') == node['driver_sha256'] + devices = node['rank_devices'] + result['assumptions'] = {'prefill_flops_per_token': node['prefill_flops_per_token'], + 'peak_flops_per_device': node['peak_flops_per_device']} + hbm = raw.get('hbm') + if result['driver_ok'] and isinstance(hbm, list) and len(hbm) == len(devices): + values = {} + for item in hbm: + if not isinstance(item, dict): + continue + device, left, right, value = (item.get('id'), item.get('started_at'), item.get('finished_at'), + item.get('hbm_bandwidth_percent')) + if (type(device) is not int or device in values or device not in devices or + not finite(left, minimum=start, maximum=end) or not finite(right, minimum=left, maximum=end) or + not finite(value, minimum=0, maximum=100)): + continue + values[device] = (left, right, value) + if set(values) == set(devices): + first, last = min(v[0] for v in values.values()), max(v[1] for v in values.values()) + if last - first <= MAX_LOG_SKEW: + result['hbm'] = {'mean_percent': sum(v[2] for v in values.values()) / len(values), + 'device_count': len(values), 'sample_at': last - offset, + 'span_seconds': last - first, + 'valid_until': first - offset - uncertainty + HBM_MAX_AGE} + rows = raw.get('engine_rows') + groups = _phase_groups(rows, devices, born, end) + for phase in ('prefill', 'decode'): + result[phase] = groups.get(phase) + if result[phase] is not None: + result[phase]['log_at'] -= offset + result[phase]['valid_until'] = (result[phase]['log_at'] - result[phase]['log_skew_seconds'] - + uncertainty + MAX_AGE) + if result['prefill'] is not None: + rate = result['prefill']['tokens_per_second'] + efficiency = rate * node['prefill_flops_per_token'] / (len(devices) * node['peak_flops_per_device']) + result['prefill']['mfu_estimate'] = efficiency if finite(efficiency, minimum=0) else None + result['prefill']['peak_flops'] = len(devices) * node['peak_flops_per_device'] + steps = [v['step'] for v in groups.values()] + if steps and isinstance(rows, list): + latest = max(groups.values(), key=lambda value: value['log_at']) + result['phase_timers'] = _phase_timers(rows, latest['step'], devices, born, latest['_rank_at']) + result['phase_timers']['step'] = latest['step'] + for value in groups.values(): + value.pop('_rank_at', None) + return result + + +def summarize(nodes, sampled_at): + """Keep independent device groups separate and reject duplicate host identities.""" + nodes = deepcopy(nodes) + for node in nodes: + host, identity = node.get('host_fingerprint'), node.get('source_id') + duplicate = any(other is not node and ((host and other.get('host_fingerprint') == host) or + (identity and other.get('source_id') == identity)) for other in nodes) + if duplicate: + node.update(source_ok=False, duplicate_source=True, prefill=None, decode=None, hbm=None, phase_timers=None) + out = {'schema': 1, 'kind': 'deepinfer-telemetry', 'sampled_at': sampled_at, 'nodes': nodes, + 'prefill_tps': None, 'decode_tps': None, + 'prefill_mfu': None, 'hbm_percent': None} + if len(nodes) != 1 or not nodes[0].get('source_ok'): + return out + current = nodes[0] + for phase in ('prefill', 'decode'): + part = current.get(phase) + if isinstance(part, dict): + out[phase + '_tps'] = part['tokens_per_second'] + if phase == 'prefill': + out['prefill_mfu'] = part['mfu_estimate'] + if isinstance(current.get('hbm'), dict): + out['hbm_percent'] = current['hbm']['mean_percent'] + return out + + +def public_snapshot(value): + value = deepcopy(value) + for node in value.get('nodes', []): + node.pop('host_fingerprint', None) + node.pop('source_id', None) + for phase in ('prefill', 'decode'): + if isinstance(node.get(phase), dict): + node[phase].pop('_observation_id', None) + return value + + +def ssh_sample(node, script): + start = time.time() + try: + before = read_backend(node['backend_port']) + args = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5', '-o', 'ConnectionAttempts=1', + '-o', 'StrictHostKeyChecking=yes', node['ssh_target'], 'python3 - ' + node['container']] + result = subprocess.run(args, input=script, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + timeout=20, check=False, creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0)) + end = time.time() + if result.returncode or len(result.stdout) > 2 * 1024 * 1024: + return summarize_node(None, node, start, end) + sampled = summarize_node(json.loads(result.stdout), node, start, end) + after = read_backend(node['backend_port']) + ready = all(value.get('available') is True and value.get('admission_ready') == 1 and + value.get('workers_alive') == len(node['rank_devices']) for value in (before, after)) + sampled['backend_ready'] = ready + if not ready: + sampled.update(prefill=None, decode=None, phase_timers=None) + return sampled + except (OSError, UnicodeError, ValueError, subprocess.SubprocessError): + return summarize_node(None, node, start, time.time()) + + +class HistoryWorker: + """Slow storage can never block the live sampling or HTTP threads.""" + + def __init__(self, config, history=None, path=None): + self.configs = {node['backend_port']: node for node in config['nodes']} + self.history, self.path = history, path + self.queue = queue.Queue(maxsize=1) + self.lock = threading.Lock() + self.cache = None + self.dropped = 0 + self.changed = threading.Event() + self.stop_event = threading.Event() + self.thread = threading.Thread(target=self.run, daemon=True, name='telemetry-history') + self.thread.start() + + def submit(self, value): + try: + self.queue.put_nowait(deepcopy(value)) + except queue.Full: + with self.lock: + self.dropped += 1 + + def current(self): + with self.lock: + return deepcopy(self.cache), self.dropped + + def run(self): + while not self.stop_event.is_set(): + try: + value = self.queue.get(timeout=.2) + except queue.Empty: + continue + try: + if self.history is None: + self.history = open_history(self.path) + if self.history is None: + raise ValueError('TELEMETRY_HISTORY_UNAVAILABLE') + result = {} + for node in value['nodes']: + config = self.configs[node['backend_port']] + self.history.record(node, config, value['sampled_at']) + host = node.get('host_fingerprint') if node.get('source_ok') else None + result[node['backend_port']] = {'host': host, + 'history': self.history.series(config, host, value['sampled_at']) if host else None} + with self.lock: + self.cache = {'sampled_at': value['sampled_at'], 'nodes': result} + except (OSError, sqlite3.Error, ValueError, OverflowError): + with self.lock: + self.cache = None + finally: + self.changed.set() + self.queue.task_done() + + def close(self): + self.stop_event.set() + self.thread.join(timeout=1) + + +class Collector: + def __init__(self, config, sample=ssh_sample, history=None, history_path=None): + self.config = validate_config(config) + self.sample = sample + self.script = Path(__file__).with_name('remote_observe.py').read_bytes() + self.lock = threading.Lock() + self.value = summarize([], 0) + self.history_worker = (HistoryWorker(self.config, history=history, path=history_path) + if history is not None or history_path is not None else None) + self.stop_event = threading.Event() + + def collect(self): + with ThreadPoolExecutor(max_workers=len(self.config['nodes'])) as executor: + nodes = list(executor.map(lambda node: self.sample(node, self.script), self.config['nodes'])) + value = summarize(nodes, time.time()) + with self.lock: + self.value = value + if self.history_worker is not None: + self.history_worker.submit(value) + return value + + def snapshot(self): + with self.lock: + value = deepcopy(self.value) + now = time.time() + age = now - value['sampled_at'] + if not 0 <= age <= 2 * self.config['interval_seconds'] + 20: + return summarize([], 0) + for node in value['nodes']: + uncertainty = node.get('clock_uncertainty_seconds') + for phase in ('prefill', 'decode'): + part = node.get(phase) + if (part is not None and (not finite(uncertainty, minimum=0) or + not finite(part.get('valid_until')) or now >= part['valid_until'])): + node[phase] = None + part = node.get('hbm') + if (part is not None and (not finite(uncertainty, minimum=0) or + not finite(part.get('valid_until')) or now >= part['valid_until'])): + node['hbm'] = None + timers = node.get('phase_timers') + valid_steps = {node[p]['step'] for p in ('prefill', 'decode') if node.get(p) is not None} + if isinstance(timers, dict) and timers.get('step') not in valid_steps: + node['phase_timers'] = None + value = summarize(value['nodes'], value['sampled_at']) + cached, dropped = self.history_worker.current() if self.history_worker is not None else (None, 0) + history_ok = (isinstance(cached, dict) and finite(cached.get('sampled_at')) and + 0 <= now - cached['sampled_at'] <= 2 * self.config['interval_seconds'] + 20) + value['history_available'] = bool(history_ok) + value['history_dropped'] = dropped + if history_ok: + for node in value['nodes']: + entry = cached['nodes'].get(node['backend_port']) + node['history'] = (entry['history'] if entry and node.get('source_ok') and + entry['host'] == node.get('host_fingerprint') else None) + return value + + def close(self): + self.stop_event.set() + if self.history_worker is not None: + self.history_worker.close() + + def run(self): + while not self.stop_event.is_set(): + try: + self.collect() + except Exception: + with self.lock: + self.value = summarize([], 0) + self.stop_event.wait(self.config['interval_seconds']) + + +class Server(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = False + + def __init__(self, port, collector): + if type(port) is not int or not 0 <= port <= 65535: + raise ValueError('INVALID_PORT') + self.collector = collector + super().__init__(('127.0.0.1', port), Handler) + + def server_bind(self): + import socket + if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) + super().server_bind() + + def handle_error(self, request, client_address): + pass + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def do_GET(self): + hosts = self.headers.get_all('Host', []) + origins = self.headers.get_all('Origin', []) + allowed = {f'127.0.0.1:{self.server.server_port}', f'localhost:{self.server.server_port}'} + trusted = (len(hosts) == 1 and hosts[0].lower() in allowed and len(origins) <= 1 and + (not origins or origins[0].lower() == 'http://' + hosts[0].lower()) and + self.headers.get('Sec-Fetch-Site', '').lower() not in ('cross-site', 'same-site')) + if not trusted: + status, body = 403, b'Forbidden' + elif self.path != '/snapshot': + status, body = 404, b'Not found' + else: + status = 200 + body = json.dumps(public_snapshot(self.server.collector.snapshot()), allow_nan=False, separators=(',', ':')).encode() + self.send_response(status) + self.send_header('Content-Type', 'application/json' if status == 200 else 'text/plain') + self.send_header('Content-Length', str(len(body))) + self.send_header('Cache-Control', 'no-store') + self.end_headers() + self.wfile.write(body) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--config', required=True, type=Path) + parser.add_argument('--port', type=int, default=19883) + parser.add_argument('--history-database', type=Path) + parser.add_argument('--once', action='store_true') + args = parser.parse_args() + config = json.loads(args.config.read_text(encoding='utf-8')) + collector = Collector(config, history_path=args.history_database) + if args.once: + collector.collect() + if collector.history_worker is not None: + collector.history_worker.changed.wait(timeout=3) + print(json.dumps(public_snapshot(collector.snapshot()), allow_nan=False, separators=(',', ':'))) + collector.close() + return + server = Server(args.port, collector) + thread = threading.Thread(target=collector.run, daemon=True, name='telemetry-collector') + thread.start() + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + collector.close() + server.server_close() + + +def open_history(path): + if path is None: + return None + try: + return History(path) + except (OSError, sqlite3.Error, ValueError, OverflowError): + return None + + +if __name__ == '__main__': + main() diff --git a/deploy_stats/telemetry_history.py b/deploy_stats/telemetry_history.py new file mode 100644 index 0000000..e0c25c6 --- /dev/null +++ b/deploy_stats/telemetry_history.py @@ -0,0 +1,127 @@ +"""Persist bounded, device-group step-rate samples without inventing missing data.""" +import hashlib +import json +import math +import os +from contextlib import closing +from pathlib import Path +import re +import sqlite3 + + +APPLICATION_ID = 0x4C445448 +SCHEMA_VERSION = 1 +BUCKET_SECONDS = 600 +BUCKET_COUNT = 144 +SOURCE = 'deepinfer-log-step-approx-v1' +HASH = re.compile(r'[0-9a-f]{64}\Z') + + +def _finite(value, minimum=None, maximum=None): + try: + return (type(value) in (int, float) and math.isfinite(value) and + (minimum is None or value >= minimum) and (maximum is None or value <= maximum)) + except OverflowError: + return False + + +def _config_key(config): + keys = ('backend_port', 'ssh_target', 'container', 'rank_devices', 'source_sha256', + 'image_fingerprint', 'driver_sha256') + value = {key: config[key] for key in keys} + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest() + + +class History: + """One file, one writer; readers open their own SQLite connections.""" + + def __init__(self, path): + self.path = Path(path).resolve() + try: + fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) + except FileExistsError: + created = False + else: + os.close(fd) + created = True + with closing(self._connect()) as connection, connection: + if created: + connection.execute(f'PRAGMA application_id = {APPLICATION_ID}') + connection.execute(f'PRAGMA user_version = {SCHEMA_VERSION}') + connection.execute('''CREATE TABLE step_samples ( + config_key TEXT NOT NULL, + host_key TEXT NOT NULL, + source_id TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN ('prefill', 'decode')), + observation_id TEXT NOT NULL, + step INTEGER NOT NULL, + bucket INTEGER NOT NULL, + rate REAL NOT NULL, + PRIMARY KEY (config_key, host_key, source_id, phase, observation_id) + ) WITHOUT ROWID''') + connection.execute('CREATE INDEX step_samples_by_bucket ON step_samples (config_key, host_key, bucket)') + elif (connection.execute('PRAGMA application_id').fetchone()[0] != APPLICATION_ID or + connection.execute('PRAGMA user_version').fetchone()[0] != SCHEMA_VERSION): + raise ValueError('INVALID_TELEMETRY_HISTORY') + # Only an identified history file may be modified. + connection.execute('PRAGMA journal_mode = WAL') + + def _connect(self): + # mode=rw does not silently create a replacement if an existing database disappears. + return sqlite3.connect(self.path.as_uri() + '?mode=rw', uri=True, timeout=1) + + def record(self, node, config, now): + if (not isinstance(node, dict) or node.get('source_ok') is not True or + node.get('backend_ready') is not True or node.get('clock_ok') is not True or + node.get('backend_port') != config.get('backend_port') or + not isinstance(node.get('host_fingerprint'), str) or not HASH.fullmatch(node['host_fingerprint']) or + not isinstance(node.get('source_id'), str) or not HASH.fullmatch(node['source_id']) or + not _finite(now, minimum=0) or + not _finite(node.get('clock_uncertainty_seconds'), minimum=0, maximum=20)): + return 0 + uncertainty = node['clock_uncertainty_seconds'] + rows = [] + for phase in ('prefill', 'decode'): + part = node.get(phase) + if not isinstance(part, dict): + continue + at, skew, step, rate, until, identity = (part.get(k) for k in + ('log_at', 'log_skew_seconds', 'step', 'tokens_per_second', 'valid_until', '_observation_id')) + if (not _finite(at, minimum=0, maximum=now + uncertainty) or + not _finite(skew, minimum=0, maximum=5) or type(step) is not int or not 0 <= step < 2**63 or + not isinstance(identity, str) or not HASH.fullmatch(identity) or + not _finite(rate, minimum=0, maximum=1e15) or not _finite(until) or until <= now): + continue + first, last = at - skew - uncertainty, at + uncertainty + if first < 0 or int(first // BUCKET_SECONDS) != int(last // BUCKET_SECONDS): + continue + rows.append((_config_key(config), node['host_fingerprint'], node['source_id'], phase, + identity, step, int(first // BUCKET_SECONDS) * BUCKET_SECONDS, rate)) + if not rows: + return 0 + with closing(self._connect()) as connection, connection: + before = connection.total_changes + connection.executemany('INSERT OR IGNORE INTO step_samples VALUES (?, ?, ?, ?, ?, ?, ?, ?)', rows) + return connection.total_changes - before + + def series(self, config, host_key, now): + if not isinstance(host_key, str) or not HASH.fullmatch(host_key) or not _finite(now, minimum=0): + return None + end = int(now // BUCKET_SECONDS) * BUCKET_SECONDS + start = end - BUCKET_COUNT * BUCKET_SECONDS + if start < 0: + return None + with closing(self._connect()) as connection, connection: + rows = connection.execute('''SELECT bucket, phase, SUM(rate), COUNT(*) FROM step_samples + WHERE config_key = ? AND host_key = ? AND bucket >= ? AND bucket < ? + GROUP BY bucket, phase''', (_config_key(config), host_key, start, end)).fetchall() + grouped = {(bucket, phase): (total, count) for bucket, phase, total, count in rows} + values = [] + for bucket in range(start, end, BUCKET_SECONDS): + point = {'start': bucket, 'end': bucket + BUCKET_SECONDS} + for phase in ('prefill', 'decode'): + total, count = grouped.get((bucket, phase), (0, 0)) + point[phase + '_tps'] = total / count if count and _finite(total, minimum=0) else None + point[phase + '_samples'] = count + values.append(point) + return {'method': SOURCE, 'bucket_seconds': BUCKET_SECONDS, 'points': values} diff --git a/deploy_stats/web.py b/deploy_stats/web.py new file mode 100644 index 0000000..b766740 --- /dev/null +++ b/deploy_stats/web.py @@ -0,0 +1,182 @@ +"""Serve the read-only dashboard exclusively on IPv4 loopback.""" +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import socket +import sqlite3 +from urllib.parse import parse_qs, urlsplit + +from .backends import read_backend, read_telemetry +from .motor_metrics import MotorCollector, MotorHistory +from .report import active_report, report + +STATIC = Path(__file__).resolve().parent / 'static' +ASSETS = {'/': ('index.html', 'text/html; charset=utf-8'), + '/app.css': ('app.css', 'text/css; charset=utf-8'), + '/app-telemetry-v7.js': ('app.js', 'text/javascript; charset=utf-8')} + + +class Dashboard(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = False + + def __init__(self, port, database, backend_ports=(), telemetry_port=None, *, + motor_management_ports=(), motor_history_path=None): + if type(port) is not int or not 0 <= port <= 65535: + raise ValueError('INVALID_PORT') + self.database = Path(database) + if len(backend_ports) > 8 or any(type(p) is not int or not 1 <= p <= 65535 for p in backend_ports): + raise ValueError('INVALID_BACKEND_PORTS') + self.backend_ports = tuple(dict.fromkeys(backend_ports)) + if telemetry_port is not None and (type(telemetry_port) is not int or not 1 <= telemetry_port <= 65535): + raise ValueError('INVALID_TELEMETRY_PORT') + self.telemetry_port = telemetry_port + if motor_management_ports and (backend_ports or telemetry_port is not None): + raise ValueError('MIXED_BACKEND_SOURCES') + if motor_history_path is not None and not motor_management_ports: + raise ValueError('MOTOR_SOURCE_REQUIRED') + self.motor_collector = None + super().__init__(('127.0.0.1', port), Handler) + try: + history = MotorHistory(motor_history_path) if motor_history_path is not None else None + self.motor_collector = (MotorCollector(motor_management_ports, history=history) + if motor_management_ports else None) + if self.motor_collector is not None: + self.motor_collector.start() + except Exception: + super().server_close() + raise + + def server_bind(self): + if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) + super().server_bind() + + def handle_error(self, request, client_address): + pass + + def server_close(self): + if self.motor_collector is not None: + self.motor_collector.close() + super().server_close() + + def motor_snapshot(self): + value = self.motor_collector.snapshot() + value['available'] = value.get('sampled_at', 0) > 0 + return value + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def trusted(self): + hosts = self.headers.get_all('Host', []) + allowed = {f'127.0.0.1:{self.server.server_port}', f'localhost:{self.server.server_port}'} + if len(hosts) != 1 or hosts[0].lower() not in allowed: + return False + origins = self.headers.get_all('Origin', []) + if len(origins) > 1 or origins and origins[0].lower() != 'http://' + hosts[0].lower(): + return False + return self.headers.get('Sec-Fetch-Site', '').lower() not in ('cross-site', 'same-site') + + def send(self, status, body, content_type): + self.send_response(status) + self.send_header('Content-Type', content_type) + self.send_header('Content-Length', str(len(body))) + self.send_header('Cache-Control', 'no-store') + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Content-Security-Policy', "default-src 'self'; base-uri 'none'; frame-ancestors 'none'") + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if not self.trusted(): + self.send(403, b'Forbidden', 'text/plain; charset=utf-8') + return + url = urlsplit(self.path) + if (self.server.motor_collector is not None and + url.path in ('/api/telemetry', '/api/active', '/api/stats')): + self.send(409, b'{"error":"DASHBOARD_RELOAD_REQUIRED"}', 'application/json; charset=utf-8') + return + if url.path in ASSETS: + name, media = ASSETS[url.path] + try: + self.send(200, (STATIC / name).read_bytes(), media) + except OSError: + self.send(503, b'Unavailable', 'text/plain; charset=utf-8') + return + if url.path in ('/api/telemetry', '/api/v5/telemetry'): + if url.query: + self.send(400, b'{"error":"INVALID_TELEMETRY_QUERY"}', 'application/json; charset=utf-8') + return + value = (self.server.motor_snapshot() if self.server.motor_collector is not None + else read_telemetry(self.server.telemetry_port)) + body = json.dumps(value, allow_nan=False, separators=(',', ':')).encode() + self.send(200, body, 'application/json; charset=utf-8') + return + if url.path in ('/api/active', '/api/v5/active'): + if url.query: + self.send(400, b'{"error":"INVALID_ACTIVE_QUERY"}', 'application/json; charset=utf-8') + return + try: + value = active_report(self.server.database, source='gateway-inprocess-v1' + if self.server.motor_collector is not None else None) + body = json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(',', ':')).encode() + self.send(200, body, 'application/json; charset=utf-8') + except (ValueError, sqlite3.Error, OSError, OverflowError): + self.send(503, b'{"error":"STATS_UNAVAILABLE"}', 'application/json; charset=utf-8') + return + if url.path in ('/api/stats', '/api/v5/stats'): + values = parse_qs(url.query, keep_blank_values=True) + defaults = {'window': '24h', 'offset': '0', 'limit': '100', 'sort': 'finished', 'direction': 'desc', + 'session_offset': '0', 'session_limit': None, 'session_sort': 'last_finished', + 'session_direction': 'desc', 'snapshot_at': None, 'snapshot_id': None} + if set(values) - set(defaults) or any(len(value) != 1 for value in values.values()): + self.send(400, b'{"error":"INVALID_WINDOW"}', 'application/json; charset=utf-8') + return + try: + params = {key: values[key][0] if key in values else default for key, default in defaults.items()} + if params['session_limit'] is None: + params['session_limit'] = params['limit'] + if any(not params[key].isascii() or not params[key].isdigit() or len(params[key]) > 10 + for key in ('offset', 'limit', 'session_offset', 'session_limit')): + raise ValueError('INVALID_STATS_PAGE') + if params['snapshot_id'] is not None and (not params['snapshot_id'].isascii() or + not params['snapshot_id'].isdigit() or len(params['snapshot_id']) > 19): + raise ValueError('INVALID_STATS_SNAPSHOT') + if params['snapshot_at'] is not None and len(params['snapshot_at']) > 32: + raise ValueError('INVALID_STATS_SNAPSHOT') + value = report(self.server.database, params['window'], + request_offset=int(params['offset']), request_limit=int(params['limit']), + session_offset=int(params['session_offset']), session_limit=int(params['session_limit']), + request_sort=params['sort'], request_dir=params['direction'], + session_sort=params['session_sort'], session_dir=params['session_direction'], + snapshot_at=float(params['snapshot_at']) if params['snapshot_at'] is not None else None, + snapshot_id=int(params['snapshot_id']) if params['snapshot_id'] is not None else None, + source='gateway-inprocess-v1' if self.server.motor_collector is not None else None) + if self.server.motor_collector is not None: + value['collector_source'] = 'gateway-inprocess-v1' + if self.server.motor_collector is None: + value['backends'] = [read_backend(port) for port in self.server.backend_ports] + else: + value['backends'] = [] + for node in self.server.motor_snapshot().get('nodes', []): + metrics = node.get('engine_metrics') or {} + value['backends'].append({ + 'port': node['backend_port'], 'label': node['label'], + 'available': node['source_ok'] and bool(metrics), 'running': metrics.get('running'), + 'waiting': metrics.get('waiting'), 'workers_alive': metrics.get('observed_endpoints'), + 'workers_expected': metrics.get('expected_endpoints'), + 'admission_ready': int(node['backend_ready']), 'input_total': metrics.get('input_total'), + 'hit_rate': metrics.get('hit_rate'), 'partial': metrics.get('partial', True), + 'valid_until': metrics.get('valid_until'), + }) + body = json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(',', ':')).encode() + self.send(200, body, 'application/json; charset=utf-8') + except ValueError: + self.send(400, b'{"error":"INVALID_OR_UNAVAILABLE_STATS"}', 'application/json; charset=utf-8') + except (sqlite3.Error, OSError, OverflowError): + self.send(503, b'{"error":"STATS_UNAVAILABLE"}', 'application/json; charset=utf-8') + return + self.send(404, b'Not found', 'text/plain; charset=utf-8') diff --git a/docs/gateway.md b/docs/gateway.md index efba0fe..5304efc 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -20,6 +20,10 @@ python -m deploy_gateway --key-store local/gateway-keys.json --bindings-file loc DeepInfer 等自行调度 DP rank 的后端使用 `--backend-managed-affinity`：网关保留上述会话头和 `x-deepinfer-cache-namespace`，不注入 vLLM 的 `X-data-parallel-rank`。机器层绑定与原生 token 前缀缓存调度各自保留；机器故障或满载迁移后需要在新机器重新建立缓存。 +后端如在独立管理端口提供就绪检查，可重复设置 `--backend-readiness-port 19001=19101`，将推理端口映射到 loopback 管理端口。映射后网关请求管理端口的 `/readiness`，仅在 HTTP 200 且 JSON 的 `ready` 为 `true` 时认为该后端可用；其他后端继续使用其推理端口的 `/health`。MindIE Motor Coordinator 的推理和管理接口分离时应配置此映射，并使用 `--backend-managed-affinity`。 + +严格 P/D 分离部署还应设置 `--backend-pd-model `，并为每个后端配置管理端口。网关会同时读取 `/instances`，要求该模型至少有一个 `active`、`available` 且 `healthy` 的 P 和 D；请求失败、响应无效或任一角色缺失时，将该协调器视为不可用。Motor 的 `ready` 单独为真不保证 P、D 同时存在。再设置 `--backend-pd-probe-endpoints` 后，网关还会对候选实例报告的每个 rank 请求 `/health`；该模式要求网关主机能直接访问这些端点。Motor 报告的健康状态可能滞后，端点探测也不能证明实际推理或集体通信正常；同一协调器内的故障实例摘除和恢复仍需另行验证。管理端口须限制为本机访问；该检查不提供 Coordinator 自身的注册状态持久化或入口高可用。 + 重复 `--backend-port` 可配置任意数量的 loopback 后端，替代默认的两个端口。例如三个后端使用 `--backend-port 19001 --backend-port 19002 --backend-port 19008`。`--max-active-per-backend` 默认 32；全池同时等待上限由 `--max-waiting` 控制，默认 32，最长等待由 `--queue-timeout` 控制，默认 5 秒。选机和占用配额在同一把锁内完成；全池满载、队列满额或等待超时返回 429 和 `Retry-After: 1`，没有健康且准入的后端返回 503。该等待时限不包含模型执行时间。请求体最多 64 MiB，读取请求体也计入服务请求槽位；客户端断开时关闭对应上游连接并释放在途配额。 多个公开端口必须由同一个反向代理转发至同一网关进程，才能共享流控和会话绑定；不要为每个入口各启动一套网关。反向代理需保留 SSE 实时传输、客户端取消和认证头，关闭包含请求或认证材料的访问日志。可先启动一个新入口，验收后排空旧入口，再通过代理的平滑重载将旧端口作为别名加入。 diff --git a/docs/usage.md b/docs/usage.md index 4020466..c66f2de 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -40,3 +40,87 @@ SQLite 使用 WAL / NORMAL；已提交记录可跨普通进程退出保留，正 通知进程每约 30 秒在后台通过 SSH 只读查询，两台分别超时，查询不会阻塞健康观察。仅允许固定报告命令、已验证的路径与目标；远端结果仅提取有界数字，原始文本和异常不进入消息。以通知机本地接收时间判断缓存是否超过 90 秒，避免跨机时钟偏差造成误判。事件与窗口仍按部署机时间；与通知机的估计偏差超过 5 秒时显示提示，不自动修改系统时钟。 `@小鲁班 状态` 保留部署数量和健康状态，并按实例返回五个窗口的输入、输出、缓存、命中率、均速和生成速度。IP 仍只显示最后一段；查询游标、投递去重及失败不自动重发的合同不变。统计故障不等同于 vLLM 故障。 + +## 统一入口的 IP 用量与本机页面 + +**进程内采集候选仍在隔离验证，未经现场切换审查不要启用。** 候选在既有认证网关内观察已写出的响应，不另加 HTTP 转发跳；不改变原请求认证、后端选择或响应内容。采集记录器启动失败时网关按原路径运行，页面应显示采集失联；网关启动失败时记录器必须结束，避免空心跳。网关的拒绝体排空单独限制为最多 480 个在途连接、每连接最多 128 MiB 和 30 秒，超限或排空器无法接收时回到原关闭行为，不保证所有过载响应完整。切换前必须验证两个网关和监督进程的实际源码、端口及状态文件归属，由调用方暂停新请求，等待包括慢上传和排队在内的已接受请求全部结束后再关闭准入；否则关闭准入也可能拒绝已接受的请求。切换过程中仍可能短暂返回 502 或 503。Windows 独占绑定仅防止同地址监听冲突，不替代对全部监听地址、PID 和已接受连接的检查。原有入口如另有独立网关，需要分别验证和切换。 + +进程内候选从 Caddy 默认生成的单值 `X-Forwarded-For` 获取直接对端 IP，忽略客户端可自行提供的 `X-LLMDeploy-Client-IP`。仅在已核实 Caddy 未配置可信代理、未改写此头并确由该 Caddy 转发的入口上使用；配置变化须重新验证。头缺失、多值或非法时显示未知。Caddy 前还有其他代理时，显示的是其直接对端而非自动追溯最终客户端；本机进程直接连接网关时可以伪造该头，不能把来源 IP 当作安全身份。 + +候选的 32 MiB 限制是**保留原始采集缓冲**的合计，不是进程 RSS 上限；JSON 解码与 Python 对象会额外占用内存。单个事件或 JSON 响应超过 4 MiB、事件 data 行超过 4096、JSON 结构或转义工作超过保护上限，以及捕获资源不足时，相关统计保守显示未知，不影响原响应。超过 64 KiB 且未发现符合协议的最终用量候选的 SSE 事件可跳过解析：它会清除此前用量；只有之后重新观察到有效的协议结束用量及结束标记，才计量该请求。跳过之前若尚未确认首增量，首增量等待和速度也显示未知。此口径验证最终用量和结束顺序，不保证跳过的正文事件可解析；后端或协议版本变化时须重新核验。速度与原有账本一样是入口估算，不是引擎阶段实测。 + +`deploy_stats` 是独立的入口统计组件，适用于固定 DeepInfer recipe API 的 Chat Completions、Responses 和 Messages JSON/SSE。设计上它放在 Caddy 与**现有认证网关之间**，转发请求和响应，从正常结束的原生 `usage` 提取输入、输出及缓存命中。页面和采集代理都只监听 `127.0.0.1`。这是与上面的 vLLM Stat Logger 分开的账本，不能把两者直接相加，也不自动补齐启用前的历史。 + +**当前入口代理尚未通过部署审查，不要接入真实流量。** Windows 上，在请求体较大、网关延迟拒绝且未读取完整请求体时，连接复位可能导致错误响应正文截断。固定等待或预读不能保证保留未收到的字节。必须先修复并验证拒绝端关闭行为，或改用不增加此 HTTP 跳转的采集方式，再重新审查真实入口切换。 + +Chat Completions 的最终流式用量语义来自[固定版本的 recipe 源码](https://github.com/deepseek-ai/deepseek-recipe/blob/935817c66f5d6e64838f419692545f06787e485f/deepseek-recipe/src/protocol/openai/chat_completion/response/chunk_generator.rs)；更换后端或协议版本时须重新核对，不按旧版本假设填零。 + +页面展示今天、滚动 24 小时、7 天和累计的总 Token、输入、输出、缓存命中、窗口输出 tok/s 与各请求 IP；另有最近 24 小时和近 14 日的统计。**总 Token = 输入 + 输出**，缓存命中是输入子集。窗口输出 tok/s 为已计量输出 / 窗口墙钟秒数，包含空闲时间，不是单请求解码速度。所有 Token 按请求完成时间入桶；24 小时曲线显示完整的 10 分钟区间，左侧 24 小时汇总包含未满区间。这不是逐 token 生成时刻的曲线，长请求在结束前没有最终用量。 + +24 小时曲线展示输入和输出 Token、平均请求缓存命中率以及设备 Prefill 和 Decode 采样吞吐。每个区间的平均请求命中率是各已计量请求的 `缓存输入 / 输入` 的算术平均，输入为零的请求不计入；这与概览中 `缓存输入之和 / 输入之和` 的加权命中率不同。命中率汇总按请求数加权；设备吞吐曲线的计算和覆盖边界见下文。无可测样本显示未知；覆盖有缺口时均值仅代表已观测样本，不是全部请求的下限。 + +左侧导航分别打开用量概览、当前连接、会话记录、请求来源、每日用量和后端状态；窄屏时导航显示在页面顶部。当前连接、会话汇总、请求记录、请求来源和每日用量分别支持每页 10、20 或 50 条，默认每页 20 条；点击表头可切换升序和降序。会话及请求在完整结果上排序后分页，未知值排在已知值之后。翻页时保留查询快照，避免新请求导致重复或漏行；会话和请求任一不在首页时共用同一快照。请求来源和每日用量在非首页时分别保留自身快照；概览和当前连接页每约 2 秒通过只读轻量接口查询当前连接，连接随采样变化，翻页结果可能变动；失联或快照超过 3 秒时显示未知并回到第一页。顶部刷新会回到首页并获取新快照；后端状态表支持排序。 + +页面同时展示当前在途推理请求、可判定请求成功率和可见请求速率、入口观测的首个可见增量与代理内耗时 p95，以及按网关身份口径分组的会话汇总和逐请求记录。成功率只以已计量成功、明确失败和中断的请求为分母；结果未知的请求被排除，因此不是全部请求的成功率。指标选择参考 [vLLM 生产指标](https://docs.vllm.ai/en/latest/usage/metrics/)与 [NVIDIA 推理指标定义](https://docs.nvidia.com/nim/benchmarking/llm/metrics)。采集进程每秒把内存中的当前在途集合写成快照；快照超过 3 秒或心跳失联时显示未知。列表只覆盖经过本代理且请求头已识别的推理连接；客户端绕行、入口回退和切换前的连接不在其中。后端通常只在响应结束时提供用量，进行中的输入、输出、命中率和速度为未知而非零。 + +新采集的会话依次使用 `x-session-affinity`、`x-deepseek-harness-session-id`、`x-session-id`，以及 JSON 的 `prompt_cache_key` 或首条用户内容，按网关的认证主体和身份规则计算哈希指纹。指纹不是加密：知道同一 API key 的人仍可能通过候选内容比对低熵输入，账本和页面必须限制访问。请求体只在内存中短暂处理，不写入账本或日志；原始字节缓冲单请求最多 1 MiB、全进程合计最多 32 MiB，JSON 解码和规范化还会产生额外临时内存。超限、请求体完整识别之前中断或无有效身份时显示未知；请求体已识别后取消，仍可能保留会话指纹。由首条用户内容推导的身份与网关一样只是近似：不同会话的首条内容相同或历史被压缩时可能合并或变化。历史上未识别的会话由于未保存请求体，无法回填。会话汇总只包含可识别会话，未计量用量及覆盖缺口显示为下限；命中率只针对已计量请求。页面可翻页查看各会话和请求，不保存原始会话标识或请求正文。 + +当前后端没有导出逐请求引擎 prefill 或 decode 时间。对有非空文本、推理或工具增量的 SSE，代理可观测从入口接收请求到首个可见增量的时间，以及从首个增量到响应结束的时间；前者包含上传、排队、网络和首输出等，后者包含流式输出和响应收尾。页面提供（输入−缓存命中）/首增量等待、（输出−1）/首增量后时长及两段时间占比作为**入口估算**；首个增量可能包含多个 Token，后者不等于精确 decode tok/s。会话汇总的时间是可观测请求时间之和，并发时不能当作会话墙钟时间。非流式、没有可见增量或终止用量缺失时相关值显示未知。p50/p95 为窗口内可用样本的线性插值，不代表未覆盖请求。 + +会话汇总的 Prefill、Decode 入口估算分别是可测成功请求的未命中输入或输出减一之和，除以相应入口时间之和，按请求耗时加权而非并发总吞吐；总览首增量等待为可测成功请求的算术平均。Prefill 速度排除等待时间为零的样本，Decode 速度排除输出不超过 1 或后段时间为零的样本。缓存命中率为已计量请求的缓存输入之和 / 输入之和。无可测样本时显示未知。 + +总览可选接入独立的只读引擎和设备采集器。每个实例只有同一日志 step 收齐全部配置的 DP rank、阶段一致、设备映射正确且步速差和日志时间差在界限内时，才展示各 rank 当前步速率之和；这是日志采样近似，不是严格共同墙钟窗口的精确总吞吐。默认跨设备组视图仅相加有有效样本的实例，忽略缺失组并展示有效组数和配置组数，全部缺失时显示“—”；也可选择单个实例。各组并非同时测量，采样时间范围含误差超过 30 秒时显示跨度。跨组 Prefill-MFU 以有效组的设备峰值算力加权，驱动 HBM 百分比以有效组的设备数加权，不将百分比直接相加。日志时间是写出时间而非执行时间，异步队列可能延迟，且日志不包含 worker 启动标识；因此不能证明所展示 step 与当前工作进程的实时执行严格对应。推测解码和未暴露的回退路径均可能造成偏差，Decode 计数不保证等于客户端收到的 Token。 + +Prefill-MFU 采用上述 Prefill 采样吞吐乘以操作者配置的模型 FLOP/token，除以对应设备理论峰值之和，仍为模型假设下的阶段估算，不是硬件 FLOP 实测；未经明确计入的 Attention 等计算不包含在内。设备 HBM 带宽百分比来自驱动的设备级采样，可能包含其他工作负载或混合阶段；目前没有可验证的 Decode 阶段带宽归因，因此 Decode-MBU 显示未知，而不把设备整体带宽冒充纯 Decode。表格中的阶段步耗时来自引擎 CPU 计时日志及其有效 rank，包含调度等开销，不是每请求 GPU/NPU kernel 耗时。 + +采集器通过 SSH 只读获取严格数值化的引擎日志摘要和 DCMI 设备读数，不保存或向本机返回原始日志、请求内容或认证信息。需在私有配置中绑定后端端口、容器、rank 与设备映射、已核验的引擎源码与 native 哈希、镜像指纹和 DCMI 库哈希，以及模型 FLOP/token 和设备理论峰值；模型、版本或映射变化后应重新核对。SSH 使用现有密钥或代理，配置不放密码、密钥或其他凭据；机器标识和实际配置只放被忽略的私有目录。 + +[采集配置示例](../hosts/telemetry.example.json)中的模型参数和哈希均为占位值，不能用于真实估算。复制并核验私有配置后，可用 `python -m deploy_stats.telemetry --config <私有配置> --once` 做单次只读预检；持续采集使用 `--port` 指定 loopback 端口，看板通过 `dashboard --telemetry-port <端口>` 读取该采集器。可另用 `--history-database <私有 SQLite 路径>` 保存设备采样历史。每个完整的 10 分钟桶只取已收到的各步速率采样均值，不是该桶 Token 数除以墙钟时长；跨组显示有样本组的同桶采样均值之和，并标出每桶有效组数，采样时刻不一定重合。空心点和虚线表示部分组缺失。跨组曲线的 24 小时数字是有采样的桶均值，覆盖组数可能随桶变化；单组则按采样数加权。全部缺失的桶留空，启用前的设备历史无法补齐。配置或主机身份变化后只展示当前身份的历史；文件不会自动删除旧记录，需自行监控磁盘空间。存储缓慢或不可用时实时采样继续运行，但历史采样可能缺失。入口用量曲线仍来自入口账本。 + +这些数字仅来自客户端最终响应中可解析的用量。认证网关可能在响应头发出前重试另一后端；前一次失败尝试即使已经消耗 Token，也不会出现在最终响应的用量中，入口采集器无法感知或补计。取消或中断的请求也可能已在后端消耗 Token。因此，即使采集覆盖和可见响应完整，这些数值也不是后端物理总消耗或计费账本。 + +请求中断、上游错误、未知协议、缺失或互相矛盾的用量字段记为未计量，不当成零。Chat 流的用量后若再出现其他数据事件，或 Messages 流的用量后再出现非 `message_stop` 事件，先前快照不再视为最终用量；只有随后再次收到有效用量并正常结束，才计量该请求。无采集覆盖显示空缺；`≥` 表示已观测下限，命中率仅针对已计量请求。统计通过异步队列写盘，页面只展示已提交数据，可能滞后于请求完成。队列满时记录丢失数并将相关采集运行周期视为不完整；运行周期与心跳用于标识中断，队列丢失数可能包含所选窗口以外的时间。采集进程失联或异常退出而未正常排空时，其运行周期内的统计也保守标为不完整，因为上次心跳前仍可能存在未写盘事件。SQLite 使用 WAL / NORMAL，异常退出或断电可能丢失尾部；这不是故障下逐 token 零丢失的计费账本。 + +页面的运行覆盖只反映采集进程心跳，不证明 Caddy 已经切换或所有入口流量均经过该代理。启动代理但尚未切换、入口配置后来变化或客户端绕过入口时，仍可能有未记录流量；必须单独核验真实入口路由，不能将心跳 100% 当作入口完整性证明。示例配置将原认证网关作为回退入口：采集代理的连接失败或其他往返网络错误可能触发被动健康标记，使随后新请求暂时绕过采集器。回退期间的请求不会记入账本，即使采集进程随后恢复也无法补计。 + +采集代理最多同时处理 480 个连接；Windows 下分派到三个独立事件循环，超限时尝试返回 429 并标记统计缺口。对于带请求体的请求，代理在转发请求体前会等待上游提前返回的认证或准入拒绝：4 KiB 以下最多约 10ms，4 KiB 及以上最多约 100ms。Windows 定时器精度可能使等待提前结束，固定等待不能消除所有拒绝与上传之间的 TCP 竞态。上线前必须核对连接上限不低于实际网关需要的并发与等待配额，并实测对首输出延迟的影响；扩大服务池后需重新评估，不能把连接数当作模型吞吐能力。 + +账本只保留 IP、不可逆会话指纹、请求时间、HTTP 状态及数值用量，不保存认证信息、原始会话 ID、提示词、模型回答或原始 Token。IP、指纹和账本属于私有运行数据，目录需限制本机访问，只放忽略的 `local/` 或操作者自己的私有位置。旧版账本首次由采集器打开时会原地迁移并保留历史；旧记录没有阶段时间和会话指纹。页面拒绝跨站 Origin 和非 loopback Host，且不提供修改接口；同一台机器上有权限读取账本的用户仍可读取这些 IP。 + +可选 `--backend-port` 指定本机回环地址上的 DeepInfer 原生 `/metrics` 端口，每个端口单独读取运行数、排队数、存活 worker、接入状态、累计输入和累计缓存命中。后端不可达、计数缺项或累计值不一致时显示未知；累计数按各后端进程生命周期统计，重启会重置。当前接口没有 KV 占用、设备利用率或逐请求引擎阶段时延，不以估算代替。 + +## Motor 与 vLLM 指标 + +Motor 部署可在现有网关命令中增加 `--statistics-database <私有账本>`，将响应观察器装入网关进程；不修改客户端请求、流内容或上游认证。账本打开失败时网关仍提供服务，采集不可用。只在 Caddy 已验证覆盖客户端转发地址且网关确为入口目标时，来源 IP 才有入口侧的可信来源；直接访问网关的客户端不享有该保证。实际流量是否经过已装配的网关仍需在上线后独立核验，采集器心跳不能证明入口接线。 + +看板以 `dashboard --database <同一账本> --motor-management-port <管理端口> [--motor-management-port <管理端口>] --motor-history-database <新的私有历史文件>` 运行；Motor 模式不能同时使用旧的 `--backend-port` 或 `--telemetry-port`。管理端口必须通过本机回环隧道连接可信的 Motor 管理服务。看板每约 30 秒只读 `/instances` 和已注册 worker 的 `/metrics`，最多四个并发 worker 请求；采样前后核对实例角色、端点和就绪状态，管理端不一致、采样期间拓扑变化、worker 缺失或计数重置时保守标记部分或未知。两次拓扑读取一致仍不能证明整个间隔内没有发生过变化。速率按相同进程和相同指标序列的相邻计数差 / 墙钟间隔计算：Prefill 取本地实际计算输入，Decode 取生成 Token；两阶段不相加作为请求总量。后端计数的范围可能包含其他入口。 + +通过导出进程启动时间和计数回退识别重启，但不能保证发现导出进程未退出的所有内部引擎重启。真实阶段耗时仅为 vLLM 相邻采样间请求阶段直方图的聚合均值，不是单会话时间。Prefix Cache 命中率按 Prefill 侧 Token 加权，不能替代入口每请求命中率的算术平均，也不把 Decode 的跨实例 KV 传输当作用户前缀复用。历史文件按十分钟保存速率采样均值，仅显示当前实例身份的历史；历史开始之前、采集故障和 worker 变化期间不补造数据。当前未采集到可归因的 NPU 阶段算力和 HBM 流量，Prefill-MFU、Decode-MBU 和 HBM 利用率保持未知。 + +Decode 采集 vLLM 的草稿步数、草稿 Token 和接受的草稿 Token。推测解码草稿接受率 = 接受的草稿 Token / 草稿 Token；跨指标端点或设备组先分别相加分子和分母，再相除，不平均各组百分率。这些通用指标没有方法标签，可能是 MTP 或其他推测解码，不能仅凭指标断言方法。总览的累计值混合各进程各自启动以来的计数，进程重启会重置；最近 24 小时曲线则在现有私有历史库中按 10 分钟记录两次采样间的计数增量，归入采样结束时所在区间。历史增量不能证明整段时间连续覆盖，所以有数据的点也标记为时间覆盖未核验；计数重置、序列或模型不一致、无草稿、采集前的区间不补造数据。覆盖分母是当前 Motor 已注册的指标端点，不能证明包含所有 engine 或已离线设备组。后端计数可能包含其他入口，无法在并发下按请求、IP 或会话分摊；单请求 MTP 指标须由后端明确输出，不能用组计数差代替。 + +Motor 设备组的 vLLM 数据只读取与注册模型名称一致的指标标签；模型变化时重新建立采样基线，并使用独立历史身份。旧历史未绑定模型，保留在数据库中但不自动归属新模型。某些引擎不提供注册模型名或对应标签时，设备指标显示未知。 + +请求记录直接展示已保存的输入、输出和缓存命中。未命中输入仅在缓存明细已知时按输入减缓存命中计算；未知缓存不当作零。此值是入口响应口径的估算，不能保证等于引擎实际执行的 Prefill Token；后端 `local_compute` 计数是设备级数据，不可直接归因单个请求。老账本中已完整记录输入及缓存的请求可直接计算，无需改写历史行；部分用量或中断的请求不补造结果。 + +Motor 模式只用新网关进程记录的来源标记判断当前采集心跳，旧采集器的心跳不会冒充现网心跳。心跳覆盖不证明入口路由覆盖，因此该模式下总量均标为已观测下限；历史账本继续保留，未能验证的时段不宣称完整。如果最终响应提供输入和输出但没有缓存明细，仅记录这两项并标为部分用量；缓存命中率的分母只包含有缓存明细的请求。流式响应未提供最终用量时请求标记为未计量，不能从引擎聚合计数反推每 IP 或会话的用量。 + +Motor 模式使用 v7 页面脚本和 v5 接口；旧页面脚本的资源路径不可用，避免新表头与旧渲染逻辑混用。旧版页面继续轮询旧接口时会显示统计不可用，而不会把新计数器按旧日志口径展示。切换后用户需要刷新看板标签页。 + +下面是**待单独授权部署**时的示例；目录、端口和原网关地址须先核实，命令不会由开发或测试自动执行： + +```powershell +New-Item -ItemType Directory -Path local/stats -Force +python -m deploy_stats proxy --port 19882 --upstream-port 19010 --database local/stats/usage.sqlite3 +python -m deploy_stats dashboard --port 19881 --database local/stats/usage.sqlite3 --backend-port 19001 --backend-port 19002 +``` + +示例中的 `19010` 仅代表既有网关的 loopback 端口。[Caddy 示例](../services/deepinfer/Caddyfile.stats.example)仅供审阅。先在隔离环境验证代理，再按维护流程核实入口及现有连接，将 Caddy 的 `reverse_proxy` 首选目标改为 `127.0.0.1:19882`，连接失败时回退到同一个既有认证网关，并在该反代块内设置： + +```caddyfile +header_up X-LLMDeploy-Client-IP {remote_host} +``` + +Caddy 必须覆盖客户端传入的同名头；采集器不信任客户端的 `X-Forwarded-For`，缺失、重复或非法来源显示为“未知来源”。`{remote_host}` 是 Caddy 的直接对端：若 Caddy 前还有其他代理，显示的是该代理 IP，需先明确可信代理链，不能自行相信转发头。回退只在尚未连接采集代理时重试 POST；不要配置会在已连接后重放 POST 的 `lb_retry_match`，不要对认证入口配置无凭据的主动健康探测。参见 Caddy 的[负载均衡与重试说明](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy#load-balancing)、[反向代理头部说明](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy#headers)和[请求占位符](https://caddyserver.com/docs/caddyfile/concepts#placeholders)。所有入口别名须采用相同配置，直接回退网关的请求不在统计范围。 + +切换前保留原 Caddy 配置，验证认证失败控制、JSON/SSE 字节内容、并发、取消、大请求、来源 IP 和页面；核对源版本与真实用量字段，检查记录器心跳及缺失数。验证通过才恢复通知。回退时把 Caddy 的目标恢复到原网关，保留账本并停止本次采集/页面进程；回退期间形成统计空缺，不得视为零。上述均需另行授权，离线测试和合成预览不构成实际部署或真机验收。 diff --git a/hosts/telemetry.example.json b/hosts/telemetry.example.json new file mode 100644 index 0000000..57725a6 --- /dev/null +++ b/hosts/telemetry.example.json @@ -0,0 +1,24 @@ +{ + "interval_seconds": 30, + "nodes": [ + { + "label": "示例实例", + "backend_port": 19001, + "ssh_target": "operator@example.invalid", + "container": "example-container", + "image_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000", + "driver_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "source_sha256": { + "stats": "0000000000000000000000000000000000000000000000000000000000000000", + "backend": "0000000000000000000000000000000000000000000000000000000000000000", + "parallel": "0000000000000000000000000000000000000000000000000000000000000000", + "engine": "0000000000000000000000000000000000000000000000000000000000000000", + "sampler": "0000000000000000000000000000000000000000000000000000000000000000", + "native": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "rank_devices": [0], + "prefill_flops_per_token": 1, + "peak_flops_per_device": 1 + } + ] +} diff --git a/services/deepinfer/Caddyfile.stats.example b/services/deepinfer/Caddyfile.stats.example new file mode 100644 index 0000000..2b4e582 --- /dev/null +++ b/services/deepinfer/Caddyfile.stats.example @@ -0,0 +1,19 @@ +# Example only: both ingress aliases pass through one local statistics proxy. +# Configure exposure/TLS separately and preserve the existing authenticated gateway. +:19000, :19003 { + bind 127.0.0.1 + request_body { + max_size 64MB + } + reverse_proxy 127.0.0.1:19882 127.0.0.1:19010 { + # Keep the original authenticated gateway available if the collector is + # marked unhealthy; requests on the fallback are not accounted. + lb_policy first + fail_duration 2s + lb_try_duration 6s + lb_try_interval 20ms + # Override any client-supplied value with Caddy's direct peer address. + header_up X-LLMDeploy-Client-IP {remote_host} + flush_interval 10ms + } +} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8966260 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Local test package and shared integration helpers.""" diff --git a/tests/caddy_helpers.py b/tests/caddy_helpers.py new file mode 100644 index 0000000..fd94db1 --- /dev/null +++ b/tests/caddy_helpers.py @@ -0,0 +1,50 @@ +import os +from pathlib import Path +import subprocess +import unittest + + +def synthetic_caddy_owns_listener(process, port): + """Check ownership before sending requests to a released test port.""" + if process.poll() is not None: + return False + if os.name == 'nt': + result = subprocess.run(['netstat', '-ano', '-p', 'tcp'], capture_output=True, + timeout=10, creationflags=subprocess.CREATE_NO_WINDOW) + if result.returncode: + return False + listeners = [] + for line in result.stdout.decode('ascii', errors='replace').splitlines(): + fields = line.split() + if len(fields) != 5 or fields[0] != 'TCP' or fields[3] != 'LISTENING': + continue + address, separator, candidate_port = fields[1].rpartition(':') + if separator and candidate_port.isdigit() and int(candidate_port) == port: + listeners.append((address, fields[4])) + return listeners == [('127.0.0.1', str(process.pid))] and process.poll() is None + proc = Path('/proc') + if not (proc/'net/tcp').is_file(): + raise unittest.SkipTest('synthetic Caddy listener ownership cannot be verified') + listeners = [] + for name in ('tcp', 'tcp6'): + path = proc/'net'/name + if not path.is_file(): + continue + for line in path.read_text().splitlines()[1:]: + fields = line.split() + address, candidate_port = fields[1].split(':') + if int(candidate_port, 16) == port and fields[3] == '0A': + listeners.append((name, address, fields[9])) + if len(listeners) != 1 or listeners[0][:2] != ('tcp', '0100007F'): + return False + try: + descriptors = list((proc/str(process.pid)/'fd').iterdir()) + except OSError: + return False + for descriptor in descriptors: + try: + if os.readlink(descriptor) == f'socket:[{listeners[0][2]}]': + return process.poll() is None + except OSError: + continue + return False diff --git a/tests/test_dashboard_efficiency.py b/tests/test_dashboard_efficiency.py new file mode 100644 index 0000000..835dc0c --- /dev/null +++ b/tests/test_dashboard_efficiency.py @@ -0,0 +1,538 @@ +import shutil +import subprocess +import unittest +from pathlib import Path + + +class DashboardEfficiencyTests(unittest.TestCase): + def test_motor_backend_row_expires_without_waiting_for_next_poll(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'), fs = require('node:fs'), vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const start = source.indexOf('function renderBackends('), end = source.indexOf('\n}\n', start); +assert(start >= 0 && end > start); +let now = 100, timer; +const body = {children: [], replaceChildren() {this.children=[];}, appendChild(row) {this.children.push(row);}}; +const context = {backendExpiry: null, Date: class extends Date {static now() {return now * 1000;}}, + $: () => body, localPage: (_name, rows) => rows, document: {createElement: () => ({cells: []})}, + cell: (row, value) => row.cells.push(value), count: String, percent: String, empty: () => {}, + clearTimeout: () => {}, setTimeout: (fn) => {timer=fn; return 1;}}; +const render = vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +const data = {backends: [{port: 1, label: 'fixture', available: true, valid_until: 105, + running: 4, waiting: 0, workers_alive: 1, workers_expected: 1, admission_ready: 1, input_total: 10, hit_rate: .5}]}; +render(data); assert.equal(body.children[0].cells[1], '4'); +now = 106; timer(); assert.equal(body.children[0].cells[1], '不可用'); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, encoding='utf-8', timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_partial_usage_displays_input_output_but_not_unknown_cache(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'), fs = require('node:fs'), vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const start = source.indexOf('function display('), end = source.indexOf('\n}\n', start); +assert(start >= 0 && end > start); +const display = vm.runInNewContext('(' + source.slice(start, end + 2) + ')', + {number: {format: String}, integer: {format: String}}); +const partial = {known: 0, partial: 1, complete: false}; +assert.equal(display(120, partial), '≥120'); +assert.equal(display(20, partial), '≥20'); +assert.equal(display(0, partial, false, false), '—'); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_motor_rates_count_only_relevant_phase_groups_and_label_partial_workers(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'), fs = require('node:fs'), vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +function extract(name, context) { + const start = source.indexOf(`function ${name}(`), end = source.indexOf('\n}\n', start); + assert(start >= 0 && end > start); return vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +} +const context = {}; +context.freshTelemetry = extract('freshTelemetry', context); +const values = extract('telemetryValues', context); +const base = {source_ok: true, backend_ready: true, clock_ok: true, clock_uncertainty_seconds: 0, expected_endpoints: 8}; +const part = (rate, valid, partial=false) => ({tokens_per_second: rate, valid_endpoints: valid, + expected_endpoints: 8, partial, log_at: 100, log_skew_seconds: 30, valid_until: 150}); +const data = {available: true, kind: 'vllm-motor-telemetry', nodes: [ + {...base, backend_port: 1, role: 'prefill', prefill: part(20, 7, true)}, + {...base, backend_port: 2, role: 'decode', decode: part(70, 8)}]}; +const result = values(data, 'all', 110); +assert.equal(result.prefill, 20); assert.equal(result.decode, 70); +assert.equal(result.coverage.prefill.total, 1); assert.equal(result.coverage.decode.total, 1); +assert.equal(result.coverage.prefill.workers_valid, 7); assert.equal(result.coverage.prefill.workers_total, 8); +assert.equal(result.coverage.prefill.partial, true); assert.equal(result.coverage.decode.partial, false); +assert.equal(result.mfu, null); assert.equal(result.hbm, null); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_old_and_new_phase_cards_cannot_cross_write(self): + from deploy_stats.web import ASSETS + root = Path(__file__).resolve().parents[1]/'deploy_stats/static' + html = (root/'index.html').read_text(encoding='utf-8') + script = (root/'app.js').read_text(encoding='utf-8') + legacy_ids = ('overview-prefill-tps', 'overview-prefill-samples', 'overview-decode-tps', + 'overview-decode-samples', 'overview-mfu', 'overview-mfu-detail', + 'overview-mbu', 'overview-mbu-detail', 'overview-efficiency-note') + for legacy in legacy_ids: + self.assertNotIn(f'id="{legacy}"', html) + self.assertNotIn(f"'{legacy}'", script) + self.assertIn('/app-telemetry-v7.js', html) + self.assertIn('/app-telemetry-v7.js', ASSETS) + self.assertNotIn('/app-telemetry-v6.js', ASSETS) + self.assertNotIn('/app-telemetry-v5.js', ASSETS) + self.assertNotIn('/app-telemetry-v4.js', ASSETS) + self.assertNotIn('/app-telemetry-v3.js', ASSETS) + self.assertNotIn('/app-telemetry-v2.js', ASSETS) + self.assertIn('chart-device-prefill', html) + self.assertIn('chart-device-decode', html) + self.assertIn('value="all">跨设备组（全部）', html) + self.assertIn("let telemetryNodePort = 'all'", script) + self.assertNotIn('id="chart-prefill"', html) + self.assertNotIn('id="chart-decode"', html) + self.assertNotIn('/app-trends-v1.js', ASSETS) + self.assertNotIn('/app.js', ASSETS) + + def test_telemetry_values_preserve_unknown_and_real_zero(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const freshStart = source.indexOf('function freshTelemetry('); +const freshEnd = source.indexOf('\n}\n', freshStart); +assert(freshStart >= 0 && freshEnd > freshStart); +const freshTelemetry = vm.runInNewContext('(' + source.slice(freshStart, freshEnd + 2) + ')'); +const start = source.indexOf('function telemetryValues('); +const end = source.indexOf('\n}\n', start); +assert(start >= 0 && end > start); +const read = vm.runInNewContext('(' + source.slice(start, end + 2) + ')', {freshTelemetry}); +let data = {available: true, nodes: [{backend_port: 1, source_ok: true, backend_ready: true, + clock_ok: true, clock_uncertainty_seconds: .1, + prefill: {tokens_per_second: 0, mfu_estimate: .25, valid_until: 101}, + decode: {tokens_per_second: 100, valid_until: 101}, hbm: {mean_percent: 35, valid_until: 101}}]}; +let value = read(data, 1, 100); +assert.equal(value.prefill, 0); +assert.equal(value.decode, 100); +assert.equal(value.mfu, .25); +assert.equal(value.hbm, 35); +value = read({available: true, nodes: [{backend_port: 1, source_ok: true, backend_ready: true, + clock_ok: true, clock_uncertainty_seconds: .1, + prefill: {tokens_per_second: null, mfu_estimate: -1, valid_until: 101}, + decode: {tokens_per_second: Infinity, valid_until: 101}, hbm: {mean_percent: 101, valid_until: 101}}]}, 1, 100); +for (const key of ['prefill', 'decode', 'mfu', 'hbm']) assert.equal(value[key], null); +assert.equal(read(data, 1, 101).prefill, null); +assert.equal(read(data, 2, 100).prefill, null); +assert.equal(read({available: false, nodes: data.nodes}, 1, 100).prefill, null); +data.nodes[0].backend_ready = false; +value = read(data, 1, 100); +assert.equal(value.prefill, null); assert.equal(value.decode, null); assert.equal(value.mfu, null); +assert.equal(value.hbm, 35); // Device reading remains separately valid. +data.nodes[0].backend_ready = true; data.nodes[0].clock_ok = false; +value = read(data, 1, 100); +for (const key of ['prefill', 'decode', 'mfu', 'hbm']) assert.equal(value[key], null); +data.nodes[0].clock_ok = true; data.nodes[0].clock_uncertainty_seconds = null; +assert.equal(read(data, 1, 100).decode, null); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_backend_table_does_not_show_unready_or_clock_invalid_rates(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'), fs = require('node:fs'), vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +function extract(name, ctx) { + const start = source.indexOf(`function ${name}(`), end = source.indexOf('\n}\n', start); + assert(start >= 0 && end > start); return vm.runInNewContext('(' + source.slice(start, end + 2) + ')', ctx); +} +const body = {children: [], replaceChildren() {this.children = [];}, appendChild(row) {this.children.push(row);}}; +const context = {telemetrySnapshot: null, telemetryExpiry: null, telemetryNodePort: 'all', telemetryOptionsKey: null, + Date: class extends Date {static now() {return 100000;}}, clearTimeout: () => {}, setTimeout: () => 1, + $: (id) => id === 'telemetry-body' ? body : null, document: {createElement: () => ({cells: []})}, + cell: (row, value) => row.cells.push(value), setTextIfPresent: () => {}, + number: {format: String}, speed: (v) => `${v} tok/s`, percent: (v) => `${v * 100}%`, + clock: {format: () => 'time'}, combinedDeviceHistory: () => null, renderDeviceHistory: () => {}, + latestGroupHistory: () => null, historicalPrefillMfu: () => null, renderMtp: () => {}, validMtp: () => null}; +context.freshTelemetry = extract('freshTelemetry', context); context.telemetryValues = extract('telemetryValues', context); +const render = extract('renderTelemetry', context); +const n = {backend_port: 1, source_ok: true, backend_ready: true, clock_ok: true, clock_uncertainty_seconds: .1, + prefill: {tokens_per_second: 20, mfu_estimate: .2, log_at: 95, log_skew_seconds: 0, valid_until: 110}, + decode: {tokens_per_second: 100, log_at: 95, log_skew_seconds: 0, valid_until: 110}, + hbm: {mean_percent: 30, device_count: 1, sample_at: 95, span_seconds: 0, valid_until: 110}}; +const data = {available: true, sampled_at: 100, nodes: [n]}; +render(data); assert.match(body.children[0].cells[2], /^100 tok\/s/); +n.backend_ready = false; render(data); +assert.equal(body.children[0].cells[1], '—'); assert.equal(body.children[0].cells[2], '—'); +assert.equal(body.children[0].cells[4], '30% · 1 卡'); +n.backend_ready = true; n.clock_ok = false; render(data); +assert.equal(body.children[0].cells[2], '—'); assert.equal(body.children[0].cells[4], '—'); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_device_history_uses_only_device_samples_and_preserves_gaps(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const start = source.indexOf('function renderDeviceHistory('); +const end = source.indexOf('\n}\n', start); +assert(start >= 0 && end > start); +const elements = {}; +function makeElement() { return {textContent: '', children: [], replaceChildren() {this.children=[];}, appendChild(v) {this.children.push(v);}}; } +for (const phase of ['prefill', 'decode']) { + elements[`chart-device-${phase}`] = makeElement(); + elements[`chart-device-${phase}-svg`] = makeElement(); +} +const context = {$: (id) => elements[id], svgElement: () => makeElement(), setTextIfPresent: () => {}, attachTrendTooltip: () => {}, + clock: {format: () => 'time'}, dateTime: {format: () => 'date'}, speed: (v) => `${v.toFixed(2)} tok/s`, count: String}; +const render = vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +const points = Array.from({length: 144}, (_, i) => ({start: i*600, end: (i+1)*600, + prefill_tps: null, decode_tps: null, prefill_samples: 0, decode_samples: 0, prefill_estimate_tps: 99999})); +points[1].prefill_tps = 0; points[1].prefill_samples = 1; +points[3].prefill_tps = 3000; points[3].prefill_samples = 2; +const current = {history: {method: 'deepinfer-log-step-approx-v1', bucket_seconds: 600, points}}; +render({available: true, history_available: true}, current); +assert.equal(elements['chart-device-prefill'].textContent, '2000.00 tok/s · 3 个采样'); +assert.equal(elements['chart-device-decode'].textContent, '暂无设备采样'); +const lines = elements['chart-device-prefill-svg'].children.filter((x) => x.children.length > 0); +assert.equal(lines.length, 2); // Two isolated markers, no line across the missing bucket. +render({available: true, history_available: false}, current); +assert.equal(elements['chart-device-prefill'].textContent, '设备历史采样未连接'); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_cross_group_values_ignore_missing_and_weight_only_valid_groups(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +function extract(name, context) { + const start = source.indexOf(`function ${name}(`), end = source.indexOf('\n}\n', start); + assert(start >= 0 && end > start); + return vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +} +const freshTelemetry = extract('freshTelemetry', {}); +const read = extract('telemetryValues', {freshTelemetry}); +function group(port, rate, mfu, peak, devices, hbm, at = 95) { + return {backend_port: port, source_ok: true, backend_ready: true, clock_ok: true, clock_uncertainty_seconds: .1, + prefill: {tokens_per_second: rate, mfu_estimate: mfu, peak_flops: peak, log_at: at, log_skew_seconds: 0, valid_until: 110}, + decode: {tokens_per_second: rate * 2, log_at: at, log_skew_seconds: 0, valid_until: 110}, + hbm: {mean_percent: hbm, device_count: devices, sample_at: at, span_seconds: 0, valid_until: 110}}; +} +const a = group(1, 100, .2, 100, 2, 20), b = group(2, 300, .6, 300, 6, 60); +let data = {available: true, nodes: [a, b]}; +let value = read(data, 'all', 100); +assert.equal(value.prefill, 400); assert.equal(value.decode, 800); +assert(Math.abs(value.mfu - .5) < 1e-12); assert.equal(value.hbm, 50); +assert.equal(value.coverage.prefill.valid, 2); assert.equal(value.coverage.prefill.total, 2); +assert.equal(read(data, 1, 100).prefill, 100); +b.prefill.tokens_per_second = 0; assert.equal(read(data, 'all', 100).prefill, 100); +b.prefill.log_at = 60; value = read(data, 'all', 100); +assert.equal(value.prefill, 100); assert(value.coverage.prefill.span_seconds > 30); +b.prefill.log_at = 95; b.decode = null; value = read(data, 'all', 100); +assert.equal(value.decode, 200); assert.equal(value.coverage.decode.valid, 1); +b.source_ok = false; value = read(data, 'all', 100); +assert.equal(value.prefill, 100); assert.equal(value.coverage.prefill.valid, 1); +assert.equal(value.coverage.prefill.total, 2); assert.equal(value.mfu, .2); assert.equal(value.hbm, 20); +b.source_ok = true; b.prefill = null; value = read(data, 'all', 100); +assert.equal(value.prefill, 100); assert.equal(value.mfu, .2); +assert.equal(value.coverage.mfu.valid, 1); assert.equal(value.hbm, 50); +b.backend_ready = false; value = read(data, 'all', 100); +assert.equal(value.hbm, 50); assert.equal(value.coverage.hbm.valid, 2); +b.backend_ready = true; +a.prefill.tokens_per_second = 0; value = read(data, 'all', 100); +assert.equal(value.prefill, 0); assert.equal(value.coverage.prefill.valid, 1); +a.prefill.valid_until = 100; value = read(data, 'all', 100); +assert.equal(value.prefill, null); assert.equal(value.coverage.prefill.valid, 0); +b.source_ok = true; b.backend_port = 1; assert.equal(read(data, 'all', 100).prefill, null); +assert.equal(read({available: false, nodes: [a]}, 'all', 100).prefill, null); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_overview_shows_partial_coverage_and_no_fake_decode_mbu(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +function extract(name, context) { + const start = source.indexOf(`function ${name}(`), end = source.indexOf('\n}\n', start); + assert(start >= 0 && end > start); + return vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +} +const elements = {}; +for (const id of ['live-prefill-tps', 'live-decode-tps', 'live-mfu', 'live-mbu', 'live-hbm', + 'live-prefill-samples', 'live-decode-samples', 'live-mfu-detail', 'live-hbm-detail']) elements[id] = {textContent: ''}; +const context = {telemetrySnapshot: null, telemetryExpiry: null, telemetryNodePort: 'all', telemetryOptionsKey: null, + Date: class extends Date {static now() {return 100000;}}, clearTimeout: () => {}, + $: (id) => elements[id] || null, setTextIfPresent: (id, value) => {if (elements[id]) elements[id].textContent = value;}, + number: {format: String}, speed: (v) => `${v} tok/s`, percent: (v) => `${v * 100}%`, + clock: {format: () => 'time'}, combinedDeviceHistory: () => null, renderDeviceHistory: () => {}, + latestGroupHistory: () => null, historicalPrefillMfu: () => null, renderMtp: () => {}, validMtp: () => null}; +context.freshTelemetry = extract('freshTelemetry', context); +context.telemetryValues = extract('telemetryValues', context); +const render = extract('renderTelemetry', context); +const base = {source_ok: true, backend_ready: true, clock_ok: true, clock_uncertainty_seconds: .1}; +const a = {...base, backend_port: 1, prefill: {tokens_per_second: 100, mfu_estimate: .2, peak_flops: 100, + log_at: 95, log_skew_seconds: 0, valid_until: 110}, decode: null, + hbm: {mean_percent: 20, device_count: 2, sample_at: 95, span_seconds: 0, valid_until: 110}}; +const b = {...base, backend_port: 2, prefill: null, decode: null, hbm: null}; +render({available: true, sampled_at: 100, nodes: [a, b]}); +assert.equal(elements['live-prefill-tps'].textContent, '100 tok/s'); +assert.match(elements['live-prefill-samples'].textContent, /1\/2 组有样本 · 部分数据/); +assert.match(elements['live-mfu-detail'].textContent, /1\/2 组有样本 · 部分数据/); +assert.equal(elements['live-decode-tps'].textContent, '—'); +assert.match(elements['live-decode-samples'].textContent, /0\/2 组有样本/); +assert.equal(elements['live-mbu'].textContent, '—'); +assert.equal(elements['live-hbm'].textContent, '20%'); +assert.match(elements['live-hbm-detail'].textContent, /1\/2 组有样本 · 部分数据/); +render({available:false,nodes:[]}); +assert.equal(elements['live-prefill-tps'].textContent, '—'); +assert.equal(elements['live-prefill-samples'].textContent, '采集未连接 · 组数未知'); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_overview_adds_clearly_labeled_history_only_when_live_samples_are_missing(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'), fs = require('node:fs'), vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +function extract(name, ctx) { + const start = source.indexOf(`function ${name}(`), end = source.indexOf('\n}\n', start); + assert(start >= 0 && end > start, `missing ${name}`); return vm.runInNewContext('(' + source.slice(start, end + 2) + ')', ctx); +} +const elements = {}; +for (const id of ['live-prefill-tps','live-decode-tps','live-mfu','live-mbu','live-hbm', + 'live-prefill-samples','live-decode-samples','live-mfu-detail','live-hbm-detail', + 'live-prefill-title','live-decode-title','live-mfu-title']) elements[id] = {textContent:''}; +const context = {telemetrySnapshot:null,telemetryExpiry:null,telemetryNodePort:'all',telemetryOptionsKey:null, + Date:class extends Date {static now() {return 86400000;}},clearTimeout:()=>{},setTimeout:()=>1, + $:(id)=>elements[id]||null,setTextIfPresent:(id,value)=>{if(elements[id])elements[id].textContent=value;}, + number:{format:String},speed:(v)=>`${v} tok/s`,percent:(v)=>v===null||v===undefined?'—':`${v*100}%`, + clock:{format:(v)=>`clock-${v.getTime()/1000}`},dateTime:{format:(v)=>`date-${v.getTime()/1000}`}, + renderDeviceHistory:()=>{},renderMtp:()=>{},validMtp:()=>null}; +for (const name of ['freshTelemetry','telemetryValues','combinedDeviceHistory','latestHistoricalSample','latestGroupHistory','historicalPrefillMfu']) + context[name]=extract(name,context); +const render=extract('renderTelemetry',context); +const points=Array.from({length:144},(_,i)=>({start:i*600,end:(i+1)*600, + prefill_tps:null,prefill_samples:0,decode_tps:null,decode_samples:0})); +points[142].prefill_tps=100;points[142].prefill_samples=2; +points[143].decode_tps=40;points[143].decode_samples=1; +const a={backend_port:1,source_ok:true,backend_ready:true,clock_ok:true,clock_uncertainty_seconds:.1, + prefill:null,decode:null,hbm:{mean_percent:0,device_count:2,sample_at:86399,span_seconds:0,valid_until:86401}, + assumptions:{prefill_flops_per_token:4,peak_flops_per_device:1000}, + history:{method:'deepinfer-log-step-approx-v1',bucket_seconds:600,points}}; +const b={backend_port:2,source_ok:false,backend_ready:false,clock_ok:true,clock_uncertainty_seconds:.1}; +const data={available:true,history_available:true,sampled_at:86400,nodes:[a,b]}; +render(data); +assert.equal(elements['live-prefill-tps'].textContent,'100 tok/s'); +assert.equal(elements['live-decode-tps'].textContent,'40 tok/s'); +assert.equal(elements['live-mfu'].textContent,'20%'); +assert.match(elements['live-prefill-title'].textContent,/历史/); +assert.match(elements['live-decode-title'].textContent,/历史/); +assert.match(elements['live-mfu-title'].textContent,/历史/); +assert.match(elements['live-prefill-samples'].textContent,/历史.*非实时.*1\/2/); +assert.match(elements['live-decode-samples'].textContent,/历史.*非实时.*1\/2/); +assert.match(elements['live-mfu-detail'].textContent,/当前假设.*非实时.*1\/2/); +assert.equal(elements['live-mbu'].textContent,'—'); +// Old HTML loaded across a dashboard switch lacks the new title nodes. +const savedTitles={}; +for(const id of ['live-prefill-title','live-decode-title','live-mfu-title']){savedTitles[id]=elements[id];delete elements[id];} +render(data); +for(const id of ['live-prefill-tps','live-decode-tps','live-mfu'])assert.equal(elements[id].textContent,'—'); +Object.assign(elements,savedTitles); +const savedDetail=elements['live-decode-samples'];delete elements['live-decode-samples']; +render(data);assert.equal(elements['live-decode-tps'].textContent,'—');elements['live-decode-samples']=savedDetail; +a.prefill={tokens_per_second:0,mfu_estimate:0,peak_flops:2000,log_at:86399,log_skew_seconds:0,valid_until:86401}; +a.decode={tokens_per_second:3,log_at:86399,log_skew_seconds:0,valid_until:86401}; +render(data); +assert.equal(elements['live-prefill-tps'].textContent,'0 tok/s'); +assert.doesNotMatch(elements['live-prefill-title'].textContent,/历史/); +assert.doesNotMatch(elements['live-decode-title'].textContent,/历史/); +assert.doesNotMatch(elements['live-mfu-title'].textContent,/历史/); +a.prefill=null;a.decode=null;data.history_available=false;render(data); +for(const id of ['live-prefill-tps','live-decode-tps','live-mfu'])assert.equal(elements[id].textContent,'—'); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, encoding='utf-8', timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_cross_group_history_sums_available_groups_and_labels_coverage(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const start = source.indexOf('function combinedDeviceHistory('), end = source.indexOf('\n}\n', start); +assert(start >= 0 && end > start); +const combine = vm.runInNewContext('(' + source.slice(start, end + 2) + ')'); +function group(port, rate, samples) { + const points = Array.from({length: 144}, (_, i) => ({start: i*600, end: (i+1)*600, + prefill_tps: rate, prefill_samples: samples, decode_tps: null, decode_samples: 0})); + return {backend_port: port, source_ok: true, history: {method: 'deepinfer-log-step-approx-v1', bucket_seconds: 600, points}}; +} +const a = group(1, 100, 100), b = group(2, 300, 1), data = {available: true, history_available: true}; +let result = combine(data, [a, b]); +assert.equal(result.history.points[0].prefill_tps, 400); // Sum group means, never pool samples. +assert.equal(result.history.points[0].prefill_samples, 1); // Each complete bucket has equal weight. +assert.equal(result.history.points[0].decode_tps, null); +b.history.points[2].prefill_samples = 0; +result = combine(data, [a, b]); assert.equal(result.history.points[2].prefill_tps, 100); +assert.equal(result.history.points[2].prefill_groups, 1); assert.equal(result.history.group_count, 2); +b.history = null; result = combine(data, [a, b]); +assert.equal(result.history.points[0].prefill_tps, 100); assert.equal(result.history.points[0].prefill_groups, 1); +const shifted = group(2, 300, 1); shifted.history.points.forEach((point) => {point.start += 600; point.end += 600;}); +result = combine(data, [a, shifted]); assert.equal(result.history.points[0].prefill_tps, 100); +assert.equal(result.history.points[0].prefill_groups, 1); +assert.equal(result.history.points[1].prefill_tps, 400); assert.equal(result.history.points[1].prefill_groups, 2); +a.history.points[1].start = a.history.points[0].start; a.history.points[1].end = a.history.points[0].end; +assert.equal(combine(data, [a, b]), null); // Duplicate buckets cannot be trusted. +a.history.points[1].start = 600; a.history.points[1].end = 1200; +a.source_ok = false; assert.equal(combine(data, [a, b]), null); +assert.equal(combine({available: true, history_available: false}, [a]), null); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_history_fallback_uses_same_bucket_and_weights_only_verified_groups(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert=require('node:assert/strict'),fs=require('node:fs'),vm=require('node:vm'); +const src=fs.readFileSync(process.argv[1],'utf8'),ctx={}; +for(const name of ['freshTelemetry','combinedDeviceHistory','latestHistoricalSample','latestGroupHistory','historicalPrefillMfu']){ + const start=src.indexOf(`function ${name}(`),end=src.indexOf('\n}\n',start); + assert(start>=0&&end>start);ctx[name]=vm.runInNewContext('('+src.slice(start,end+2)+')',ctx); +} +function group(port,rate,count,peak,flops) { + const points=Array.from({length:144},(_,i)=>({start:i*600,end:(i+1)*600, + prefill_tps:null,prefill_samples:0,decode_tps:null,decode_samples:0})); + points[143].prefill_tps=rate; points[143].prefill_samples=3; + return {backend_port:port,source_ok:true,clock_ok:true,clock_uncertainty_seconds:.1, + hbm:{device_count:count,valid_until:90000},assumptions:{prefill_flops_per_token:flops,peak_flops_per_device:peak}, + history:{method:'deepinfer-log-step-approx-v1',bucket_seconds:600,points}}; +} +const a=group(1,100,2,1000,4),b=group(2,300,3,2000,10),nodes=[a,b]; +const data={available:true,history_available:true}, now=86400; +let combined=ctx.combinedDeviceHistory(data,nodes); +let sample=ctx.latestHistoricalSample(combined.history,'prefill',now); +assert.equal(sample.value,400);assert.equal(sample.groups,2); +let mfu=ctx.historicalPrefillMfu(nodes,'all',sample,now); +assert.equal(mfu.value,.425);assert.equal(mfu.groups,2);assert.equal(mfu.total,2); +b.hbm.valid_until=now; mfu=ctx.historicalPrefillMfu(nodes,'all',sample,now); +assert.equal(mfu.value,.2);assert.equal(mfu.groups,1);assert.equal(mfu.total,2); +b.history.points[143].prefill_samples=0;b.history.points[142].prefill_tps=300;b.history.points[142].prefill_samples=2; +combined=ctx.combinedDeviceHistory(data,nodes);sample=ctx.latestHistoricalSample(combined.history,'prefill',now); +assert.equal(sample.value,100);assert.equal(sample.groups,1); // Never add different time buckets. +a.history.points[143].prefill_tps=0;combined=ctx.combinedDeviceHistory(data,nodes); +sample=ctx.latestHistoricalSample(combined.history,'prefill',now);assert.equal(sample.value,0); +assert.equal(ctx.latestHistoricalSample(a.history,'prefill',86399),null); // Future end cannot be historical. +a.history.points[1].start=a.history.points[0].start; +assert.equal(ctx.latestHistoricalSample(a.history,'prefill',now),null); +assert.equal(ctx.latestHistoricalSample({method:'other',bucket_seconds:600,points:[]},'prefill',now),null); +// Misaligned grids: prefer the actual newest complete bucket across groups. +const c=group(3,100,2,1000,4),d=group(4,500,3,2000,10); +d.history.points.forEach((p)=>{p.start+=600;p.end+=600;}); +d.history.points[142].prefill_tps=300;d.history.points[142].prefill_samples=1; +let latest=ctx.latestGroupHistory(data,[c,d],'all','prefill',87000); +assert.equal(latest.start,86400);assert.equal(latest.value,500);assert.equal(latest.groups,1); +// Restricting the available timeline to the prior bucket must include both groups. +d.history.points[143].prefill_samples=0; +latest=ctx.latestGroupHistory(data,[c,d],'all','prefill',87000); +assert.equal(latest.start,85800);assert.equal(latest.value,400);assert.equal(latest.groups,2); +const weighted=ctx.historicalPrefillMfu([c,d],'all',latest,87000); +assert.equal(weighted.value,.425);assert.equal(weighted.groups,2); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_history_marks_partially_observed_buckets(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + script = r''' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const start = source.indexOf('function renderDeviceHistory('), end = source.indexOf('\n}\n', start); +assert(start >= 0 && end > start); +function element(name, attrs = {}) {return {name, attrs, textContent: '', children: [], + replaceChildren() {this.children=[];}, appendChild(value) {this.children.push(value);}};} +const elements = {}; +for (const phase of ['prefill', 'decode']) { + elements[`chart-device-${phase}`] = element('label'); + elements[`chart-device-${phase}-svg`] = element('svg'); +} +const context = {$: (id) => elements[id], svgElement: element, setTextIfPresent: () => {}, attachTrendTooltip: () => {}, + clock: {format: () => 'time'}, dateTime: {format: () => 'date'}, speed: (value) => `${value} tok/s`, count: String}; +const render = vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +const points = Array.from({length: 144}, (_, i) => ({start:i*600,end:(i+1)*600, + prefill_tps:null,prefill_samples:0,prefill_groups:0,decode_tps:null,decode_samples:0,decode_groups:0})); +points[1].prefill_tps = 100; points[1].prefill_samples = 1; points[1].prefill_groups = 1; +points[2].prefill_tps = 120; points[2].prefill_samples = 1; points[2].prefill_groups = 1; +points[3].prefill_tps = 200; points[3].prefill_samples = 1; points[3].prefill_groups = 2; +render({available:true,history_available:true},{history:{method:'deepinfer-log-step-approx-v1', + bucket_seconds:600,aggregate:true,group_count:2,points}}); +assert.match(elements['chart-device-prefill'].textContent, /3 个有采样区间 · 2 个仅部分组/); +const children = elements['chart-device-prefill-svg'].children; +assert(children.some((item) => item.name === 'polyline' && item.attrs.class === 'prefill partial')); +const partial = children.find((item) => item.name === 'circle' && item.attrs.class === 'prefill partial'); +assert.match(partial.children[0].textContent, /1\/2 组采样均值之和 · 部分数据/); +''' + source = Path(__file__).resolve().parents[1]/'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_gateway.py b/tests/test_gateway.py index 84ed705..bce64f2 100644 --- a/tests/test_gateway.py +++ b/tests/test_gateway.py @@ -5,6 +5,7 @@ from pathlib import Path import socket import socketserver +import sys import tempfile import threading import unittest @@ -12,7 +13,7 @@ from unittest.mock import patch from deploy_gateway.routing import SessionRouter, Unavailable -from deploy_gateway.server import Gateway +from deploy_gateway.server import Gateway, main class Model(BaseHTTPRequestHandler): @@ -56,6 +57,105 @@ def do_POST(self): self.wfile.write(value) +class Readiness(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + self.server.paths.append(self.path) + self.send_response(self.server.status) + body = self.server.instance_body if self.path == '/instances' and hasattr(self.server, 'instance_body') else self.server.body + self.send_header('Content-Length', str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass + + +class GatewayCliTests(unittest.TestCase): + def test_statistics_paths_cannot_overlap_gateway_state_or_sidecars(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + base = ['gateway', '--key-store', str(root/'keys'), '--bindings-file', str(root/'bindings'), + '--backend-state', str(root/'state.json'), '--control-file', str(root/'control')] + for database in (root/'state.json', root/'state.new', root/'bindings.new', root/'control.new', root/'keys'): + with self.subTest(database=database), patch.object(sys, 'argv', base + ['--statistics-database', str(database)]), \ + patch('deploy_stats.inprocess.install') as install, patch('sys.stderr'), self.assertRaises(SystemExit): + main() + install.assert_not_called() + self.assertFalse(database.exists()) + database = root/'metrics' + sidecar_state = root/'metrics-wal' + base = ['gateway', '--key-store', str(root/'keys'), '--bindings-file', str(root/'bindings'), + '--backend-state', str(sidecar_state), '--statistics-database', str(database)] + with patch.object(sys, 'argv', base), patch('deploy_stats.inprocess.install') as install, \ + patch('sys.stderr'), self.assertRaises(SystemExit): + main() + install.assert_not_called() + self.assertFalse(database.exists()) + + def test_statistics_database_cannot_be_hardlink_to_runtime_file(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + state, database = root/'state.json', root/'usage.sqlite3' + state.write_text('synthetic state', encoding='utf-8') + try: + database.hardlink_to(state) + except OSError: + self.skipTest('Hard links are not supported by the test filesystem') + argv = ['gateway', '--key-store', str(root/'keys'), '--bindings-file', str(root/'bindings'), + '--backend-state', str(state), '--statistics-database', str(database)] + with patch.object(sys, 'argv', argv), patch('deploy_stats.inprocess.install') as install, \ + patch('sys.stderr'), self.assertRaises(SystemExit): + main() + install.assert_not_called() + self.assertEqual(state.read_text(encoding='utf-8'), 'synthetic state') + + def test_statistics_recorder_is_tagged_and_closed(self): + argv = ['gateway', '--key-store', 'keys.json', '--bindings-file', 'bindings.json', + '--backend-state', 'state.json', '--statistics-database', 'usage.sqlite3'] + with patch.object(sys, 'argv', argv), patch('deploy_gateway.server.Gateway') as gateway, \ + patch('deploy_stats.inprocess.install') as install, patch('builtins.print'): + main() + self.assertEqual(install.call_args.kwargs['source'], 'gateway-inprocess-v1') + install.return_value.close.assert_called_once() + gateway.return_value.server_close.assert_called_once() + + def test_statistics_setup_failure_does_not_stop_gateway(self): + argv = ['gateway', '--key-store', 'keys.json', '--bindings-file', 'bindings.json', + '--backend-state', 'state.json', '--statistics-database', 'usage.sqlite3'] + with patch.object(sys, 'argv', argv), patch('deploy_gateway.server.Gateway') as gateway, \ + patch('deploy_stats.inprocess.install', side_effect=OSError), patch('builtins.print'): + main() + gateway.return_value.serve_forever.assert_called_once() + + def test_repeated_management_mappings_reach_gateway(self): + argv = ['gateway', '--key-store', 'keys.json', '--bindings-file', 'bindings.json', + '--backend-state', 'state.json', '--backend-port', '19001', '--backend-port', '19002', + '--backend-readiness-port', '19001=19101', '--backend-readiness-port', '19002=19102', + '--backend-pd-model', 'example', '--backend-pd-probe-endpoints', + '--statistics-database', 'usage.sqlite3'] + with patch.object(sys, 'argv', argv), patch('deploy_gateway.server.Gateway') as gateway, \ + patch('deploy_stats.inprocess.install') as install, patch('builtins.print'): + main() + self.assertEqual(gateway.call_args.kwargs['backend_readiness_ports'], {19001: 19101, 19002: 19102}) + self.assertEqual(gateway.call_args.kwargs['pd_model'], 'example') + self.assertTrue(gateway.call_args.kwargs['pd_probe_endpoints']) + self.assertEqual(install.call_args.kwargs['source'], 'gateway-inprocess-v1') + install.return_value.close.assert_called_once() + gateway.return_value.serve_forever.assert_called_once() + + def test_duplicate_or_malformed_management_mapping_is_rejected(self): + base = ['gateway', '--key-store', 'keys.json', '--bindings-file', 'bindings.json', + '--backend-state', 'state.json'] + for suffix in [['--backend-readiness-port', '19001'], + ['--backend-readiness-port', '19001=19101', '--backend-readiness-port', '19001=19102']]: + with self.subTest(suffix=suffix), patch.object(sys, 'argv', base + suffix), \ + patch('sys.stderr'), self.assertRaises(SystemExit): + main() + + class GatewayTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() @@ -132,6 +232,138 @@ def test_balance_sticky_history_auth_and_private_state(self): self.assertEqual(self.request('bad', token='incorrect-placeholder')[0], 401) self.assertEqual(sum(len(m.calls) for m in self.models), 3) + def test_management_readiness_controls_routing_and_recovery(self): + management = ThreadingHTTPServer(('127.0.0.1', 0), Readiness) + management.daemon_threads = True + management.status, management.body, management.paths = 200, b'{"ready":false}', [] + self.start(management) + gateway = Gateway(('127.0.0.1', 0), self.store, *self.ports, + self.root / 'motor-state.json', self.root / 'motor-bindings.json', 3, + backend_readiness_ports={self.ports[0]: management.server_port}) + gateway.backends.stop.set() + gateway.backends.thread.join(3) + self.start(gateway) + + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + self.assertTrue(gateway.backends.healthy[self.ports[1]]) + self.assertEqual(self.request('management-unready', gateway=gateway)[1], str(self.ports[1])) + self.assertEqual(management.paths, ['/readiness']) + + management.body = b'{"ready":true}' + gateway.backends.probe() + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + gateway.backends.probe() + self.assertTrue(gateway.backends.healthy[self.ports[0]]) + + for status, body in [(200, b'{"ready":false}'), (200, b'{"ready":1}'), + (200, b'{"ready":"true"}'), (200, b'[]'), + (200, b'invalid'), (200, b' ' * 65537), + (503, b'{"ready":true}')]: + with self.subTest(status=status, body=body[:30]): + management.status, management.body = status, body + gateway.backends.probe() + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + self.assertTrue(gateway.backends.healthy[self.ports[1]]) + management.status, management.body = 200, b'{"ready":true}' + gateway.backends.probe() + gateway.backends.probe() + self.assertTrue(gateway.backends.healthy[self.ports[0]]) + + def test_invalid_management_readiness_mapping_is_rejected(self): + bad_mappings = [{self.ports[0]: 0}, {self.ports[0]: 65536}, + {self.ports[0]: True}, {self.ports[0]: '1234'}, + {0: 1234}, {True: 1234}] + for mapping in bad_mappings: + with self.subTest(mapping=mapping), self.assertRaises(ValueError): + Gateway(('127.0.0.1', 0), self.store, *self.ports, + self.root / 'invalid-state.json', self.root / 'invalid-bindings.json', + backend_readiness_ports=mapping) + + def test_motor_pd_requires_healthy_prefill_and_decode_despite_ready(self): + management = ThreadingHTTPServer(('127.0.0.1', 0), Readiness) + management.daemon_threads = True + management.status, management.body, management.paths = 200, b'{"ready":true}', [] + prefill = {'role': 'prefill', 'pool': 'available', 'status': 'active', 'healthy': True, 'model_name': 'example'} + decode = {'role': 'decode', 'pool': 'available', 'status': 'active', 'healthy': True, 'model_name': 'example'} + management.instance_body = json.dumps({'instances': [prefill]}).encode() + self.start(management) + backup_management = ThreadingHTTPServer(('127.0.0.1', 0), Readiness) + backup_management.daemon_threads = True + backup_management.status, backup_management.body, backup_management.paths = 200, b'{"ready":true}', [] + backup_management.instance_body = json.dumps({'instances': [prefill, decode]}).encode() + self.start(backup_management) + gateway = Gateway(('127.0.0.1', 0), self.store, *self.ports, + self.root / 'pd-state.json', self.root / 'pd-bindings.json', 3, + backend_readiness_ports={self.ports[0]: management.server_port, + self.ports[1]: backup_management.server_port}, + pd_model='example') + gateway.backends.stop.set() + gateway.backends.thread.join(3) + self.start(gateway) + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + self.assertEqual(self.request('only-prefill', gateway=gateway)[1], str(self.ports[1])) + + for instances in ([decode], [prefill, {**decode, 'healthy': False}], + [prefill, {**decode, 'pool': 'unavailable'}], + [prefill, {**decode, 'healthy': 1}], + [prefill, {**decode, 'model_name': 'other'}], + [prefill, {**decode, 'role': []}], []): + with self.subTest(instances=instances): + management.instance_body = json.dumps({'instances': instances}).encode() + gateway.backends.probe() + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + for body in (b'invalid', b'[]', b'{"instances":{}}', b' ' * 1048577): + management.instance_body = body + gateway.backends.probe() + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + + management.instance_body = json.dumps({'instances': [prefill, decode]}).encode() + gateway.backends.probe() + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + gateway.backends.probe() + self.assertTrue(gateway.backends.healthy[self.ports[0]]) + + def test_pd_model_requires_all_management_mappings(self): + with self.assertRaises(ValueError): + Gateway(('127.0.0.1', 0), self.store, *self.ports, + self.root / 'pd-invalid-state.json', self.root / 'pd-invalid-bindings.json', + backend_readiness_ports={self.ports[0]: 12345}, pd_model='example') + with self.assertRaises(ValueError): + Gateway(('127.0.0.1', 0), self.store, *self.ports, + self.root / 'pd-invalid-state.json', self.root / 'pd-invalid-bindings.json', + pd_probe_endpoints=True) + + def test_pd_endpoint_probe_rejects_stale_motor_health_and_accepts_a_live_alternative(self): + management = ThreadingHTTPServer(('127.0.0.1', 0), Readiness) + management.daemon_threads = True + management.status, management.body, management.paths = 200, b'{"ready":true}', [] + self.start(management) + endpoint = lambda port: {'ip': '127.0.0.1', 'business_port': str(port), 'headless': False} + row = lambda role, port: {'role': role, 'pool': 'available', 'status': 'active', + 'healthy': True, 'model_name': 'example', 'endpoints': [endpoint(port)]} + prefill, decode = row('prefill', self.ports[0]), row('decode', self.ports[1]) + management.instance_body = json.dumps({'instances': [prefill, decode]}).encode() + self.models[1].up = False + gateway = Gateway(('127.0.0.1', 0), self.store, self.ports[0], self.ports[0] + 1, + self.root / 'endpoint-state.json', self.root / 'endpoint-bindings.json', 3, + backend_ports=[self.ports[0]], + backend_readiness_ports={self.ports[0]: management.server_port}, + pd_model='example', pd_probe_endpoints=True) + gateway.backends.stop.set() + gateway.backends.thread.join(3) + self.start(gateway) + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + + management.instance_body = json.dumps({'instances': [prefill, decode, row('decode', self.ports[0])]}).encode() + gateway.backends.probe() + gateway.backends.probe() + self.assertTrue(gateway.backends.healthy[self.ports[0]]) + for invalid in ([], [{'ip': [], 'business_port': '123'}], [endpoint(0)]): + management.instance_body = json.dumps({'instances': [prefill, {**decode, 'endpoints': invalid}]}).encode() + gateway.backends.probe() + gateway.backends.probe() + self.assertFalse(gateway.backends.healthy[self.ports[0]]) + def test_transport_failover_persists_after_recovery_and_restart(self): first = self.request('sticky') old = self.models[self.ports.index(int(first[1]))] diff --git a/tests/test_inprocess.py b/tests/test_inprocess.py new file mode 100644 index 0000000..fde20eb --- /dev/null +++ b/tests/test_inprocess.py @@ -0,0 +1,806 @@ +import hashlib +from http.client import HTTPConnection, HTTPResponse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import socket +import subprocess +import tempfile +import threading +import time +import unittest +from email.message import Message +from unittest.mock import patch + +import deploy_gateway.server as gateway_module +from deploy_common.reject_drain import MAX_REJECTIONS +from deploy_gateway.routing import affinity_key +from deploy_stats import inprocess +from deploy_stats.capture import MAX_FRAME, UsageCapture, extract_usage +from deploy_stats.inprocess import _Observer, install +from deploy_stats.report import report +from deploy_stats.session import BodyBudget +from deploy_stats.web import Dashboard +from tests.caddy_helpers import synthetic_caddy_owns_listener + + +STREAM = (b'data: {"choices":[{"delta":{"content":"synthetic"}}]}\n\n' + b'data: {"usage":{"prompt_tokens":100,"completion_tokens":7,"total_tokens":107,' + b'"prompt_tokens_details":{"cached_tokens":60}}}\n\n' b'data: [DONE]\n\n') + + +class Model(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + def log_message(self, *args): + pass + def do_GET(self): + self.send_response(200) + self.send_header('Content-Length', '0') + self.end_headers() + def do_POST(self): + body = self.rfile.read(int(self.headers.get('Content-Length', '0'))) + self.server.calls.append(body) + mode = self.server.mode + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Content-Length', str(len(STREAM) + (20 if mode == 'truncate' else 0))) + self.end_headers() + if mode == 'slow': + self.wfile.write(STREAM[:40]) + self.wfile.flush() + self.server.started.set() + self.server.release.wait(5) + try: + self.wfile.write(STREAM[40:]) + self.wfile.flush() + except OSError: + self.server.cancelled.set() + return + self.wfile.write(STREAM) + self.wfile.flush() + if mode == 'truncate': + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + + +class InProcessCaptureTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.key = 'synthetic-gateway-key' + self.keys = self.root/'keys.json' + self.keys.write_text(json.dumps({'version': 1, 'records': {'fixture': { + 'sha256': hashlib.sha256(self.key.encode()).hexdigest(), 'active': True}}})) + self.model = ThreadingHTTPServer(('127.0.0.1', 0), Model) + self.model.daemon_threads = True + self.model.calls = [] + self.model.mode = 'normal' + self.model.started, self.model.release, self.model.cancelled = (threading.Event() for _ in range(3)) + self.start(self.model) + self.original_handler = gateway_module.Handler + self.baseline = self.gateway('baseline') + self.recorder = install(gateway_module, self.root/'usage.sqlite3') + self.assertIsNotNone(self.recorder) + self.addCleanup(self.recorder.close) + self.addCleanup(setattr, gateway_module, 'Handler', self.original_handler) + self.gateway_under_test = self.gateway('observed') + gateway_module.Handler = self.original_handler + + def start(self, server): + thread = threading.Thread(target=server.serve_forever, kwargs={'poll_interval': .02}, daemon=True) + thread.start() + def close(): + server.shutdown() + server.server_close() + thread.join(3) + self.addCleanup(close) + + def gateway(self, name): + server = gateway_module.Gateway(('127.0.0.1', 0), self.keys, + state_path=self.root/(name+'-state.json'), bindings_file=self.root/(name+'-bindings.json'), + backend_ports=[self.model.server_port], backend_managed_affinity=True) + self.start(server) + return server + + def request(self, server, payload, headers=None): + connection = HTTPConnection('127.0.0.1', server.server_port, timeout=5) + try: + connection.request('POST', '/v1/chat/completions', payload, headers or {}) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + def wait_recorded(self): + deadline = time.monotonic() + 4 + while time.monotonic() < deadline: + with self.recorder.live_lock: + if not self.recorder.live: + self.assertTrue(self.recorder.flush()) + return + time.sleep(.01) + self.fail('observer did not finish') + + def test_response_and_session_match_existing_gateway_without_storing_content(self): + payload = b'{"messages":[{"role":"user","content":"synthetic-private-prompt"}]}' + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json', + 'X-Forwarded-For': '192.0.2.4'} + self.assertEqual(self.request(self.baseline, payload, headers), (200, STREAM)) + self.assertEqual(self.request(self.gateway_under_test, payload, headers), (200, STREAM)) + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual((result['totals']['known'], result['totals']['input'], + result['totals']['output'], result['totals']['cached']), (1, 100, 7, 60)) + self.assertEqual(result['requests'][0]['ip'], '192.0.2.4') + message = Message() + message['Authorization'] = 'Bearer ' + self.key + digest = hashlib.sha256(self.key.encode()).hexdigest() + self.assertEqual(result['requests'][0]['session'], affinity_key(message, payload, digest)) + self.assertIsNotNone(result['requests'][0]['first_visible']) + for file in self.root.glob('usage.sqlite3*'): + data = file.read_bytes() + for private in (self.key.encode(), b'synthetic-private-prompt'): + self.assertNotIn(private, data) + + def test_inprocess_source_uses_only_single_caddy_forwarded_address(self): + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json', + 'X-Forwarded-For': '192.0.2.4', 'X-LLMDeploy-Client-IP': '198.51.100.9'} + self.assertEqual(self.request(self.gateway_under_test, b'{}', headers), (200, STREAM)) + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual(result['requests'][0]['ip'], '192.0.2.4') + headers.pop('X-Forwarded-For') + self.assertEqual(self.request(self.gateway_under_test, b'{}', headers), (200, STREAM)) + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual(result['totals']['unknown_source'], 1) + for forwarded in ('192.0.2.4, 198.51.100.2', 'not-an-ip'): + headers['X-Forwarded-For'] = forwarded + self.assertEqual(self.request(self.gateway_under_test, b'{}', headers), (200, STREAM)) + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual(result['totals']['unknown_source'], 3) + message = Message() + message['X-Forwarded-For'] = '192.0.2.4' + message['X-Forwarded-For'] = '198.51.100.2' + observer = _Observer(self.recorder, BodyBudget(), message, 'chat') + self.assertIsNone(observer.source) + observer.finish() + + @unittest.skipUnless(os.environ.get('LLMDEPLOY_CADDY'), 'Set LLMDEPLOY_CADDY for Caddy integration') + def test_caddy_default_forwarding_overwrites_spoofed_client_addresses(self): + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + front = probe.getsockname()[1] + config = self.root/'Caddyfile-default-xff' + config.write_text('{\n admin off\n auto_https off\n}\n' + f'http://127.0.0.1:{front} {{\n bind 127.0.0.1\n' + f' reverse_proxy 127.0.0.1:{self.gateway_under_test.server_port}\n}}\n', encoding='utf-8') + env = dict(os.environ, XDG_DATA_HOME=str(self.root), XDG_CONFIG_HOME=str(self.root), + APPDATA=str(self.root), LOCALAPPDATA=str(self.root)) + flags = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0 + caddy = subprocess.Popen([os.environ['LLMDEPLOY_CADDY'], 'run', '--config', str(config), + '--adapter', 'caddyfile'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=env, creationflags=flags) + try: + deadline = time.monotonic() + 5 + while not synthetic_caddy_owns_listener(caddy, front): + if caddy.poll() is not None or time.monotonic() >= deadline: + self.fail('synthetic Caddy listener ownership could not be verified') + time.sleep(.02) + cases = ([], [('X-Forwarded-For', '192.0.2.55')], + [('X-Forwarded-For', '192.0.2.55, 198.51.100.8')], + [('X-Forwarded-For', '192.0.2.55'), ('x-forwarded-for', '198.51.100.8')], + [('X-LLMDeploy-Client-IP', '192.0.2.55')]) + for extra in cases: + self.assertTrue(synthetic_caddy_owns_listener(caddy, front)) + conn = HTTPConnection('127.0.0.1', front, timeout=5) + try: + conn.putrequest('POST', '/v1/chat/completions') + conn.putheader('Authorization', 'Bearer '+self.key) + conn.putheader('Content-Length', '2') + for name, value in extra: + conn.putheader(name, value) + conn.endheaders(b'{}') + response = conn.getresponse() + self.assertEqual((response.status, response.read()), (200, STREAM)) + finally: + conn.close() + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual(result['totals']['known'], len(cases)) + self.assertEqual(result['totals']['unknown_source'], 0) + self.assertEqual({item['ip'] for item in result['clients']}, {'127.0.0.1'}) + finally: + if caddy.poll() is None: + caddy.terminate() + caddy.wait(timeout=5) + + def test_dashboard_api_updates_after_successive_gateway_requests(self): + dashboard = Dashboard(0, self.recorder.path) + self.start(dashboard) + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json', + 'X-Forwarded-For': '192.0.2.4'} + def stats(): + conn = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=5) + try: + conn.request('GET', '/api/stats?window=total') + response = conn.getresponse() + self.assertEqual(response.status, 200) + return json.loads(response.read()) + finally: + conn.close() + self.assertEqual(stats()['totals']['known'], 0) + for expected in (1, 2): + self.assertEqual(self.request(self.gateway_under_test, b'{}', headers), (200, STREAM)) + self.wait_recorded() + deadline = time.monotonic() + 4 + value = stats() + while value['totals']['known'] < expected and time.monotonic() < deadline: + time.sleep(.02) + value = stats() + self.assertEqual((value['totals']['known'], value['totals']['input'], value['totals']['output'], + value['totals']['cached']), (expected, 100*expected, 7*expected, 60*expected)) + + def test_motor_readiness_tagged_capture_and_dashboard_end_to_end(self): + class Readiness(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + def do_GET(self): + body = self.server.instance_body if self.path == '/instances' else b'{"ready":true}' + self.send_response(200) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + readiness = ThreadingHTTPServer(('127.0.0.1', 0), Readiness) + endpoint = {'ip': '127.0.0.1', 'business_port': str(self.model.server_port), 'headless': False} + readiness.instance_body = json.dumps({'instances': [ + {'role': role, 'pool': 'available', 'status': 'active', 'healthy': True, + 'model_name': 'example', 'endpoints': [endpoint]} + for role in ('prefill', 'decode')]}).encode() + self.start(readiness) + database = self.root/'motor-usage.sqlite3' + old_handler = gateway_module.Handler + recorder = install(gateway_module, database, source='gateway-inprocess-v1') + self.assertIsNotNone(recorder) + self.addCleanup(recorder.close) + try: + gateway = gateway_module.Gateway(('127.0.0.1', 0), self.keys, + state_path=self.root/'motor-state.json', bindings_file=self.root/'motor-bindings.json', + backend_ports=[self.model.server_port], backend_managed_affinity=True, + backend_readiness_ports={self.model.server_port: readiness.server_port}, + pd_model='example', pd_probe_endpoints=True) + finally: + gateway_module.Handler = old_handler + self.start(gateway) + self.assertTrue(gateway.backends.healthy[self.model.server_port]) + with patch('deploy_stats.web.MotorCollector') as collector: + collector.return_value.snapshot.return_value = {'sampled_at': time.time(), 'nodes': []} + dashboard = Dashboard(0, database, motor_management_ports=[readiness.server_port]) + self.start(dashboard) + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json', + 'X-Forwarded-For': '192.0.2.4'} + self.assertEqual(self.request(gateway, b'{}', headers), (200, STREAM)) + self.assertTrue(recorder.flush()) + deadline = time.monotonic() + 4 + value = report(database, 'total', source='gateway-inprocess-v1') + while value['totals']['known'] < 1 and time.monotonic() < deadline: + time.sleep(.02) + value = report(database, 'total', source='gateway-inprocess-v1') + self.assertTrue(value['collector_live']) + self.assertEqual((value['totals']['input'], value['totals']['output'], value['totals']['cached']), (100, 7, 60)) + conn = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=5) + try: + conn.request('GET', '/api/v5/stats?window=total') + response = conn.getresponse() + payload = json.loads(response.read()) + self.assertEqual(response.status, 200) + self.assertTrue(payload['collector_live']) + self.assertEqual(payload['requests'][0]['ip'], '192.0.2.4') + self.assertEqual(payload['totals']['known'], 1) + finally: + conn.close() + + def test_vllm_stream_without_cache_details_records_partial_usage(self): + old_handler = gateway_module.Handler + recorder = install(gateway_module, self.root/'partial-usage.sqlite3', source='gateway-inprocess-v1') + self.assertIsNotNone(recorder) + self.addCleanup(recorder.close) + try: + gateway = gateway_module.Gateway(('127.0.0.1', 0), self.keys, + state_path=self.root/'partial-state.json', bindings_file=self.root/'partial-bindings.json', + backend_ports=[self.model.server_port], backend_managed_affinity=True) + finally: + gateway_module.Handler = old_handler + self.start(gateway) + partial_stream = (b'data: {"choices":[{"delta":{"content":"synthetic"}}]}\n\n' + b'data: {"usage":{"prompt_tokens":120,"completion_tokens":20,"total_tokens":140}}\n\n' + b'data: [DONE]\n\n') + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json', + 'X-Forwarded-For': '192.0.2.4'} + with patch(f'{__name__}.STREAM', partial_stream): + self.assertEqual(self.request(gateway, b'{}', headers), (200, partial_stream)) + deadline = time.monotonic() + 4 + value = report(recorder.path, 'total', source='gateway-inprocess-v1') + while value['totals']['partial'] < 1 and time.monotonic() < deadline: + time.sleep(.02) + value = report(recorder.path, 'total', source='gateway-inprocess-v1') + self.assertEqual((value['totals']['partial'], value['totals']['input'], value['totals']['output']), (1, 120, 20)) + self.assertIsNone(value['totals']['hit_rate']) + self.assertIsNone(value['requests'][0]['cached']) + + @unittest.skipUnless(os.environ.get('LLMDEPLOY_CADDY'), 'Set LLMDEPLOY_CADDY for Caddy integration') + def test_caddy_reload_keeps_one_active_stream(self): + # This checks an active stream, not idle keep-alive connections. + def port(): + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + return probe.getsockname()[1] + front, admin = port(), port() + before, maintenance, after = (self.root/'before.Caddyfile', self.root/'maintenance.Caddyfile', + self.root/'after.Caddyfile') + template = ('{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\n' + 'http://127.0.0.1:{front} {{\n bind 127.0.0.1\n reverse_proxy 127.0.0.1:{gateway} {{\n' + '{header} }}\n}}\n') + values = {'admin': admin, 'front': front, 'gateway': self.gateway_under_test.server_port} + before.write_text(template.format(**values, header=''), encoding='utf-8') + maintenance.write_text(('{{\n admin 127.0.0.1:{admin}\n auto_https off\n}}\n' + 'http://127.0.0.1:{front} {{\n bind 127.0.0.1\n respond 503\n}}\n').format(**values), + encoding='utf-8') + after.write_text(template.format(**values, + header=' header_up X-LLMDeploy-Client-IP {remote_host}\n'), encoding='utf-8') + env = dict(os.environ, XDG_DATA_HOME=str(self.root), XDG_CONFIG_HOME=str(self.root), + APPDATA=str(self.root), LOCALAPPDATA=str(self.root)) + flags = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0 + caddy = subprocess.Popen([os.environ['LLMDEPLOY_CADDY'], 'run', '--config', str(before), + '--adapter', 'caddyfile'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=env, creationflags=flags) + client = None + try: + deadline = time.monotonic() + 5 + while not (synthetic_caddy_owns_listener(caddy, front) and + synthetic_caddy_owns_listener(caddy, admin)): + if caddy.poll() is not None or time.monotonic() >= deadline: + self.fail('synthetic Caddy listener ownership could not be verified') + time.sleep(.02) + self.model.mode = 'slow' + client = socket.create_connection(('127.0.0.1', front), timeout=5) + client.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: 127.0.0.1:{front}\r\n' + f'Authorization: Bearer {self.key}\r\nContent-Length: 2\r\n\r\n{{}}').encode()) + self.assertTrue(self.model.started.wait(3)) + received = bytearray(client.recv(4096)) + self.assertIn(b'200', received) + self.model.mode = 'normal' + def reload(config): + self.assertTrue(synthetic_caddy_owns_listener(caddy, admin)) + self.assertTrue(synthetic_caddy_owns_listener(caddy, front)) + result = subprocess.run([os.environ['LLMDEPLOY_CADDY'], 'reload', '--address', + f'127.0.0.1:{admin}', '--config', str(config), '--adapter', 'caddyfile'], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env=env, creationflags=flags, timeout=5) + self.assertEqual(result.returncode, 0) + reload(maintenance) + self.assertTrue(synthetic_caddy_owns_listener(caddy, front)) + conn = HTTPConnection('127.0.0.1', front, timeout=5) + conn.request('POST', '/v1/chat/completions', b'{}') + maintenance_response = conn.getresponse() + self.assertEqual(maintenance_response.status, 503) + maintenance_response.read() + conn.close() + reload(after) + self.assertTrue(synthetic_caddy_owns_listener(caddy, front)) + conn = HTTPConnection('127.0.0.1', front, timeout=5) + conn.request('POST', '/v1/chat/completions', b'{}', + {'Authorization': 'Bearer '+self.key, 'X-LLMDeploy-Client-IP': '192.0.2.55'}) + response = conn.getresponse() + self.assertEqual((response.status, response.read()), (200, STREAM)) + conn.close() + self.model.release.set() + while b'data: [DONE]\n\n' not in received: + chunk = client.recv(4096) + self.assertTrue(chunk) + received.extend(chunk) + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + sources = {item['ip'] for item in result['clients']} + self.assertIn('127.0.0.1', sources) + self.assertNotIn('192.0.2.55', sources) + finally: + self.model.release.set() + if client: + client.close() + if caddy.poll() is None: + caddy.terminate() + caddy.wait(timeout=5) + + def test_terminal_marker_does_not_make_a_truncated_response_complete(self): + self.model.mode = 'truncate' + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json'} + self.assertEqual(self.request(self.gateway_under_test, b'{}', headers), (200, STREAM)) + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual((result['totals']['known'], result['totals']['interrupted']), (0, 1)) + + def test_recording_and_parser_faults_never_change_response(self): + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json'} + for target in ('deploy_stats.inprocess.SafeCapture.feed', 'deploy_stats.inprocess._Observer.session'): + with self.subTest(target=target), patch(target, side_effect=RuntimeError('synthetic failure')): + self.assertEqual(self.request(self.gateway_under_test, b'{}', headers), (200, STREAM)) + self.wait_recorded() + self.assertGreaterEqual(self.recorder.dropped, 2) + + def test_unavailable_recorder_does_not_change_handler(self): + module = type('SyntheticModule', (), {'Handler': self.original_handler}) + with patch('deploy_stats.inprocess.Recorder', side_effect=OSError('synthetic failure')): + self.assertIsNone(install(module, self.root/'unavailable.sqlite3')) + self.assertIs(module.Handler, self.original_handler) + + def test_bridge_start_failure_closes_recorder_and_restores_handler(self): + module = type('SyntheticModule', (), {'Handler': self.original_handler}) + with patch('deploy_stats.inprocess.Recorder') as recorder: + def unavailable(): + raise OSError('synthetic bind failure') + with self.assertRaises(OSError): + inprocess.serve(module, self.root/'failed-start.sqlite3', unavailable) + recorder.return_value.close.assert_called_once() + self.assertIs(module.Handler, self.original_handler) + + def test_recorder_failure_does_not_prevent_bridge_start(self): + module = type('SyntheticModule', (), {'Handler': self.original_handler}) + called = [] + def unavailable_server(): + called.append(True) + raise OSError('synthetic bind failure') + with patch('deploy_stats.inprocess.Recorder', side_effect=OSError('synthetic storage failure')): + with self.assertRaises(OSError): + inprocess.serve(module, self.root/'failed-recorder.sqlite3', unavailable_server) + self.assertEqual(called, [True]) + self.assertIs(module.Handler, self.original_handler) + + @unittest.skipUnless(hasattr(socket, 'SO_EXCLUSIVEADDRUSE'), 'Windows exclusive socket option') + def test_gateway_cannot_bind_over_same_address_listener(self): + duplicate = None + try: + with self.assertRaises(OSError): + duplicate = gateway_module.Gateway(('127.0.0.1', self.gateway_under_test.server_port), self.keys, + state_path=self.root/'duplicate-state.json', bindings_file=self.root/'duplicate-bindings.json', + backend_ports=[self.model.server_port], backend_managed_affinity=True) + finally: + if duplicate is not None: + duplicate.server_close() + + def test_early_auth_rejection_keeps_the_response_complete_during_upload(self): + connection = socket.create_connection(('127.0.0.1', self.gateway_under_test.server_port), timeout=5) + try: + connection.sendall(b'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n' + b'Authorization: Bearer synthetic-invalid\r\nContent-Length: 1048576\r\n\r\n' + + b'x' * 1024) + response = HTTPResponse(connection) + response.begin() + self.assertEqual(response.status, 401) + self.assertEqual(json.loads(response.read())['error']['message'], 'Invalid bearer API key') + for _ in range(15): + connection.sendall(b'x' * 65536) + finally: + connection.close() + deadline = time.monotonic() + 3 + while self.gateway_under_test.rejection_drainer._slots._value != MAX_REJECTIONS and time.monotonic() < deadline: + time.sleep(.01) + self.assertEqual(self.gateway_under_test.rejection_drainer._slots._value, MAX_REJECTIONS) + + def test_install_is_idempotent_and_does_not_start_second_recorder(self): + module = type('SyntheticModule', (), {'Handler': self.original_handler}) + first = install(module, self.root/'idempotent.sqlite3') + self.assertIsNotNone(first) + self.addCleanup(first.close) + installed_handler = module.Handler + with patch('deploy_stats.inprocess.Recorder') as second_recorder: + self.assertIsNone(install(module, self.root/'idempotent.sqlite3')) + second_recorder.assert_not_called() + self.assertIs(module.Handler, installed_handler) + + def test_discarded_oversize_sse_releases_budget_for_other_requests(self): + for long_line in (False, True): + with self.subTest(long_line=long_line): + budget = BodyBudget(MAX_FRAME + 256 * 1024) + observer = _Observer(self.recorder, budget, Message(), 'chat') + observer.response(200, 'text/event-stream', '') + if long_line: + observer.chunk(b'data: prior\n') + while not observer.capture.reader.discard_line: + observer.chunk(b'x' * 65536) + else: + while not observer.capture.reader.discard_event: + observer.chunk(b'data: ' + b'x' * 65000 + b'\n') + self.assertEqual(budget.used, 0) + other = _Observer(self.recorder, budget, Message(), 'chat') + other.response(200, 'text/event-stream', '') + other.chunk(STREAM) + other.complete(True) + other.finish() + observer.finish() + self.assertEqual(budget.used, 0) + self.assertTrue(self.recorder.flush()) + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual(result['totals']['known'], 1 if not long_line else 2) + + def test_discarded_sse_frame_never_reports_stale_usage_or_late_first_output(self): + budget = BodyBudget(MAX_FRAME + 256 * 1024) + observer = _Observer(self.recorder, budget, Message(), 'chat') + observer.response(200, 'text/event-stream', '') + observer.chunk(b'data: {"usage":{"prompt_tokens":2,"completion_tokens":1,' + b'"prompt_tokens_details":{"cached_tokens":1}}}\n\n') + while not observer.capture.reader.discard_event: + observer.chunk(b'data: ' + b'x' * 65000 + b'\n') + observer.chunk(b'\n' + STREAM) + observer.complete(True) + observer.finish() + self.assertTrue(self.recorder.flush()) + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual((result['totals']['known'], result['totals']['missing']), (0, 1)) + self.assertIsNone(result['requests'][0]['first_visible']) + + def test_large_inprocess_frames_are_skipped_without_stopping_other_requests(self): + budget = BodyBudget(MAX_FRAME + 256 * 1024) + for content_type in ('text/event-stream', 'application/json'): + with self.subTest(content_type=content_type): + observer = _Observer(self.recorder, budget, Message(), 'chat') + observer.response(200, content_type, '') + if content_type == 'text/event-stream': + observer.chunk(b'data: ') + for _ in range(65): + observer.chunk(b'x' * 65536) + reader = observer.capture.reader + self.assertTrue(reader.discard_event if reader.kind == 'sse' else reader.too_large) + self.assertEqual(budget.used, 0) + observer.complete(True) + observer.finish() + + def test_invalid_later_sse_never_reuses_previous_usage(self): + bad_events = ( + b'{"usage":{"prompt_tokens":1,"completion_tokens":1,' + b'"prompt_tokens_details":{"cached_tokens":2}}}', + b'{"usage": BROKEN}', + b'{"unclosed":', + ) + for bad in bad_events: + for first_before in (False, True): + with self.subTest(bad=bad, first_before=first_before): + first = [] + parser = UsageCapture('chat', 'text/event-stream', on_first_output=lambda: first.append(True)) + if first_before: + parser.feed(b'data: {"choices":[{"delta":{"content":"visible"}}]}\n\n') + parser.feed(b'data: {"usage":{"prompt_tokens":2,"completion_tokens":1,' + b'"prompt_tokens_details":{"cached_tokens":1}}}\n\n') + parser.feed(b'data: ' + bad + b'\n\n') + parser.feed(STREAM) + self.assertIsNone(parser.finish()) + self.assertEqual(first, [True] if first_before else []) + + def test_many_empty_or_mixed_data_lines_are_bounded_and_do_not_starve_other_requests(self): + for line in (b'data:\n', b'data: x\ndata:\n'): + with self.subTest(line=line): + budget = BodyBudget(MAX_FRAME) + observer = _Observer(self.recorder, budget, Message(), 'chat') + observer.response(200, 'text/event-stream', '') + for _ in range(12): + observer.chunk(line * 500) + self.assertTrue(observer.capture.reader.discard_event) + self.assertEqual(budget.used, 0) + other = _Observer(self.recorder, budget, Message(), 'chat') + other.response(200, 'text/event-stream', '') + other.chunk(STREAM) + other.complete(True) + other.finish() + observer.finish() + self.assertEqual(budget.used, 0) + self.assertTrue(self.recorder.flush()) + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual(result['totals']['known'], 1 if line == b'data:\n' else 2) + + def test_large_simple_responses_completion_retains_usage(self): + budget = BodyBudget(32 * 1024 * 1024) + observer = _Observer(self.recorder, budget, Message(), 'responses') + observer.response(200, 'text/event-stream', '') + observer.chunk(b'data: {"type":"response.output_text.delta","delta":"x"}\n\n') + text = 'x' * (1024 * 1024) + ('[{"值":"中文🙂","s":"\\\\","q":"\\\""}]' * 1000) + completed = {'type': 'response.completed', 'response': {'output': text, + 'usage': {'input_tokens': 100, 'output_tokens': 7, 'input_tokens_details': {'cached_tokens': 60}}}} + frame = b'data: ' + json.dumps(completed, ensure_ascii=False).encode() + b'\n\n' + for start in range(0, len(frame), 65536): + observer.chunk(frame[start:start + 65536]) + observer.complete(True) + observer.finish() + self.assertTrue(self.recorder.flush()) + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual((result['totals']['known'], result['totals']['input'], result['totals']['output'], + result['totals']['cached']), (1, 100, 7, 60)) + + def test_structurally_dense_json_is_missing_but_other_requests_still_record(self): + budget = BodyBudget(MAX_FRAME) + for content_type in ('text/event-stream', 'application/json'): + with self.subTest(content_type=content_type): + observer = _Observer(self.recorder, budget, Message(), 'chat') + observer.response(200, content_type, '') + body = json.dumps({'choices': [], 'extra': [0] * 20000, + 'usage': {'prompt_tokens': 100, 'completion_tokens': 7, + 'prompt_tokens_details': {'cached_tokens': 60}}}).encode() + frame = b'data: ' + body + b'\n\ndata: [DONE]\n\n' if content_type.startswith('text/') else body + for start in range(0, len(frame), 8192): + observer.chunk(frame[start:start + 8192]) + observer.complete(True) + observer.finish() + self.assertEqual(budget.used, 0) + self.assertTrue(self.recorder.flush()) + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual((result['totals']['known'], result['totals']['missing']), (0, 2)) + + def test_large_sse_delta_requires_later_usage_and_preserves_first_output_certainty(self): + large = b'data: ' + json.dumps({'choices': [{'delta': {'content': 'x' * 70000}}], + 'usage': None}).encode() + b'\n\n' + usage = (b'data: {"choices":[{"finish_reason":"stop"}],"usage":{"prompt_tokens":100,"completion_tokens":7,' + b'"prompt_tokens_details":{"cached_tokens":60}}}\n\n') + first_delta = b'data: {"choices":[{"delta":{"content":"x"}}]}\n\n' + for first_before, final_usage in ((False, True), (True, True), (True, False)): + with self.subTest(first_before=first_before, final_usage=final_usage): + first = [] + parser = UsageCapture('chat', 'text/event-stream', on_first_output=lambda: first.append(True), + max_structure=16384, max_delta=65536) + if first_before: + parser.feed(first_delta) + parser.feed(usage) + parser.feed(large) + if final_usage: + parser.feed(usage) + parser.feed(b'data: [DONE]\n\n') + self.assertEqual(parser.finish(), (100, 7, 60) if final_usage else None) + self.assertEqual(first, [True] if first_before else []) + + def test_chat_usage_after_skipped_event_requires_verified_finish(self): + large = b'data: ' + json.dumps({'choices': [{'delta': {'content': 'x' * 70000}}]}).encode() + b'\n\n' + usage = (b'data: {"usage":{"prompt_tokens":100,"completion_tokens":7,' + b'"prompt_tokens_details":{"cached_tokens":60}}}\n\n') + finish = b'data: {"choices":[{"finish_reason":"stop"}]}\n\n' + for finish_first in (False, True): + with self.subTest(finish_first=finish_first): + parser = UsageCapture('chat', 'text/event-stream', max_delta=65536) + parser.feed(large) + if finish_first: + parser.feed(finish) + parser.feed(usage + b'data: [DONE]\n\n') + self.assertEqual(parser.finish(), (100, 7, 60) if finish_first else None) + + def test_data_after_done_never_restores_usage(self): + large = b'data: ' + json.dumps({'choices': [{'delta': {'content': 'x' * 70000}}]}).encode() + b'\n\n' + final = (b'data: {"choices":[{"finish_reason":"stop"}],"usage":{"prompt_tokens":100,' + b'"completion_tokens":7,"prompt_tokens_details":{"cached_tokens":60}}}\n\n') + parser = UsageCapture('chat', 'text/event-stream', max_delta=65536) + parser.feed(large + b'data: [DONE]\n\n' + final) + self.assertIsNone(parser.finish()) + + def test_fragmented_long_sse_line_does_not_rescan_previous_bytes(self): + class CountedBuffer(bytearray): + scanned = 0 + def find(self, value, start=0, end=None): + self.scanned += (len(self) if end is None else min(end, len(self))) - start + return super().find(value, start) if end is None else super().find(value, start, end) + parser = UsageCapture('chat', 'text/event-stream') + parser.buffer = CountedBuffer() + for _ in range(32): + parser.feed(b'x' * 65536) + self.assertLessEqual(parser.buffer.scanned, 2 * 32 * 65536) + + def test_duplicate_usage_or_count_keys_do_not_produce_known_usage(self): + for body in ( + b'{"usage":{"prompt_tokens":2,"completion_tokens":1,"prompt_tokens_details":{"cached_tokens":1}},' + b'"usage":{"prompt_tokens":20,"completion_tokens":1,"prompt_tokens_details":{"cached_tokens":1}}}', + b'{"usage":{"prompt_tokens":2,"prompt_tokens":20,"completion_tokens":1,' + b'"prompt_tokens_details":{"cached_tokens":1}}}', + ): + with self.subTest(body=body): + parser = UsageCapture('chat', 'text/event-stream') + parser.feed(b'data: ' + body + b'\n\ndata: [DONE]\n\n') + self.assertIsNone(parser.finish()) + + def test_present_cache_hit_and_miss_counts_must_be_valid_and_consistent(self): + valid = {'prompt_tokens': 100, 'completion_tokens': 7, 'prompt_cache_hit_tokens': 60, + 'prompt_cache_miss_tokens': 40, 'prompt_tokens_details': {'cached_tokens': 60}} + self.assertEqual(extract_usage('chat', {'usage': valid}), (100, 7, 60)) + invalid = [dict(valid, prompt_cache_miss_tokens=99), dict(valid, prompt_cache_hit_tokens='bad'), + dict(valid, prompt_cache_miss_tokens='bad'), + dict(valid, prompt_tokens_details={'cached_tokens': 'bad'}), + dict(valid, prompt_tokens_details='bad')] + for usage in invalid: + with self.subTest(usage=usage): + self.assertIsNone(extract_usage('chat', {'usage': usage})) + + def test_large_event_with_nested_null_and_final_usage_is_not_skipped(self): + payload = {'extra': {'usage': None, 'text': 'x' * 70000}, 'choices': [{'finish_reason': 'stop'}], + 'usage': {'prompt_tokens': 100, 'completion_tokens': 7, + 'prompt_tokens_details': {'cached_tokens': 60}}} + parser = UsageCapture('chat', 'text/event-stream', max_delta=65536, max_structure=16384) + parser.feed(b'data: ' + json.dumps(payload).encode() + b'\n\ndata: [DONE]\n\n') + self.assertEqual(parser.finish(), (100, 7, 60)) + + def test_large_nonterminal_event_with_usage_waits_for_verified_terminal(self): + first = [] + parser = UsageCapture('chat', 'text/event-stream', on_first_output=lambda: first.append(True), + max_delta=65536, max_structure=16384) + frame = {'choices': [{'delta': {'content': 'x' * 70000}}], + 'usage': {'prompt_tokens': 100, 'completion_tokens': 7, + 'prompt_tokens_details': {'cached_tokens': 60}}} + parser.feed(b'data: ' + json.dumps(frame).encode() + b'\n\n') + self.assertTrue(parser.skipped) + self.assertEqual(first, []) + parser.feed(b'data: {"choices":[{"finish_reason":"stop"}],"usage":{"prompt_tokens":100,' + b'"completion_tokens":7,"prompt_tokens_details":{"cached_tokens":60}}}\n\n' + b'data: [DONE]\n\n') + self.assertEqual(parser.finish(), (100, 7, 60)) + + def test_nested_finish_marker_cannot_validate_large_chat_usage(self): + payload = {'meta': {'finish_reason': 'stop'}, 'content': 'x' * 70000, + 'usage': {'prompt_tokens': 100, 'completion_tokens': 7, + 'prompt_tokens_details': {'cached_tokens': 60}}} + parser = UsageCapture('chat', 'text/event-stream', max_delta=65536) + parser.feed(b'data: ' + json.dumps(payload).encode() + b'\n\ndata: [DONE]\n\n') + self.assertIsNone(parser.finish()) + + def test_capture_budget_exhaustion_does_not_interrupt_response(self): + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json'} + with patch('deploy_stats.inprocess.CAPTURE_BUDGET', 10): + recorder = install(gateway_module, self.root/'limited-usage.sqlite3') + self.assertIsNotNone(recorder) + self.addCleanup(recorder.close) + gateway = self.gateway('limited') + gateway_module.Handler = self.original_handler + self.assertEqual(self.request(gateway, b'{}', headers), (200, STREAM)) + deadline = time.monotonic() + 4 + while recorder.live and time.monotonic() < deadline: + time.sleep(.01) + self.assertTrue(recorder.flush()) + result = report(recorder.path, 'total', now=time.time() + .01) + self.assertEqual((result['totals']['known'], result['totals']['missing']), (0, 1)) + self.assertGreaterEqual(recorder.dropped, 1) + + def test_large_request_session_uses_gateway_fingerprint_without_storing_body(self): + payload = json.dumps({'messages': [{'role': 'user', 'content': 'x' * (1024 * 1024 + 123)}]}).encode() + headers = {'Authorization': 'Bearer '+self.key, 'Content-Type': 'application/json'} + self.assertEqual(self.request(self.gateway_under_test, payload, headers), (200, STREAM)) + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + message = Message() + message['Authorization'] = 'Bearer ' + self.key + digest = hashlib.sha256(self.key.encode()).hexdigest() + self.assertEqual(result['requests'][0]['session'], affinity_key(message, payload, digest)) + self.assertEqual(result['totals']['unknown_source'], 1) + for file in self.root.glob('usage.sqlite3*'): + self.assertNotIn(b'x' * 100, file.read_bytes()) + + def test_client_cancellation_is_not_counted_as_complete(self): + self.model.mode = 'slow' + conn = socket.create_connection(('127.0.0.1', self.gateway_under_test.server_port), timeout=5) + try: + conn.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n' + f'Authorization: Bearer {self.key}\r\nContent-Length: 2\r\n\r\n{{}}').encode()) + self.assertTrue(self.model.started.wait(3)) + self.assertTrue(conn.recv(4096)) + conn.shutdown(socket.SHUT_RDWR) + conn.close() + # The existing gateway may not unblock a silent upstream read on + # Windows until the upstream sends again; keep that baseline. + time.sleep(.3) + self.model.release.set() + self.wait_recorded() + result = report(self.recorder.path, 'total', now=time.time() + .01) + self.assertEqual((result['totals']['known'], result['totals']['interrupted']), (0, 1)) + finally: + self.model.release.set() + conn.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_motor_metrics.py b/tests/test_motor_metrics.py new file mode 100644 index 0000000..c6f3384 --- /dev/null +++ b/tests/test_motor_metrics.py @@ -0,0 +1,386 @@ +import json +from contextlib import closing +from http.client import HTTPConnection +import sqlite3 +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from deploy_stats.motor_metrics import MotorCollector, MotorHistory, parse_metrics +from deploy_stats.capture import SafeCapture, extract_usage +from deploy_stats.report import active_report, report +from deploy_stats.store import Recorder +from deploy_stats.web import Dashboard + + +def instance(name, role, ports): + return {'id': name, 'role': role, 'status': 'active', 'healthy': True, + 'model_name': 'synthetic', + 'endpoints': [{'id': str(port), 'ip': '127.0.0.1', 'business_port': str(port)} + for port in ports]} + + +def metrics(start, prefill, decode, count=0, seconds=0): + labels = 'engine="0",model_name="synthetic"' + return '\n'.join([ + f'process_start_time_seconds {start}', + f'vllm:prompt_tokens_by_source_total{{{labels},source="local_compute"}} {prefill}', + f'vllm:generation_tokens_total{{{labels}}} {decode}', + f'vllm:request_prefill_time_seconds_sum{{{labels}}} {seconds}', + f'vllm:request_prefill_time_seconds_count{{{labels}}} {count}', + f'vllm:request_decode_time_seconds_sum{{{labels}}} {seconds}', + f'vllm:request_decode_time_seconds_count{{{labels}}} {count}', + ]) + + +class SequentialExecutor: + def __init__(self, **_kwargs): + pass + def __enter__(self): + return self + def __exit__(self, *_args): + return False + def map(self, function, values): + return map(function, values) + + +class MotorMetricsTests(unittest.TestCase): + def setUp(self): + self.now = 1000.0 + self.topology = [instance('p', 'prefill', [12001, 12002]), + instance('d', 'decode', [12003])] + self.data = {port: metrics(50, 100, 20, 2, 4) for port in (12001, 12002, 12003)} + self.failed = set() + + def topology(port): + return json.dumps({'instances': self.topology}) + + def endpoint(ip, port): + if port in self.failed: + raise OSError('fixture failure') + return self.data[port] + + self.collector = MotorCollector([13001, 13002], fetch_topology=topology, + fetch_endpoint=endpoint, wall_time=lambda: self.now, + monotonic=lambda: self.now, interval=30) + + def sample(self, role): + value = self.collector.collect() + return next(node for node in value['nodes'] if node['role'] == role), value + + def test_duplicate_coordinators_deduplicated_and_concurrent_rate_uses_counter_delta(self): + first, result = self.sample('prefill') + self.assertIsNone(first['prefill']) + self.assertEqual(len(result['nodes']), 2) + self.now += 30 + self.data[12001] = metrics(50, 400, 999, 5, 10) + self.data[12002] = metrics(50, 700, 999, 5, 13) + self.data[12003] = metrics(50, 9000, 170, 5, 10) + result = self.collector.collect() + prefill = next(node for node in result['nodes'] if node['role'] == 'prefill') + decode = next(node for node in result['nodes'] if node['role'] == 'decode') + self.assertEqual(prefill['prefill']['tokens_per_second'], 30) + self.assertEqual(decode['decode']['tokens_per_second'], 5) + self.assertEqual(prefill['prefill']['valid_endpoints'], 2) + self.assertEqual(prefill['prefill']['expected_endpoints'], 2) + self.assertEqual(prefill['phase_timers']['prefill']['mean_ms'], 2500) + self.assertIsNone(prefill['decode']) + self.assertIsNone(decode['prefill']) + self.assertIsNone(result['prefill_mfu']) + self.assertIsNone(result['hbm_percent']) + + def test_missing_worker_is_partial_and_a_gap_does_not_become_zero(self): + self.collector.collect() + self.now += 30 + self.failed.add(12002) + self.data[12001] = metrics(50, 400, 20) + node, _ = self.sample('prefill') + self.assertEqual(node['prefill']['tokens_per_second'], 10) + self.assertEqual((node['prefill']['valid_endpoints'], node['prefill']['expected_endpoints']), (1, 2)) + self.assertTrue(node['prefill']['partial']) + self.now += 30 + self.failed.remove(12002) + self.data[12001] = metrics(50, 700, 20) + self.data[12002] = metrics(50, 900, 20) + node, _ = self.sample('prefill') + self.assertEqual(node['prefill']['tokens_per_second'], 10) + self.assertEqual(node['prefill']['valid_endpoints'], 1) + + def test_restart_and_counter_reset_do_not_produce_rates(self): + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(51, 110, 25) + self.data[12002] = metrics(50, 2, 1) + node, _ = self.sample('prefill') + self.assertIsNone(node['prefill']) + + def test_motor_numeric_instance_ids_are_accepted(self): + self.topology[0]['id'] = 123 + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(50, 400, 20) + self.data[12002] = metrics(50, 700, 20) + node, _ = self.sample('prefill') + self.assertEqual(node['prefill']['tokens_per_second'], 30) + + def test_disagreeing_coordinators_do_not_admit_unconfirmed_groups(self): + self.collector.fetch_topology = lambda port: json.dumps({ + 'instances': self.topology if port == 13001 else self.topology[1:]}) + result = self.collector.collect() + self.assertTrue(result['topology_ambiguous']) + self.assertEqual([node['role'] for node in result['nodes']], ['decode']) + + def test_role_change_during_worker_scrape_does_not_create_prefill_rate(self): + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(50, 400, 20) + self.data[12002] = metrics(50, 700, 20) + calls = {13001: 0, 13002: 0} + changed = [instance('p', 'decode', [12001, 12002]), self.topology[1]] + def topology(port): + calls[port] += 1 + return json.dumps({'instances': self.topology if calls[port] == 1 else changed}) + self.collector.fetch_topology = topology + result = self.collector.collect() + self.assertTrue(result['topology_ambiguous']) + self.assertFalse(any(node['prefill'] is not None for node in result['nodes'])) + + def test_oldest_worker_sample_controls_group_expiry(self): + with patch('deploy_stats.motor_metrics.ThreadPoolExecutor', SequentialExecutor): + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(50, 400, 20) + self.data[12002] = metrics(50, 700, 20) + def slow_second(_ip, port): + if port == 12002: + self.now += 20 + return self.data[port] + self.collector.fetch_endpoint = slow_second + node, _ = self.sample('prefill') + self.assertIsNotNone(node['prefill']) + self.assertLessEqual(node['prefill']['valid_until'], 1095) + + def test_very_old_worker_sample_cannot_be_relabelled_as_fresh(self): + with patch('deploy_stats.motor_metrics.ThreadPoolExecutor', SequentialExecutor): + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(50, 400, 20) + self.data[12002] = metrics(50, 700, 20) + def stalled_second(_ip, port): + if port == 12002: + self.now += 70 + return self.data[port] + self.collector.fetch_endpoint = stalled_second + node, _ = self.sample('prefill') + self.assertIsNone(node['prefill']) + + def test_backend_gauges_expire_at_the_oldest_worker_deadline(self): + self.collector.collect() + initial = self.collector.snapshot() + self.assertTrue(all(node['engine_metrics'] is not None for node in initial['nodes'])) + self.now += 66 + expired = self.collector.snapshot() + self.assertTrue(expired['nodes']) # The overall collector snapshot has not expired yet. + self.assertTrue(all(node['engine_metrics'] is None for node in expired['nodes'])) + + def test_slow_worker_response_uses_earliest_possible_sample_time(self): + self.topology = [instance('p', 'prefill', [12001])] + with patch('deploy_stats.motor_metrics.ThreadPoolExecutor', SequentialExecutor): + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(50, 400, 20) + def slow_worker(_ip, port): + self.now += 20 + return self.data[port] + self.collector.fetch_endpoint = slow_worker + node, _ = self.sample('prefill') + self.assertEqual(node['prefill']['valid_until'], 1095) + self.assertEqual(node['engine_metrics']['sampled_at'], 1030) + self.assertEqual(node['engine_metrics']['valid_until'], 1095) + + def test_duplicate_worker_address_is_not_counted_twice(self): + self.topology.append(instance('duplicate', 'prefill', [12001])) + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(50, 400, 20) + self.data[12002] = metrics(50, 700, 20) + node, result = self.sample('prefill') + usable = [n for n in result['nodes'] if n['role'] == 'prefill' and n['prefill']] + self.assertEqual(sum(n['prefill']['tokens_per_second'] for n in usable), 20) + self.assertTrue(result['topology_ambiguous']) + + def test_counter_series_must_remain_identical(self): + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(50, 400, 20).replace('engine="0"', 'engine="1"') + self.data[12002] = metrics(50, 700, 20) + node, _ = self.sample('prefill') + self.assertEqual(node['prefill']['tokens_per_second'], 20) + self.assertEqual(node['prefill']['valid_endpoints'], 1) + + def test_reassigned_instance_cannot_reuse_previous_topology_history(self): + first, _ = self.sample('prefill') + self.now += 30 + self.topology[0] = instance('p', 'prefill', [12001, 12003]) + self.topology[1] = instance('d', 'decode', [12002]) + second, _ = self.sample('prefill') + self.assertNotEqual(first['backend_port'], second['backend_port']) + self.assertIsNone(second['prefill']) + + def test_parser_rejects_duplicate_or_nonfinite_counter(self): + raw = metrics(50, 100, 20) + with self.assertRaises(ValueError): + parse_metrics(raw + '\n' + raw.splitlines()[1]) + with self.assertRaises(ValueError): + parse_metrics(raw.replace(' 100', ' NaN')) + + def test_whitespace_and_one_endpoint_failure_do_not_break_other_workers(self): + self.assertIn('process_start_time_seconds', parse_metrics('process_start_time_seconds 1\n \n')) + self.collector.collect() + self.now += 30 + self.data[12001] = metrics(50, 400, 20) + def broken_endpoint(_ip, port): + if port == 12002: + raise RuntimeError('fixture failure') + return self.data[port] + self.collector.fetch_endpoint = broken_endpoint + node, _ = self.sample('prefill') + self.assertEqual(node['prefill']['tokens_per_second'], 10) + self.assertTrue(node['prefill']['partial']) + + def test_history_persists_samples_without_filling_unknown_buckets(self): + with tempfile.TemporaryDirectory() as tmp: + history = MotorHistory(Path(tmp) / 'history.sqlite3') + history.record('group', 'prefill', 100801, 10) + history.record('group', 'prefill', 100820, 20, partial=True) + points = history.series('group', 102000)['points'] + point = next(p for p in points if p['start'] == 100800) + self.assertEqual(point['prefill_tps'], 15) + self.assertEqual(point['prefill_samples'], 2) + self.assertTrue(point['prefill_partial']) + self.assertIsNone(point['decode_tps']) + self.assertIsNone(next(p for p in points if p['start'] == 101400)['prefill_tps']) + + def test_legacy_heartbeat_does_not_claim_motor_capture_is_live(self): + with tempfile.TemporaryDirectory() as tmp: + database = Path(tmp) / 'usage.sqlite3' + legacy = Recorder(database) + try: + self.assertTrue(active_report(database)['collector_live']) + self.assertFalse(active_report(database, source='gateway-inprocess-v1')['collector_live']) + current = Recorder(database, source='gateway-inprocess-v1') + try: + self.assertTrue(active_report(database, source='gateway-inprocess-v1')['collector_live']) + with closing(sqlite3.connect(database)) as connection, connection: + connection.execute('UPDATE runs SET started=? WHERE id=?', (time.time() - 172800, current.run)) + # Even a full-window heartbeat cannot prove Caddy actually routed traffic through this run. + self.assertFalse(report(database, 'today', source='gateway-inprocess-v1')['totals']['complete']) + finally: + current.close() + self.assertFalse(active_report(database, source='gateway-inprocess-v1')['collector_live']) + finally: + legacy.close() + + def test_dashboard_serves_motor_snapshot_without_trusting_legacy_heartbeat(self): + with tempfile.TemporaryDirectory() as tmp: + database = Path(tmp) / 'usage.sqlite3' + legacy = Recorder(database) + try: + node, snapshot = self.sample('prefill') + with patch('deploy_stats.web.MotorCollector') as collector: + collector.return_value.snapshot.return_value = snapshot + dashboard = Dashboard(0, database, motor_management_ports=[13001, 13002]) + worker = threading.Thread(target=dashboard.serve_forever, kwargs={'poll_interval': .02}, daemon=True) + worker.start() + try: + conn = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=3) + conn.request('GET', '/api/v5/telemetry') + response = conn.getresponse() + value = json.loads(response.read()) + conn.close() + self.assertEqual(response.status, 200) + self.assertTrue(value['available']) + self.assertEqual(value['kind'], 'vllm-motor-telemetry') + conn = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=3) + conn.request('GET', '/api/v5/stats') + response = conn.getresponse() + value = json.loads(response.read()) + conn.close() + self.assertEqual(response.status, 200) + self.assertFalse(value['collector_live']) + self.assertEqual(len(value['backends']), 2) + for path in ('/api/v5/active', '/app-telemetry-v7.js'): + conn = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=3) + conn.request('GET', path) + response = conn.getresponse() + response.read() + conn.close() + self.assertEqual(response.status, 200, path) + for path in ('/api/stats', '/api/active', '/api/telemetry'): + conn = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=3) + conn.request('GET', path) + response = conn.getresponse() + response.read() + conn.close() + self.assertEqual(response.status, 409, path) + conn = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=3) + conn.request('GET', '/app-telemetry-v6.js') + response = conn.getresponse() + response.read() + conn.close() + self.assertEqual(response.status, 404) + finally: + dashboard.shutdown() + dashboard.server_close() + worker.join(3) + collector.return_value.close.assert_called_once() + finally: + legacy.close() + + def test_partial_vllm_usage_preserves_input_and_output_without_inventing_cache(self): + usage = {'usage': {'prompt_tokens': 120, 'completion_tokens': 20, 'total_tokens': 140}} + self.assertIsNone(extract_usage('chat', usage)) + self.assertEqual(extract_usage('chat', usage, allow_partial=True), (120, 20, None)) + reader = SafeCapture('chat', 'text/event-stream', allow_partial=True) + reader.feed(b'data: {"choices":[{"delta":{"content":"fixture"}}]}\n\n') + reader.feed(b'data: {"usage":{"prompt_tokens":120,"completion_tokens":20,"total_tokens":140}}\n\n') + reader.feed(b'data: [DONE]\n\n') + self.assertEqual(reader.finish(), (120, 20, None)) + with tempfile.TemporaryDirectory() as tmp: + database = Path(tmp) / 'usage.sqlite3' + recorder = Recorder(database, source='gateway-inprocess-v1') + try: + now = time.time() + recorder.submit((now, '127.0.0.1', 2, 200, 'partial', 120, 20, None, 'a'*64, .5)) + recorder.submit((now, '127.0.0.1', 2, 200, 'known', 100, 7, 60, 'a'*64, .5)) + self.assertTrue(recorder.flush()) + result = report(database, 'total', now=now + 1, source='gateway-inprocess-v1') + self.assertEqual((result['totals']['known'], result['totals']['partial']), (1, 1)) + self.assertEqual((result['totals']['input'], result['totals']['output'], result['totals']['cached']), (220, 27, 60)) + self.assertEqual(result['totals']['hit_rate'], .6) + partial = next(row for row in result['requests'] if row['state'] == 'partial') + self.assertEqual((partial['input'], partial['output'], partial['cached'], partial['hit_rate']), (120, 20, None, None)) + self.assertIsNone(partial['input_per_wait']) + self.assertIsNotNone(partial['output_per_after_first']) + self.assertEqual(result['sessions'][0]['input'], 220) + self.assertEqual(result['sessions'][0]['hit_rate'], .6) + with closing(sqlite3.connect(database)) as connection: + legacy = connection.execute('SELECT state,input_tokens,output_tokens,cached_tokens FROM requests ORDER BY id LIMIT 1').fetchone() + self.assertEqual(legacy, ('missing', None, None, None)) + finally: + recorder.close() + + def test_partial_does_not_accept_invalid_or_ambiguous_usage(self): + base = {'prompt_tokens': 120, 'completion_tokens': 20, 'total_tokens': 140} + for invalid in ({**base, 'total_tokens': 999}, {**base, 'prompt_tokens_details': []}, + {**base, 'prompt_tokens_details': {'cached_tokens': -1}}, + {**base, 'prompt_cache_miss_tokens': 80}): + self.assertIsNone(extract_usage('chat', {'usage': invalid}, allow_partial=True)) + self.assertIsNone(extract_usage('messages', {'usage': {'input_tokens': 120, 'output_tokens': 20}}, allow_partial=True)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_mtp_usage.py b/tests/test_mtp_usage.py new file mode 100644 index 0000000..aabcc20 --- /dev/null +++ b/tests/test_mtp_usage.py @@ -0,0 +1,308 @@ +"""Contract tests for request token accounting and speculative metrics.""" +from contextlib import closing +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import tempfile +import time +import unittest + +from deploy_stats.report import report +from deploy_stats.store import _enable_partial_schema, _open +from deploy_stats.motor_metrics import MotorCollector, MotorHistory, _mtp_change, _mtp_counts, parse_metrics + + +class RequestAccountingTests(unittest.TestCase): + def test_session_prefill_rate_excludes_zero_wait_from_numerator(self): + from deploy_stats.report import _request_view, _session_views + rows = [(100, None, 'known', 100, 2, 0, 1, 's', 0, 200, 1), + (101, None, 'known', 10, 2, 0, 2, 's', 1, 200, 2)] + views = [_request_view(row) for row in rows] + self.assertIsNone(views[0]['input_per_wait']) + self.assertEqual(_session_views(views)[0]['input_per_wait'], 10) + + def test_cached_and_uncached_requests_use_only_known_usage(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + with closing(_open(path)) as db: + _enable_partial_schema(db) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('run', now - 100, now, now, 0)) + statement = '''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) + VALUES(?,?,?,?,?,?,?,?,?,?,?)''' + db.execute(statement, ('run', now - 3, '192.0.2.1', 1, 200, 'known', 100, 8, 60, 's', .5)) + db.execute(statement, ('run', now - 2, '192.0.2.1', 1, 200, 'known', 20, 2, 0, 's', .5)) + partial = db.execute(statement, ('run', now - 1, '192.0.2.1', 1, 200, 'missing', + None, None, None, 's', .5)).lastrowid + db.execute('INSERT INTO stats_partial_usage VALUES (?,?,?)', (partial, 30, 5)) + db.commit() + result = report(path, 'total', now=now + 1, request_sort='uncached_input', request_dir='desc') + self.assertEqual([item['uncached_input'] for item in result['requests']], [40, 20, None]) + self.assertEqual([item['cached'] for item in result['requests']], [60, 0, None]) + self.assertEqual(result['totals']['uncached_input'], 60) + session = result['sessions'][0] + self.assertEqual((session['input'], session['output'], session['cached'], session['uncached_input']), + (150, 15, 60, 60)) + self.assertEqual((session['known'], session['partial']), (2, 1)) + + def test_no_cache_data_does_not_become_zero_uncached(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + with closing(_open(path)) as db: + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('run', now - 10, now, now, 0)) + db.execute('''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) + VALUES(?,?,?,?,?,?,?,?,?,?,?)''', ('run', now - 1, None, 1, 200, 'missing', + None, None, None, 's', None)) + db.commit() + result = report(path, 'total', now=now + 1) + self.assertIsNone(result['requests'][0]['uncached_input']) + self.assertIsNone(result['totals']['uncached_input']) + self.assertIsNone(result['sessions'][0]['uncached_input']) + + +def mtp_metrics(start, steps, drafted, accepted, *, model='synthetic', engine='0'): + labels = f'engine="{engine}",model_name="{model}"' + return '\n'.join((f'process_start_time_seconds {start}', + f'vllm:spec_decode_num_drafts_total{{{labels}}} {steps}', + f'vllm:spec_decode_num_draft_tokens_total{{{labels}}} {drafted}', + f'vllm:spec_decode_num_accepted_tokens_total{{{labels}}} {accepted}')) + + +class MtpTests(unittest.TestCase): + def test_matching_labels_and_counter_reset_are_required(self): + before = parse_metrics(mtp_metrics(10, 10, 50, 20)) + after = parse_metrics(mtp_metrics(10, 15, 75, 35)) + self.assertEqual(_mtp_counts(after, 'synthetic'), (15, 75, 35)) + self.assertEqual(_mtp_change(after, before, 'synthetic'), (5, 25, 15)) + self.assertIsNone(_mtp_counts(after, 'different-model')) + self.assertIsNone(_mtp_change(before, after, 'synthetic')) + mismatched = mtp_metrics(10, 15, 75, 35).replace( + 'vllm:spec_decode_num_accepted_tokens_total{engine="0"', + 'vllm:spec_decode_num_accepted_tokens_total{engine="1"') + self.assertIsNone(_mtp_counts(parse_metrics(mismatched), 'synthetic')) + self.assertIsNone(_mtp_counts(parse_metrics(mtp_metrics(10, 10, 20, 21)), 'synthetic')) + self.assertIsNone(_mtp_counts(parse_metrics(mtp_metrics(10, 10, '10.5', 2)), 'synthetic')) + + def test_counts_are_weighted_and_partial_not_fabricated_as_zero(self): + now = [102000.0] + topology = [{'id': 'd', 'role': 'decode', 'status': 'active', 'healthy': True, + 'model_name': 'synthetic', + 'endpoints': [{'ip': '127.0.0.1', 'business_port': 12001}, + {'ip': '127.0.0.1', 'business_port': 12002}]}] + data = {12001: mtp_metrics(10, 10, 100, 90), 12002: mtp_metrics(10, 10, 900, 90)} + with tempfile.TemporaryDirectory() as tmp: + history = MotorHistory(Path(tmp) / 'history.sqlite3') + collector = MotorCollector([13001], fetch_topology=lambda _: json.dumps({'instances': topology}), + fetch_endpoint=lambda _ip, port: data[port], history=history, + wall_time=lambda: now[0], monotonic=lambda: now[0]) + first = collector.collect()['nodes'][0] + self.assertAlmostEqual(first['mtp']['lifetime']['acceptance_rate'], .18) + self.assertIsNone(first['mtp']['interval']) + now[0] += 30 + data[12001] = mtp_metrics(10, 11, 110, 99) + data[12002] = mtp_metrics(10, 11, 990, 99) + second = collector.collect()['nodes'][0] + interval = second['mtp']['interval'] + self.assertEqual((interval['drafted'], interval['accepted']), (100, 18)) + self.assertAlmostEqual(interval['acceptance_rate'], .18) + self.assertFalse(interval['partial']) + now[0] += 30 + data[12001] = mtp_metrics(11, 1, 5, 1) # restarted; no delta for this worker + data[12002] = mtp_metrics(10, 12, 1080, 108) + third = collector.collect()['nodes'][0] + self.assertTrue(third['mtp']['interval']['partial']) + self.assertEqual(third['mtp']['interval']['valid_endpoints'], 1) + self.assertEqual(third['mtp']['interval']['drafted'], 90) + now[0] += 600 + point = next(p for p in history.mtp_series( + hashlib.sha256(json.dumps( + ['mtp-v1', 'd', 'decode', (('127.0.0.1', 12001), ('127.0.0.1', 12002)), 'synthetic'], + separators=(',', ':')).encode()).hexdigest(), now[0])['points'] + if p['drafted'] is not None) + self.assertEqual((point['drafted'], point['accepted']), (190, 27)) + self.assertTrue(point['partial']) + + def test_zero_drafted_is_unknown_rate(self): + from deploy_stats.motor_metrics import _mtp_summary + data = _mtp_summary([(0, 0, 0)], 1) + self.assertIsNone(data['acceptance_rate']) + self.assertIsNone(data['mean_acceptance_length']) + + def test_invalid_optional_mtp_does_not_discard_other_metrics(self): + baseline = 'process_start_time_seconds 10\nvllm:generation_tokens_total{model_name="synthetic"} 123\n' + for bad in ('NaN', '1.5'): + with self.subTest(bad=bad): + values = parse_metrics(baseline + mtp_metrics(10, 3, bad, 1).split('\n', 1)[1]) + self.assertEqual(next(iter(values['vllm:generation_tokens_total'].values())), 123) + self.assertIsNone(_mtp_counts(values, 'synthetic')) + duplicate = mtp_metrics(10, 3, 10, 1).split('\n', 1)[1] + values = parse_metrics(baseline + duplicate + '\n' + duplicate.splitlines()[0]) + self.assertIn('vllm:generation_tokens_total', values) + self.assertIsNone(_mtp_counts(values, 'synthetic')) + + def test_sparse_history_does_not_claim_full_time_coverage(self): + with tempfile.TemporaryDirectory() as tmp: + history = MotorHistory(Path(tmp) / 'history.sqlite3') + history.record_mtp('g', 90030, 1, 10, 5) + point = next(p for p in history.mtp_series('g', 91200)['points'] if p['start'] == 90000) + self.assertTrue(point['partial']) + self.assertEqual(point['time_coverage'], 'unverified') + + def test_model_change_cannot_reuse_rates_or_record_mismatched_metrics(self): + now = [102000.0] + topology = [{'id': 'd', 'role': 'decode', 'status': 'active', 'healthy': True, + 'model_name': 'a', 'endpoints': [{'ip': '127.0.0.1', 'business_port': 12001}]}] + data = [None] + def metrics_for(model, amount): + return (f'process_start_time_seconds 10\n' + f'vllm:generation_tokens_total{{model_name="{model}"}} {amount}\n') + with tempfile.TemporaryDirectory() as tmp: + collector = MotorCollector([13001], fetch_topology=lambda _: json.dumps({'instances': topology}), + fetch_endpoint=lambda *_: data[0], history=MotorHistory(Path(tmp) / 'history.sqlite3'), + wall_time=lambda: now[0], monotonic=lambda: now[0]) + data[0] = metrics_for('a', 100) + first = collector.collect()['nodes'][0] + now[0] += 30 + data[0] = metrics_for('a', 130) + self.assertEqual(collector.collect()['nodes'][0]['decode']['tokens_per_second'], 1) + now[0] += 600 + topology[0]['model_name'] = 'b' + data[0] = metrics_for('b', 200) + second = collector.collect()['nodes'][0] + self.assertNotEqual(first['backend_port'], second['backend_port']) + self.assertIsNone(second['decode']) + self.assertFalse(any(point['decode_samples'] for point in second['history']['points'])) + now[0] += 30 + data[0] = metrics_for('c', 300) + collector.collect() + now[0] += 30 + data[0] = metrics_for('c', 330) + self.assertIsNone(collector.collect()['nodes'][0]['decode']) + + def test_older_history_file_is_upgraded_without_losing_rate_buckets(self): + import sqlite3 + from deploy_stats.motor_metrics import APPLICATION_ID, SCHEMA_VERSION + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'history.sqlite3' + with closing(sqlite3.connect(path)) as db, db: + db.execute(f'PRAGMA application_id = {APPLICATION_ID}') + db.execute(f'PRAGMA user_version = {SCHEMA_VERSION}') + db.execute('''CREATE TABLE rate_buckets (group_key TEXT NOT NULL, phase TEXT NOT NULL, + bucket INTEGER NOT NULL, rate_sum REAL NOT NULL, samples INTEGER NOT NULL, + partial_samples INTEGER NOT NULL, PRIMARY KEY (group_key, phase, bucket)) WITHOUT ROWID''') + db.execute('INSERT INTO rate_buckets VALUES (?,?,?,?,?,?)', ('g', 'decode', 90000, 60, 2, 0)) + history = MotorHistory(path) + self.assertEqual(next(p for p in history.series('g', 91200)['points'] + if p['start'] == 90000)['decode_tps'], 30) + history.record_mtp('g', 90001, 10, 100, 25) + with closing(sqlite3.connect(path)) as db: + self.assertEqual(db.execute('PRAGMA user_version').fetchone()[0], SCHEMA_VERSION) + self.assertEqual(db.execute('SELECT rate_sum, samples FROM rate_buckets').fetchone(), (60, 2)) + self.assertAlmostEqual(next(p for p in history.mtp_series('g', 91200)['points'] + if p['start'] == 90000)['acceptance_rate'], .25) + + +class MtpDashboardTests(unittest.TestCase): + def run_javascript(self, script): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard calculations') + source = Path(__file__).resolve().parents[1] / 'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_summary_weights_drafts_and_preserves_missing_and_stale_data(self): + self.run_javascript(r''' +const assert = require('node:assert/strict'), fs = require('node:fs'), vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'), context = {}; +function extract(name) { + const start = source.indexOf(`function ${name}(`), end = source.indexOf('\n}\n', start); + assert(start >= 0 && end > start); return vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +} +context.freshTelemetry = extract('freshTelemetry'); context.validMtp = extract('validMtp'); +const summary = extract('mtpSummary'); +const data = {available: true, kind: 'vllm-motor-telemetry'}; +const part = (drafted, accepted) => ({sampled_at: 90, valid_until: 110, steps: 10, drafted, accepted, + valid_endpoints: 1, expected_endpoints: 1, partial: false}); +const group = (backend_port, sample) => ({backend_port, role: 'decode', source_ok: true, backend_ready: true, + expected_endpoints: 1, mtp: {lifetime: sample}}); +const a = group(1, part(100, 90)), b = group(2, part(900, 90)); +let value = summary(data, [a, b], 'all', 100, 'lifetime'); +assert.equal(value.rate, .18); assert.equal(value.partial, false); +value = summary(data, [a, b], 1, 100, 'lifetime'); assert.equal(value.rate, .9); +b.source_ok = false; value = summary(data, [a, b], 'all', 100, 'lifetime'); +assert.equal(value.rate, .9); assert.equal(value.partial, true); assert.equal(value.groups, 1); +b.source_ok = true; b.mtp.lifetime.valid_until = 99; +assert.equal(summary(data, [b], 'all', 100, 'lifetime'), null); +assert.equal(summary(data, [group(3, part(0, 0))], 'all', 100, 'lifetime').rate, null); +assert.equal(summary(data, [group(3, part(1, 2))], 'all', 100, 'lifetime'), null); +assert.equal(summary(data, [a, {...a}], 'all', 100, 'lifetime'), null); +''') + + def test_history_weights_counts_marks_partial_and_keeps_gaps(self): + self.run_javascript(r''' +const assert = require('node:assert/strict'), fs = require('node:fs'), vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'); +const start = source.indexOf('function renderMtpHistory('), end = source.indexOf('\n}\n', start); +assert(start >= 0 && end > start); +const element = () => ({children: [], textContent: '', replaceChildren() {this.children = [];}, + appendChild(value) {this.children.push(value);}}); +const svg = element(), label = element(); let descriptions; +const context = {$: (id) => id === 'chart-mtp-svg' ? svg : label, + svgElement: (name, attrs) => Object.assign(element(), {name, attrs}), + attachTrendTooltip: (_svg, values) => {descriptions = values;}, + dateTime: {format: () => 'time'}, percent: (v) => `${v * 100}%`, count: String}; +const render = vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +const points = () => Array.from({length: 144}, (_, i) => ({start: i * 600, end: (i + 1) * 600, + drafted: null, accepted: null, samples: 0, partial: false})); +const first = points(), second = points(); +Object.assign(first[1], {drafted: 100, accepted: 90, samples: 1}); +Object.assign(second[1], {drafted: 900, accepted: 90, samples: 1}); +Object.assign(first[3], {drafted: 10, accepted: 5, samples: 1}); +const group = (backend_port, values) => ({backend_port, role: 'decode', mtp_history: + {method: 'vllm-mtp-counter-delta-v1', bucket_seconds: 600, points: values}}); +const a = group(1, first), b = group(2, second), data = {available: true, history_available: true, + kind: 'vllm-motor-telemetry'}; +render(data, [a, b], 'all'); +assert.match(descriptions[1], /18%/); assert.match(descriptions[1], /2\/2/); +assert.equal(descriptions[2], null); assert.match(descriptions[3], /50%/); +assert.match(descriptions[3], /1\/2.*部分数据/); +assert.match(descriptions[1], /时间覆盖未核验/); +assert.match(label.textContent, /2 个部分区间/); +assert.equal(svg.children.filter((v) => v.name === 'circle').length, 2); +render(data, [a, {...a}], 'all'); assert.equal(label.textContent, '等待 MTP 历史采样'); +''') + + def test_recent_coverage_is_not_borrowed_from_lifetime(self): + self.run_javascript(r''' +const assert = require('node:assert/strict'), fs = require('node:fs'), vm = require('node:vm'); +const source = fs.readFileSync(process.argv[1], 'utf8'), texts = {}; +const context = {setTextIfPresent: (id, value) => {texts[id] = value;}, percent: (v) => `${v * 100}%`, + renderMtpHistory: () => {}}; +function extract(name) { + const start = source.indexOf(`function ${name}(`), end = source.indexOf('\n}\n', start); + assert(start >= 0 && end > start); return vm.runInNewContext('(' + source.slice(start, end + 2) + ')', context); +} +for (const name of ['freshTelemetry', 'validMtp', 'mtpSummary']) context[name] = extract(name); +const render = extract('renderMtp'); +const part = (valid, drafted, accepted) => ({sampled_at: 90, valid_until: 110, steps: 10, drafted, accepted, + valid_endpoints: valid, expected_endpoints: 8, partial: valid < 8}); +const node = {backend_port: 1, role: 'decode', source_ok: true, backend_ready: true, expected_endpoints: 8, + mtp: {lifetime: part(8, 100, 90), interval: part(1, 100, 20)}}; +render({available: true, kind: 'vllm-motor-telemetry'}, [node], 'all', 100); +assert.match(texts['mtp-detail'], /8\/8 端点/); +assert.match(texts['mtp-detail'], /最近采样 20%（1\/1 组，1\/8 端点，部分数据）/); +''') + + def test_partial_mtp_markers_remain_hollow(self): + css = (Path(__file__).resolve().parents[1] / 'deploy_stats/static/app.css').read_text(encoding='utf-8') + self.assertIn('.lane svg circle.mtp.partial{fill:#fff}', css) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_stats.py b/tests/test_stats.py new file mode 100644 index 0000000..81bc113 --- /dev/null +++ b/tests/test_stats.py @@ -0,0 +1,1652 @@ +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +from datetime import datetime, timezone, timedelta +from email.message import Message +import hashlib +from http.client import HTTPConnection, HTTPResponse, IncompleteRead, parse_headers +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from io import BytesIO +import json +import os +from pathlib import Path +import queue +import sqlite3 +import socket +import subprocess +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +from deploy_stats.capture import UsageCapture, extract_usage +from deploy_stats.proxy import Proxy, _headers, client_ip, session_fingerprint +from deploy_stats.backends import read_backend +from deploy_stats.report import active_report, report +from deploy_stats.session import BodyBudget, SessionBodyCapture +from deploy_stats.store import _open, Recorder +from deploy_stats.web import Dashboard +from deploy_gateway.server import Gateway +from deploy_gateway.routing import affinity_key +from tests.caddy_helpers import synthetic_caddy_owns_listener + + +def usage(input_tokens=100, output_tokens=7, cached=60): + return {'prompt_tokens': input_tokens, 'completion_tokens': output_tokens, + 'total_tokens': input_tokens + output_tokens, + 'prompt_tokens_details': {'cached_tokens': cached}, 'prompt_cache_hit_tokens': cached} + + +def sse(value): + return b'data: ' + json.dumps(value, ensure_ascii=False).encode() + b'\n\n' + + +class CaptureTests(unittest.TestCase): + def test_pinned_protocol_usage_and_invalid_counts(self): + self.assertEqual(extract_usage('chat', {'usage': usage()}), (100, 7, 60)) + self.assertEqual(extract_usage('responses', {'response': {'usage': { + 'input_tokens': 100, 'output_tokens': 7, 'input_tokens_details': {'cached_tokens': 60}, + 'total_tokens': 107}}}), (100, 7, 60)) + self.assertEqual(extract_usage('messages', {'usage': {'input_tokens': 35, 'output_tokens': 7, + 'cache_read_input_tokens': 60, 'cache_creation_input_tokens': 5}}), (100, 7, 60)) + for invalid in ({**usage(), 'completion_tokens': True}, {**usage(), 'total_tokens': 999}, + {**usage(), 'prompt_cache_hit_tokens': 50}, {**usage(), 'prompt_tokens': -1}): + self.assertIsNone(extract_usage('chat', {'usage': invalid})) + + def test_fragmented_chat_sse_requires_final_marker(self): + content = sse({'choices': [{'delta': {'content': 'synthetic'}}]}) + content += sse({'choices': [{'finish_reason': 'stop'}], 'usage': usage()}) + b'data: [DONE]\n\n' + for size in (1, 2, 7, 257): + parser = UsageCapture('chat', 'text/event-stream; charset=utf-8') + for at in range(0, len(content), size): + parser.feed(content[at:at + size]) + self.assertEqual(parser.finish(), (100, 7, 60)) + parser = UsageCapture('chat', 'text/event-stream') + parser.feed(content.removesuffix(b'data: [DONE]\n\n')) + self.assertIsNone(parser.finish()) + + def test_responses_and_messages_stream_terminal_events(self): + response = {'type': 'response.completed', 'response': {'usage': {'input_tokens': 100, + 'output_tokens': 7, 'input_tokens_details': {'cached_tokens': 60}, 'total_tokens': 107}}} + parser = UsageCapture('responses', 'text/event-stream') + parser.feed(sse(response)) + self.assertEqual(parser.finish(), (100, 7, 60)) + parser = UsageCapture('messages', 'text/event-stream') + parser.feed(sse({'type': 'message_delta', 'usage': {'input_tokens': 40, + 'cache_read_input_tokens': 60, 'cache_creation_input_tokens': 0, 'output_tokens': 7}})) + self.assertIsNone(parser.finish()) + parser.feed(sse({'type': 'message_stop'})) + self.assertEqual(parser.finish(), (100, 7, 60)) + + def test_earlier_usage_is_not_final_if_later_chat_events_continue(self): + early = sse({'usage': usage(output_tokens=1)}) + content = sse({'choices': [{'delta': {'content': 'later'}}]}) + finish = sse({'choices': [{'finish_reason': 'stop'}]}) + final = sse({'choices': [], 'usage': usage(output_tokens=7)}) + for later in (content + finish, finish, sse({'unrecognized': 'later event'})): + with self.subTest(later=later): + parser = UsageCapture('chat', 'text/event-stream') + parser.feed(early + later + b'data: [DONE]\n\n') + self.assertIsNone(parser.finish()) + parser = UsageCapture('chat', 'text/event-stream') + parser.feed(early + later + final + b'data: [DONE]\n\n') + self.assertEqual(parser.finish(), (100, 7, 60)) + + def test_messages_usage_before_later_content_is_not_final(self): + delta = sse({'type': 'message_delta', 'usage': {'input_tokens': 40, + 'cache_read_input_tokens': 60, 'output_tokens': 7}}) + content = sse({'type': 'content_block_delta', 'delta': {'text': 'later'}}) + stop = sse({'type': 'message_stop'}) + parser = UsageCapture('messages', 'text/event-stream') + parser.feed(delta + content + stop) + self.assertIsNone(parser.finish()) + parser = UsageCapture('messages', 'text/event-stream') + parser.feed(delta + content + delta + stop) + self.assertEqual(parser.finish(), (100, 7, 60)) + + def test_messages_stop_with_conflicting_usage_is_unknown(self): + delta = sse({'type': 'message_delta', 'usage': {'input_tokens': 40, + 'cache_read_input_tokens': 60, 'output_tokens': 1}}) + stop = sse({'type': 'message_stop', 'usage': {'input_tokens': 40, + 'cache_read_input_tokens': 60, 'output_tokens': 2}}) + parser = UsageCapture('messages', 'text/event-stream') + parser.feed(delta + stop) + self.assertIsNone(parser.finish()) + + def test_done_cannot_replace_protocol_specific_terminal_event(self): + delta = sse({'type': 'message_delta', 'usage': {'input_tokens': 40, + 'cache_read_input_tokens': 60, 'output_tokens': 7}}) + parser = UsageCapture('messages', 'text/event-stream') + parser.feed(delta + b'data: [DONE]\n\n') + self.assertIsNone(parser.finish()) + parser = UsageCapture('messages', 'text/event-stream') + parser.feed(delta + sse({'type': 'message_stop'}) + b'data: [DONE]\n\n') + self.assertEqual(parser.finish(), (100, 7, 60)) + + def test_unfinished_data_after_terminal_invalidates_previous_usage(self): + body = sse({'usage': usage()}) + b'data: [DONE]\n\n' + for tail in (b'data: {"choices":[', b'data: {"choices":[]}\n'): + with self.subTest(tail=tail): + parser = UsageCapture('chat', 'text/event-stream') + parser.feed(body + tail) + self.assertIsNone(parser.finish()) + + def test_origin_header_requires_one_valid_ip(self): + headers = Message() + self.assertIsNone(client_ip(headers)) + headers['X-LLMDeploy-Client-IP'] = '::ffff:192.0.2.4' + self.assertEqual(client_ip(headers), '192.0.2.4') + headers['X-LLMDeploy-Client-IP'] = '192.0.2.5' + self.assertIsNone(client_ip(headers)) + + def test_session_fingerprint_matches_gateway_identity_without_storing_key(self): + headers = Message() + headers['Authorization'] = 'Bearer synthetic-gateway-key' + headers['X-Session-Id'] = 'synthetic-private-session' + digest = hashlib.sha256(b'synthetic-gateway-key').hexdigest() + self.assertEqual(session_fingerprint(headers), affinity_key(headers, b'{}', digest)) + other = Message() + other['Authorization'] = 'Bearer other-synthetic-key' + other['X-Session-Id'] = 'synthetic-private-session' + self.assertNotEqual(session_fingerprint(headers), session_fingerprint(other)) + headers['X-Session-Id'] = 'duplicate' + self.assertIsNone(session_fingerprint(headers)) + + def test_non_ascii_header_bytes_match_gateway_header_parser(self): + raw = (b'Authorization: Bearer synthetic-gateway-key\r\n' + b'X-Session-Id: synthetic-\xe4\xb8\xad') + actual = _headers(raw) + gateway = parse_headers(BytesIO(raw + b'\r\n\r\n')) + self.assertEqual(actual.get('X-Session-Id'), gateway.get('X-Session-Id')) + digest = hashlib.sha256(b'synthetic-gateway-key').hexdigest() + self.assertEqual(session_fingerprint(actual), affinity_key(gateway, b'{}', digest)) + self.assertIsNone(client_ip(_headers(b'X-LLMDeploy-Client-IP: \xff'))) + + def test_first_visible_delta_ignores_roles_and_empty_events(self): + observed = [] + parser = UsageCapture('chat', 'text/event-stream', on_first_output=lambda: observed.append(1)) + parser.feed(sse({'choices': [{'delta': {'role': 'assistant', 'content': ''}}]})) + self.assertEqual(observed, []) + parser.feed(sse({'choices': [{'delta': {'content': 'synthetic'}}]})) + parser.feed(sse({'choices': [{'delta': {'content': 'more'}}]})) + self.assertEqual(observed, [1]) + response = UsageCapture('responses', 'text/event-stream', on_first_output=lambda: observed.append(2)) + response.feed(sse({'type': 'response.created'})) + response.feed(sse({'type': 'response.output_text.delta', 'delta': 'synthetic'})) + messages = UsageCapture('messages', 'text/event-stream', on_first_output=lambda: observed.append(3)) + messages.feed(sse({'type': 'message_start'})) + messages.feed(sse({'type': 'content_block_delta', 'delta': {'thinking': 'synthetic'}})) + self.assertEqual(observed, [1, 2, 3]) + + def test_body_session_fallback_matches_gateway_for_fragmented_json(self): + headers = Message() + headers['Authorization'] = 'Bearer synthetic-gateway-key' + digest = hashlib.sha256(b'synthetic-gateway-key').hexdigest() + cases = [ + {'prompt_cache_key': 'synthetic-private-cache-key', + 'messages': [{'role': 'user', 'content': 'synthetic-private-prompt'}]}, + {'messages': [{'role': 'system', 'content': 'ignored'}, + {'role': 'user', 'content': 'synthetic-private-prompt'}, + {'role': 'user', 'content': 'later'}]}, + {'messages': [{'role': 'user', 'content': [{'type': 'text', 'text': '测试'}, + {'z': 2, 'a': 'synthetic'}]}]}, + {'messages': [{'role': 'assistant', 'content': 'no user'}]}, + ] + for value in cases: + body = json.dumps(value, ensure_ascii=False).encode() + for size in (1, 7, 67): + budget = BodyBudget(2 * len(body)) + capture = SessionBodyCapture(digest, budget=budget) + for start in range(0, len(body), size): + capture.feed(body[start:start + size]) + self.assertEqual(capture.finish(), affinity_key(headers, body, digest)) + self.assertEqual(budget.used, 0) + + def test_session_body_budget_and_size_fail_closed(self): + budget = BodyBudget(5) + first = SessionBodyCapture('synthetic-digest', budget=budget, max_body=10) + second = SessionBodyCapture('synthetic-digest', budget=budget, max_body=10) + first.feed(b'1234') + second.feed(b'12') + self.assertIsNone(second.finish()) + self.assertEqual(budget.used, 4) + first.close() + self.assertEqual(budget.used, 0) + oversized = SessionBodyCapture('synthetic-digest', budget=budget, max_body=3) + oversized.feed(b'1234') + self.assertIsNone(oversized.finish()) + self.assertEqual(budget.used, 0) + + +class SyntheticGateway(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def log_message(self, format, *args): + pass + + def do_GET(self): + self.send_response(200 if self.path == '/health' else 404) + self.send_header('Content-Length', '0') + self.end_headers() + + def do_POST(self): + if self.server.mode == 'reject': + time.sleep(getattr(self.server, 'reject_delay', 0)) + self.send_response(429) + self.send_header('Content-Length', '0') + self.send_header('Connection', 'close') + self.end_headers() + self.close_connection = True + return + payload = self.rfile.read(int(self.headers.get('Content-Length', 0))) + self.server.calls.append((payload, self.headers.get('Authorization'), + self.headers.get('X-LLMDeploy-Client-IP'))) + if self.server.barrier: + self.server.barrier.wait(timeout=5) + mode = self.server.mode + if mode == 'slow_headers': + self.server.stream_started.set() + self.server.stream_release.wait(3) + self.close_connection = True + return + if mode == 'slow': + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Connection', 'close') + self.end_headers() + self.wfile.write(sse({'choices': [{'delta': {'content': 'synthetic'}}]})) + self.wfile.flush() + self.server.stream_started.set() + self.server.stream_release.wait(3) + try: + self.wfile.write(sse({'usage': usage()}) + b'data: [DONE]\n\n') + self.wfile.flush() + except OSError: + pass + self.close_connection = True + return + if mode == 'json': + body = json.dumps({'content': 'synthetic-output', 'usage': usage()}).encode() + media = 'application/json' + elif mode == 'missing': + body = sse({'choices': [{'finish_reason': 'stop'}]}) + b'data: [DONE]\n\n' + media = 'text/event-stream' + else: + body = sse({'choices': [{'delta': {'content': 'synthetic-output'}}]}) + body += sse({'choices': [{'finish_reason': 'stop'}], 'usage': usage()}) + b'data: [DONE]\n\n' + media = 'text/event-stream' + self.server.last_body = body + self.send_response(200) + self.send_header('Content-Type', media) + if mode in ('slow_length', 'slow_chunked'): + if mode == 'slow_chunked': + self.send_header('Transfer-Encoding', 'chunked') + else: + self.send_header('Content-Length', str(len(body))) + self.end_headers() + if mode == 'slow_chunked': + self.wfile.write(f'{len(body):x}\r\n'.encode()) + self.wfile.write(body[:24]) + self.wfile.flush() + self.server.stream_started.set() + self.server.stream_release.wait(3) + try: + self.wfile.write(body[24:] + (b'\r\n0\r\n\r\n' if mode == 'slow_chunked' else b'')) + self.wfile.flush() + except OSError: + pass + return + if mode == 'chunked': + self.send_header('Transfer-Encoding', 'chunked') + self.end_headers() + for at in range(0, len(body), 11): + part = body[at:at + 11] + self.wfile.write(f'{len(part):x}\r\n'.encode() + part + b'\r\n') + self.wfile.write(b'0\r\n\r\n') + self.wfile.flush() + return + self.send_header('Content-Length', str(len(body) + (10 if mode == 'truncate' else 0))) + self.end_headers() + for at in range(0, len(body), 13): + self.wfile.write(body[at:at + 13]) + self.wfile.flush() + if mode == 'truncate': + self.close_connection = True + + +class ProxyTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.path = Path(self.temp.name) / 'usage.sqlite3' + class SyntheticServer(ThreadingHTTPServer): + request_queue_size = 128 + def handle_error(self, request, client_address): + pass + self.upstream = SyntheticServer(('127.0.0.1', 0), SyntheticGateway) + self.upstream.daemon_threads = True + self.upstream.mode = 'sse' + self.upstream.calls = [] + self.upstream.barrier = None + self.upstream.stream_started = threading.Event() + self.upstream.stream_release = threading.Event() + self.start(self.upstream) + self.proxy = Proxy(0, self.upstream.server_port, self.path) + self.start(self.proxy) + + def start(self, server): + thread = threading.Thread(target=server.serve_forever, kwargs={'poll_interval': .02}, daemon=True) + thread.start() + def cleanup(): + server.shutdown() + server.server_close() + thread.join(3) + self.addCleanup(cleanup) + + def request(self, ip='192.0.2.4'): + connection = HTTPConnection('127.0.0.1', self.proxy.server_port, timeout=5) + payload = b'{"prompt":"synthetic-prompt"}' + headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer synthetic-placeholder', + 'X-LLMDeploy-Client-IP': ip} + try: + connection.request('POST', '/v1/chat/completions', payload, headers) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + def wait_recorded(self): + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + with self.proxy.active_lock: + if self.proxy.active == 0: + self.assertTrue(self.proxy.recorder.flush()) + return + time.sleep(.01) + self.fail('proxy requests did not drain') + + def test_stream_is_unchanged_and_only_metadata_is_stored(self): + status, body = self.request() + self.assertEqual(status, 200) + self.assertEqual(body, self.upstream.last_body) + self.assertEqual(self.upstream.calls[-1], (b'{"prompt":"synthetic-prompt"}', + 'Bearer synthetic-placeholder', None)) + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual((data['totals']['input'], data['totals']['output'], data['totals']['cached']), (100, 7, 60)) + self.assertEqual(data['clients'][0]['ip'], '192.0.2.4') + for dbfile in Path(self.temp.name).glob('usage.sqlite3*'): + raw = dbfile.read_bytes() + for secret in (b'synthetic-prompt', b'synthetic-output', b'synthetic-placeholder'): + self.assertNotIn(secret, raw) + + def test_json_and_unknown_usage_are_separate(self): + self.upstream.mode = 'json' + self.assertEqual(self.request()[0], 200) + self.upstream.mode = 'missing' + self.assertEqual(self.request()[0], 200) + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual((data['totals']['requests'], data['totals']['known'], data['totals']['missing']), (2, 1, 1)) + self.assertFalse(data['totals']['complete']) + + def test_invalid_client_ip_is_shown_as_unknown(self): + self.assertEqual(self.request('invalid-source')[0], 200) + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual(data['totals']['unknown_source'], 1) + self.assertIsNone(data['clients'][0]['ip']) + + def test_parser_failure_does_not_interrupt_the_response(self): + with patch('deploy_stats.capture.UsageCapture.feed', side_effect=ValueError('synthetic failure')): + status, body = self.request() + self.assertEqual((status, body), (200, self.upstream.last_body)) + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual((data['totals']['known'], data['totals']['missing']), (0, 1)) + + def test_recorder_failure_does_not_interrupt_the_response(self): + self.proxy.recorder.heartbeat = .02 + with patch.object(self.proxy.recorder, 'submit', side_effect=OSError('synthetic failure')): + status, body = self.request() + deadline = time.monotonic() + 2 + while self.proxy.active and time.monotonic() < deadline: + time.sleep(.01) + self.assertEqual((status, body), (200, self.upstream.last_body)) + self.wait_recorded() + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + data = report(self.path, 'total', now=time.time() + .01) + if data['totals']['dropped']: + break + time.sleep(.02) + self.assertGreater(data['totals']['dropped'], 0) + self.assertFalse(data['totals']['complete']) + + def test_chunked_sse_is_forwarded_and_accounted(self): + self.upstream.mode = 'chunked' + status, body = self.request() + self.assertEqual((status, body), (200, self.upstream.last_body)) + self.wait_recorded() + self.assertEqual(report(self.path, 'total', now=time.time() + .01)['totals']['output'], 7) + + def test_sse_first_bytes_do_not_wait_for_remaining_body(self): + for mode in ('slow_length', 'slow_chunked'): + with self.subTest(mode=mode): + self.upstream.mode = mode + self.upstream.stream_started.clear() + self.upstream.stream_release.clear() + connection = socket.create_connection(('127.0.0.1', self.proxy.server_port), timeout=1) + connection.sendall(b'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n' + b'Content-Length: 2\r\nX-LLMDeploy-Client-IP: 192.0.2.4\r\n\r\n{}') + try: + self.assertTrue(self.upstream.stream_started.wait(1)) + received = b'' + while b'data:' not in received: + received += connection.recv(4096) + self.assertIn(b'data:', received) + finally: + self.upstream.stream_release.set() + connection.close() + + def test_truncated_stream_is_not_scored_as_zero(self): + self.upstream.mode = 'truncate' + with self.assertRaises(IncompleteRead): + self.request() + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual((data['totals']['known'], data['totals']['interrupted']), (0, 1)) + self.assertFalse(data['totals']['complete']) + + def test_concurrent_clients_are_attributed_once(self): + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda i: self.request(f'192.0.2.{i + 1}')[0], range(12))) + self.assertEqual(results, [200] * 12) + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual((data['totals']['known'], data['totals']['output'], len(data['clients'])), (12, 84, 12)) + + def test_session_summary_tracks_requests_without_storing_secrets(self): + for token, session in (('synthetic-key-a', 'synthetic-private-session'), + ('synthetic-key-a', 'synthetic-private-session'), + ('synthetic-key-b', 'synthetic-private-session')): + connection = HTTPConnection('127.0.0.1', self.proxy.server_port, timeout=5) + connection.request('POST', '/v1/chat/completions', b'{}', { + 'Authorization': 'Bearer ' + token, 'X-Session-Id': session, + 'X-LLMDeploy-Client-IP': '192.0.2.4'}) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual(data['session_count'], 2) + self.assertEqual(sorted(item['requests'] for item in data['sessions']), [1, 2]) + self.assertEqual(sorted(item['input'] for item in data['sessions']), [100, 200]) + self.assertTrue(all(item['first_visible'] is not None for item in data['requests'])) + for dbfile in Path(self.temp.name).glob('usage.sqlite3*'): + raw = dbfile.read_bytes() + for secret in (b'synthetic-key-a', b'synthetic-key-b', b'synthetic-private-session'): + self.assertNotIn(secret, raw) + + def test_body_fallback_session_groups_without_persisting_content(self): + bodies = [ + {'messages': [{'role': 'user', 'content': 'synthetic-private-first'}]}, + {'messages': [{'role': 'user', 'content': 'synthetic-private-first'}, + {'role': 'assistant', 'content': 'synthetic-response'}, + {'role': 'user', 'content': 'synthetic-second'}]}, + {'prompt_cache_key': 'synthetic-private-cache', + 'messages': [{'role': 'user', 'content': 'synthetic-private-first'}]}, + ] + for value in bodies: + connection = HTTPConnection('127.0.0.1', self.proxy.server_port, timeout=5) + connection.request('POST', '/v1/chat/completions', json.dumps(value).encode(), { + 'Authorization': 'Bearer synthetic-gateway-key', 'Content-Type': 'application/json', + 'X-LLMDeploy-Client-IP': '192.0.2.4'}) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.wait_recorded() + value = report(self.path, 'total', now=time.time() + .01) + self.assertEqual(value['totals']['unknown_session'], 0) + self.assertEqual(sorted(item['requests'] for item in value['sessions']), [1, 2]) + for dbfile in Path(self.temp.name).glob('usage.sqlite3*'): + raw = dbfile.read_bytes() + for secret in (b'synthetic-gateway-key', b'synthetic-private-first', b'synthetic-private-cache'): + self.assertNotIn(secret, raw) + + def test_body_session_parser_failure_does_not_interrupt_response(self): + with patch('deploy_stats.proxy.SessionBodyCapture.finish', side_effect=RuntimeError): + connection = HTTPConnection('127.0.0.1', self.proxy.server_port, timeout=5) + connection.request('POST', '/v1/chat/completions', b'{"messages":[]}', { + 'Authorization': 'Bearer synthetic-gateway-key', 'Content-Type': 'application/json'}) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.wait_recorded() + self.assertGreater(self.proxy.recorder.dropped, 0) + + def test_body_session_feed_or_budget_failure_does_not_interrupt_response(self): + for target in ('deploy_stats.proxy.SessionBodyCapture.feed', 'deploy_stats.session.BUDGET.reserve'): + with self.subTest(target=target), patch(target, side_effect=RuntimeError): + self.assertEqual(self.request()[0], 200) + self.wait_recorded() + self.assertGreaterEqual(self.proxy.recorder.dropped, 2) + + def test_active_snapshot_clears_when_completion_event_is_dropped(self): + self.upstream.mode = 'slow' + connection = socket.create_connection(('127.0.0.1', self.proxy.server_port), timeout=5) + connection.sendall(b'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n' + b'Content-Length: 2\r\nX-LLMDeploy-Client-IP: 192.0.2.4\r\n\r\n{}') + self.assertTrue(connection.recv(1024)) + self.assertTrue(self.upstream.stream_started.wait(2)) + deadline = time.monotonic() + 4 + while time.monotonic() < deadline: + data = report(self.path, 'total', now=time.time() + .01) + if data['active_ready'] and len(data['active']) == 1: + break + time.sleep(.05) + self.assertEqual(len(data['active']), 1) + try: + with patch.object(self.proxy.recorder.pending, 'put_nowait', side_effect=queue.Full): + connection.shutdown(socket.SHUT_RDWR) + connection.close() + deadline = time.monotonic() + 3 + while self.proxy.active and time.monotonic() < deadline: + time.sleep(.01) + self.assertEqual(self.proxy.active, 0) + deadline = time.monotonic() + 4 + while time.monotonic() < deadline: + data = report(self.path, 'total', now=time.time() + .01) + if data['active_ready'] and not data['active']: + break + time.sleep(.05) + self.assertTrue(data['active_ready']) + self.assertEqual(data['active'], []) + self.assertGreater(self.proxy.recorder.dropped, 0) + finally: + self.upstream.stream_release.set() + connection.close() + + def test_sixty_four_simultaneous_streams_are_independently_accounted(self): + self.upstream.barrier = threading.Barrier(64) + with ThreadPoolExecutor(max_workers=64) as pool: + statuses = list(pool.map(lambda i: self.request(f'192.0.2.{i + 1}')[0], range(64))) + self.assertEqual(statuses, [200] * 64) + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual((data['totals']['known'], data['totals']['output'], len(data['clients'])), (64, 448, 64)) + + def test_client_cancellation_is_recorded_as_incomplete(self): + self.upstream.mode = 'slow' + connection = socket.create_connection(('127.0.0.1', self.proxy.server_port), timeout=5) + connection.sendall(b'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n' + b'Content-Length: 2\r\nX-LLMDeploy-Client-IP: 192.0.2.4\r\n\r\n{}') + self.assertTrue(connection.recv(1024)) + self.assertTrue(self.upstream.stream_started.wait(2)) + connection.shutdown(socket.SHUT_RDWR) + connection.close() + try: + self.wait_recorded() + finally: + self.upstream.stream_release.set() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual((data['totals']['known'], data['totals']['interrupted']), (0, 1)) + + def test_client_cancellation_before_response_headers(self): + self.upstream.mode = 'slow_headers' + connection = socket.create_connection(('127.0.0.1', self.proxy.server_port), timeout=5) + connection.sendall(b'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n' + b'Content-Length: 2\r\nX-LLMDeploy-Client-IP: 192.0.2.4\r\n\r\n{}') + self.assertTrue(self.upstream.stream_started.wait(2)) + connection.shutdown(socket.SHUT_RDWR) + connection.close() + try: + self.wait_recorded() + finally: + self.upstream.stream_release.set() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual(data['totals']['interrupted'], 1) + + def test_large_body_is_forwarded_intact(self): + payload = b'x' * (1024 * 1024 + 123) + connection = HTTPConnection('127.0.0.1', self.proxy.server_port, timeout=5) + connection.request('POST', '/v1/chat/completions', payload, {'X-LLMDeploy-Client-IP': '192.0.2.4'}) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.wait_recorded() + self.assertEqual(self.upstream.calls[-1][0], payload) + + def test_large_combined_headers_reach_the_existing_gateway(self): + connection = HTTPConnection('127.0.0.1', self.proxy.server_port, timeout=5) + headers = {f'X-Synthetic-{i}': 'x' * 45000 for i in range(3)} + connection.request('POST', '/v1/chat/completions', b'{}', headers) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.wait_recorded() + self.assertEqual(len(self.upstream.calls), 1) + + @unittest.skipUnless(os.name == 'nt', 'Windows selector capacity') + def test_windows_keeps_480_streams_in_flight(self): + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + listener.listen(480) + listener.settimeout(.1) + accepted, clients = [], [] + stop = threading.Event() + + def accept_upstreams(): + while not stop.is_set(): + try: + conn, _ = listener.accept() + accepted.append(conn) + except socket.timeout: + continue + except OSError: + return + + thread = threading.Thread(target=accept_upstreams, daemon=True) + thread.start() + proxy = Proxy(0, listener.getsockname()[1], Path(self.temp.name) / 'capacity.sqlite3') + self.start(proxy) + try: + for _ in range(480): + conn = socket.create_connection(('127.0.0.1', proxy.server_port), timeout=15) + clients.append(conn) + conn.sendall(b'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n' + b'Content-Length: 2\r\nX-LLMDeploy-Client-IP: 192.0.2.4\r\n\r\n{}') + deadline = time.monotonic() + 20 + while (len(accepted) < 480 or proxy.connections < 480) and time.monotonic() < deadline: + time.sleep(.02) + self.assertEqual(len(accepted), 480) + self.assertEqual(proxy.connections, 480) + with socket.create_connection(('127.0.0.1', proxy.server_port), timeout=5) as extra: + extra.sendall(b'GET /health HTTP/1.1\r\nHost: localhost\r\n\r\n') + self.assertIn(b'429 Too Many Requests', extra.recv(1024)) + extra = HTTPConnection('127.0.0.1', proxy.server_port, timeout=5) + try: + extra.request('POST', '/v1/chat/completions', b'x' * (1024 * 1024)) + rejected = extra.getresponse() + self.assertEqual(rejected.status, 429) + rejected.read() + except (OSError, IncompleteRead): + # A peer still uploading at the limit may observe a reset. + pass + finally: + extra.close() + body = sse({'usage': usage()}) + b'data: [DONE]\n\n' + response = (b'HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: ' + + str(len(body)).encode() + b'\r\n\r\n' + body) + for conn in accepted: + conn.sendall(response) + for conn in clients: + received = bytearray() + while b'data: [DONE]\n\n' not in received: + part = conn.recv(4096) + self.assertTrue(part) + received.extend(part) + self.assertIn(b'HTTP/1.1 200 OK', received) + deadline = time.monotonic() + 10 + while proxy.connections and time.monotonic() < deadline: + time.sleep(.02) + self.assertEqual(proxy.connections, 0) + self.assertTrue(proxy.recorder.flush()) + data = report(proxy.recorder.path, 'total', now=time.time() + .01) + self.assertEqual(data['totals']['known'], 480) + finally: + for conn in clients + accepted: + conn.close() + stop.set() + listener.close() + thread.join(3) + + def test_gateway_can_reject_a_large_body_before_reading_it(self): + self.upstream.mode = 'reject' + for delay, expect in ((0, False), (.05, True), (.15, False)): + self.upstream.reject_delay = delay + for size in (2, 4095, 4096, 65535, 1024 * 1024): + connection = HTTPConnection('127.0.0.1', self.proxy.server_port, timeout=5) + headers = {'X-LLMDeploy-Client-IP': '192.0.2.4'} + if expect: + headers['Expect'] = '100-continue' + connection.request('POST', '/v1/chat/completions', b'x' * size, headers) + response = connection.getresponse() + self.assertEqual(response.status, 429) + response.read() + connection.close() + self.wait_recorded() + self.assertEqual(report(self.path, 'total', now=time.time() + .01)['totals']['failed'], 15) + + def test_large_invalid_auth_preserves_real_gateway_response(self): + keyfile = Path(self.temp.name) / 'auth-keys.json' + keyfile.write_text(json.dumps({'version': 1, 'records': {'fixture': { + 'sha256': hashlib.sha256(b'synthetic-gateway-key').hexdigest(), 'active': True}}})) + gateway = Gateway(('127.0.0.1', 0), keyfile, + state_path=Path(self.temp.name) / 'auth-state.json', + bindings_file=Path(self.temp.name) / 'auth-bindings.json', + backend_ports=[self.upstream.server_port], backend_managed_affinity=True) + self.start(gateway) + proxy = Proxy(0, gateway.server_port, Path(self.temp.name) / 'auth-stats.sqlite3') + self.start(proxy) + direct = HTTPConnection('127.0.0.1', gateway.server_port, timeout=5) + direct.request('POST', '/v1/chat/completions', b'{}', {'Authorization': 'Bearer incorrect-placeholder'}) + response = direct.getresponse() + expected = response.status, response.read() + direct.close() + self.assertEqual(expected[0], 401) + for size in (2, 4095, 4096, 65535, 65536, 1024 * 1024): + for expect in (False, True): + for _ in range(2): + connection = HTTPConnection('127.0.0.1', proxy.server_port, timeout=5) + headers = {'Authorization': 'Bearer incorrect-placeholder'} + if expect: + headers['Expect'] = '100-continue' + connection.request('POST', '/v1/chat/completions', b'x' * size, headers) + response = connection.getresponse() + self.assertEqual((response.status, response.read()), expected) + connection.close() + for size in (4095, 4096, 1024 * 1024): + sock = socket.create_connection(('127.0.0.1', proxy.server_port), timeout=5) + stop = threading.Event() + uploader = None + try: + sock.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n' + f'Authorization: Bearer incorrect-placeholder\r\nContent-Length: {size}\r\n' + 'Expect: 100-continue\r\n\r\n').encode()) + def upload(): + remaining = size + while remaining and not stop.is_set(): + amount = min(remaining, 512) + try: + sock.sendall(b'x' * amount) + except OSError: + return + remaining -= amount + time.sleep(.002) + uploader = threading.Thread(target=upload, daemon=True) + uploader.start() + response = HTTPResponse(sock) + response.begin() + self.assertEqual((response.status, response.read()), expected) + finally: + stop.set() + if uploader: + uploader.join(1) + sock.close() + + def test_dashboard_is_loopback_only_and_rejects_untrusted_hosts(self): + self.request() + self.wait_recorded() + dashboard = Dashboard(0, self.path) + self.start(dashboard) + connection = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=5) + connection.request('GET', '/api/stats?window=total') + result = connection.getresponse() + self.assertEqual(result.status, 200) + self.assertEqual(json.loads(result.read())['totals']['output'], 7) + connection.close() + for headers in ({'Host': 'untrusted.invalid'}, {'Origin': 'https://untrusted.invalid'}, + {'Sec-Fetch-Site': 'cross-site'}): + for path in ('/', '/api/active'): + connection = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=5) + connection.request('GET', path, headers=headers) + self.assertEqual(connection.getresponse().status, 403) + connection.close() + + def test_dashboard_validates_pagination_and_sort_parameters(self): + self.request() + self.wait_recorded() + dashboard = Dashboard(0, self.path) + self.start(dashboard) + queries = [ + ('?limit=10&sort=input&direction=asc&session_limit=50&session_sort=output&session_direction=desc', 200), + ('?limit=0', 400), ('?limit=201', 400), ('?session_limit=', 400), + ('?sort=not_a_column', 400), ('?session_sort=not_a_column', 400), + ('?direction=wrong', 400), ('?offset=1&offset=2', 400), + ('?snapshot_id=1', 400), ('?snapshot_at=1', 400), ('?snapshot_at=nan&snapshot_id=1', 400), + ] + for query, expected in queries: + connection = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=5) + try: + connection.request('GET', '/api/stats' + query) + response = connection.getresponse() + payload = json.loads(response.read()) + self.assertEqual(response.status, expected, query) + if expected == 200: + self.assertEqual((payload['request_limit'], payload['session_limit']), (10, 50)) + finally: + connection.close() + for path, expected in (('/api/active', 200), ('/api/active?bad=1', 400)): + connection = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=5) + try: + connection.request('GET', path) + response = connection.getresponse() + payload = json.loads(response.read()) + self.assertEqual(response.status, expected) + if expected == 200: + self.assertIn('active_ready', payload) + self.assertIn('active_sampled_at', payload) + finally: + connection.close() + + def test_existing_listener_is_never_reused(self): + with socket.socket() as occupied: + occupied.bind(('127.0.0.1', 0)) + occupied.listen(1) + port = occupied.getsockname()[1] + with self.assertRaises(OSError): + Dashboard(port, self.path) + with self.assertRaises(OSError): + Proxy(port, self.upstream.server_port, Path(self.temp.name) / 'must-not-exist.sqlite3') + self.assertFalse((Path(self.temp.name) / 'must-not-exist.sqlite3').exists()) + + def test_server_close_stops_a_running_proxy(self): + proxy = Proxy(0, self.upstream.server_port, Path(self.temp.name) / 'close.sqlite3') + thread = threading.Thread(target=proxy.serve_forever, daemon=True) + thread.start() + self.assertTrue(proxy.ready.wait(3)) + proxy.server_close() + thread.join(3) + self.assertTrue(proxy.finished.is_set()) + self.assertFalse(thread.is_alive()) + self.assertFalse(proxy.recorder.thread.is_alive()) + + def test_connection_limit_rejects_and_marks_a_gap(self): + self.upstream.mode = 'slow_headers' + self.proxy.recorder.heartbeat = .02 + with patch('deploy_stats.proxy.MAX_CONNECTIONS', 1): + first = socket.create_connection(('127.0.0.1', self.proxy.server_port), timeout=2) + first.sendall(b'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}') + self.assertTrue(self.upstream.stream_started.wait(2)) + second = socket.create_connection(('127.0.0.1', self.proxy.server_port), timeout=2) + try: + second.sendall(b'GET /health HTTP/1.1\r\nHost: localhost\r\n\r\n') + self.assertIn(b'429 Too Many Requests', second.recv(1024)) + finally: + second.close() + first.close() + self.upstream.stream_release.set() + self.wait_recorded() + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + data = report(self.path, 'total', now=time.time() + .01) + if data['totals']['dropped']: + break + time.sleep(.02) + self.assertGreater(data['totals']['dropped'], 0) + self.assertFalse(data['totals']['complete']) + + def test_real_gateway_authentication_is_preserved_through_stats_proxy(self): + keyfile = Path(self.temp.name) / 'synthetic-keys.json' + keyfile.write_text(json.dumps({'version': 1, 'records': {'fixture': { + 'sha256': hashlib.sha256(b'synthetic-gateway-key').hexdigest(), 'active': True}}})) + gateway = Gateway(('127.0.0.1', 0), keyfile, + state_path=Path(self.temp.name) / 'gateway-state.json', + bindings_file=Path(self.temp.name) / 'gateway-bindings.json', + backend_ports=[self.upstream.server_port], backend_managed_affinity=True) + self.start(gateway) + proxy = Proxy(0, gateway.server_port, Path(self.temp.name) / 'gateway-stats.sqlite3') + self.start(proxy) + for token, expected in ((None, 401), ('incorrect-placeholder', 401), ('synthetic-gateway-key', 200)): + headers = {'Content-Type': 'application/json', 'X-LLMDeploy-Client-IP': '192.0.2.8'} + if token is not None: + headers['Authorization'] = 'Bearer ' + token + connection = HTTPConnection('127.0.0.1', proxy.server_port, timeout=5) + connection.request('POST', '/v1/chat/completions', + json.dumps({'messages': [{'role': 'user', 'content': 'synthetic'}]}).encode(), headers) + response = connection.getresponse() + self.assertEqual(response.status, expected) + response.read() + connection.close() + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + with proxy.active_lock: + if proxy.active == 0: + break + time.sleep(.01) + self.assertTrue(proxy.recorder.flush()) + data = report(proxy.recorder.path, 'total', now=time.time() + .01) + self.assertEqual((data['totals']['known'], data['totals']['failed']), (1, 2)) + self.assertIsNone(self.upstream.calls[-1][1]) + for port in (gateway.server_port, proxy.server_port): + connection = HTTPConnection('127.0.0.1', port, timeout=5) + connection.request('POST', '/v1/chat/completions', b'{}', { + 'Authorization': b'Bearer invalid-\xff', 'X-Session-Id': b'synthetic-\xe4\xb8\xad'}) + response = connection.getresponse() + self.assertEqual(response.status, 401) + response.read() + connection.close() + connection = HTTPConnection('127.0.0.1', port, timeout=5) + connection.request('POST', '/v1/chat/completions', b'{}', { + 'Authorization': 'Bearer synthetic-gateway-key', 'X-Session-Id': b'synthetic-\xe4\xb8\xad'}) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + connection.close() + deadline = time.monotonic() + 3 + while proxy.active and time.monotonic() < deadline: + time.sleep(.01) + self.assertTrue(proxy.recorder.flush()) + raw = (b'Authorization: Bearer synthetic-gateway-key\r\n' + b'X-Session-Id: synthetic-\xe4\xb8\xad\r\n\r\n') + expected = affinity_key(parse_headers(BytesIO(raw)), b'{}', hashlib.sha256(b'synthetic-gateway-key').hexdigest()) + data = report(proxy.recorder.path, 'total', now=time.time() + .01) + self.assertIn(expected, [row['session'] for row in data['requests']]) + + @unittest.skipUnless(os.environ.get('LLMDEPLOY_CADDY'), 'Set LLMDEPLOY_CADDY for local Caddy integration') + def test_caddy_reload_preserves_an_existing_stream(self): + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + port = probe.getsockname()[1] + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + admin_port = probe.getsockname()[1] + old = Path(self.temp.name) / 'Caddyfile-before' + new = Path(self.temp.name) / 'Caddyfile-after' + template = '''{{ + admin 127.0.0.1:{admin} + auto_https off +}} +http://127.0.0.1:{port} {{ + bind 127.0.0.1 + log {{ + output discard + }} + reverse_proxy 127.0.0.1:{upstream} {{ + flush_interval 10ms + }} +}} +''' + old.write_text(template.format(admin=admin_port, port=port, upstream=self.upstream.server_port), encoding='utf-8') + new.write_text(template.format(admin=admin_port, port=port, upstream=self.proxy.server_port), encoding='utf-8') + environment = dict(os.environ) + environment['XDG_DATA_HOME'] = self.temp.name + environment['XDG_CONFIG_HOME'] = self.temp.name + environment['APPDATA'] = self.temp.name + environment['LOCALAPPDATA'] = self.temp.name + flags = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0 + process = subprocess.Popen([os.environ['LLMDEPLOY_CADDY'], 'run', '--config', str(old), + '--adapter', 'caddyfile'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=environment, creationflags=flags) + client = None + try: + deadline = time.monotonic() + 5 + while not (synthetic_caddy_owns_listener(process, port) and + synthetic_caddy_owns_listener(process, admin_port)): + if process.poll() is not None or time.monotonic() >= deadline: + self.fail('synthetic Caddy listener ownership could not be verified') + time.sleep(.05) + self.upstream.mode = 'slow_length' + client = socket.create_connection(('127.0.0.1', port), timeout=5) + client.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n' + 'Content-Length: 2\r\n\r\n{}').encode()) + self.assertTrue(self.upstream.stream_started.wait(2)) + received = bytearray() + while b'data:' not in received: + received.extend(client.recv(4096)) + self.upstream.mode = 'json' + self.assertTrue(synthetic_caddy_owns_listener(process, admin_port)) + self.assertTrue(synthetic_caddy_owns_listener(process, port)) + reload = subprocess.run([os.environ['LLMDEPLOY_CADDY'], 'reload', '--address', + f'127.0.0.1:{admin_port}', '--config', str(new), '--adapter', 'caddyfile'], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env=environment, creationflags=flags, timeout=5) + self.assertEqual(reload.returncode, 0) + self.assertTrue(synthetic_caddy_owns_listener(process, port)) + connection = HTTPConnection('127.0.0.1', port, timeout=5) + connection.request('POST', '/v1/chat/completions', b'{}') + response = connection.getresponse() + self.assertEqual(response.status, 200) + self.assertIn('application/json', response.getheader('Content-Type')) + response.read() + connection.close() + self.wait_recorded() + self.assertEqual(report(self.path, 'total', now=time.time() + .01)['totals']['known'], 1) + self.upstream.stream_release.set() + while b'data: [DONE]\n\n' not in received: + part = client.recv(4096) + self.assertTrue(part) + received.extend(part) + finally: + self.upstream.stream_release.set() + if client: + client.close() + if process.poll() is None: + process.terminate() + process.wait(timeout=5) + + @unittest.skipUnless(os.environ.get('LLMDEPLOY_CADDY'), 'Set LLMDEPLOY_CADDY for local Caddy integration') + def test_caddy_fallback_preserves_auth_and_never_replays_post(self): + keyfile = Path(self.temp.name) / 'fallback-keys.json' + keyfile.write_text(json.dumps({'version': 1, 'records': {'fixture': { + 'sha256': hashlib.sha256(b'synthetic-gateway-key').hexdigest(), 'active': True}}})) + gateway = Gateway(('127.0.0.1', 0), keyfile, + state_path=Path(self.temp.name) / 'fallback-state.json', + bindings_file=Path(self.temp.name) / 'fallback-bindings.json', + backend_ports=[self.upstream.server_port], backend_managed_affinity=True) + self.start(gateway) + proxy = Proxy(0, gateway.server_port, Path(self.temp.name) / 'fallback-stats.sqlite3') + self.start(proxy) + reservation = socket.socket() + if os.name == 'nt': + reservation.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) + reservation.bind(('127.0.0.1', 0)) + self.addCleanup(reservation.close) + primary_port = reservation.getsockname()[1] + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + port = probe.getsockname()[1] + config = Path(self.temp.name) / 'Caddyfile-fallback' + config.write_text('''{ + admin off + auto_https off +} +http://127.0.0.1:%d { + bind 127.0.0.1 + log { + output discard + } + reverse_proxy 127.0.0.1:%d 127.0.0.1:%d { + lb_policy first + fail_duration 2s + lb_try_duration 6s + lb_try_interval 20ms + header_up X-LLMDeploy-Client-IP {remote_host} + flush_interval 10ms + transport http { + dial_timeout 5s + } + } +} +''' % (port, primary_port, gateway.server_port), encoding='utf-8') + environment = dict(os.environ) + environment['XDG_DATA_HOME'] = self.temp.name + environment['XDG_CONFIG_HOME'] = self.temp.name + environment['APPDATA'] = self.temp.name + environment['LOCALAPPDATA'] = self.temp.name + flags = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0 + process = subprocess.Popen([os.environ['LLMDEPLOY_CADDY'], 'run', '--config', str(config), + '--adapter', 'caddyfile'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=environment, creationflags=flags) + primary = None + try: + deadline = time.monotonic() + 5 + while not synthetic_caddy_owns_listener(process, port): + if process.poll() is not None or time.monotonic() >= deadline: + self.fail('synthetic Caddy listener ownership could not be verified') + time.sleep(.05) + + def request(key): + self.assertTrue(synthetic_caddy_owns_listener(process, port)) + connection = HTTPConnection('127.0.0.1', port, timeout=5) + connection.request('POST', '/v1/chat/completions', + b'{"messages":[{"role":"user","content":"synthetic"}]}', + {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + key}) + return connection, connection.getresponse() + + connection, response = request('incorrect-placeholder') + self.assertEqual(response.status, 401) + response.read() + connection.close() + self.assertEqual(len(self.upstream.calls), 0) + connection, response = request('synthetic-gateway-key') + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.assertEqual(len(self.upstream.calls), 1) + + class Primary(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + def log_message(self, format, *args): + pass + def do_POST(self): + payload = self.rfile.read(int(self.headers['Content-Length'])) + self.server.calls += 1 + if self.server.mode == 'drop': + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + return + if self.server.mode == 'sse-drop': + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Content-Length', '100') + self.end_headers() + self.wfile.write(b'data: partial\n\n') + self.wfile.flush() + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + return + upstream = HTTPConnection('127.0.0.1', proxy.server_port, timeout=5) + upstream.request('POST', self.path, payload, { + 'Content-Type': self.headers.get('Content-Type'), + 'Authorization': self.headers.get('Authorization'), + 'X-LLMDeploy-Client-IP': self.headers.get('X-LLMDeploy-Client-IP')}) + result = upstream.getresponse() + body = result.read() + self.send_response(result.status) + self.send_header('Content-Type', result.getheader('Content-Type')) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + upstream.close() + + class PrimaryServer(ThreadingHTTPServer): + def handle_error(self, request, client_address): + pass # Deliberate reset scenarios in this test. + + reservation.close() + primary = PrimaryServer(('127.0.0.1', primary_port), Primary) + primary.mode, primary.calls = 'normal', 0 + primary.daemon_threads = True + self.start(primary) + time.sleep(2.2) + connection, response = request('synthetic-gateway-key') + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.assertEqual((primary.calls, len(self.upstream.calls)), (1, 2)) + self.assertTrue(proxy.recorder.flush()) + self.assertEqual(report(proxy.recorder.path, 'total', now=time.time() + .01)['totals']['known'], 1) + + primary.mode = 'drop' + try: + connection, response = request('synthetic-gateway-key') + self.assertNotEqual(response.status, 200) + response.read() + connection.close() + except (OSError, IncompleteRead): + pass + self.assertEqual((primary.calls, len(self.upstream.calls)), (2, 2)) + connection, response = request('synthetic-gateway-key') + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.assertEqual((primary.calls, len(self.upstream.calls)), (2, 3)) + + time.sleep(2.2) + primary.mode = 'sse-drop' + connection, response = request('synthetic-gateway-key') + self.assertEqual(response.status, 200) + self.assertEqual(response.read(5), b'data:') + with self.assertRaises(IncompleteRead): + response.read() + connection.close() + self.assertEqual((primary.calls, len(self.upstream.calls)), (3, 3)) + + time.sleep(2.2) + primary.mode = 'normal' + connection, response = request('synthetic-gateway-key') + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.assertEqual((primary.calls, len(self.upstream.calls)), (4, 4)) + finally: + process.terminate() + process.wait(timeout=5) + + @unittest.skipUnless(os.environ.get('LLMDEPLOY_CADDY'), 'Set LLMDEPLOY_CADDY for local Caddy integration') + def test_caddy_overwrites_spoofed_origin_and_preserves_response(self): + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + port = probe.getsockname()[1] + config = Path(self.temp.name) / 'Caddyfile' + config.write_text('''{ + admin off + auto_https off +} +http://127.0.0.1:%d { + bind 127.0.0.1 + log { + output discard + } + reverse_proxy 127.0.0.1:%d { + header_up X-LLMDeploy-Client-IP {remote_host} + flush_interval 10ms + } +} +''' % (port, self.proxy.server_port), encoding='utf-8') + environment = dict(os.environ) + environment['XDG_DATA_HOME'] = self.temp.name + environment['XDG_CONFIG_HOME'] = self.temp.name + environment['APPDATA'] = self.temp.name + environment['LOCALAPPDATA'] = self.temp.name + flags = subprocess.CREATE_NO_WINDOW if os.name == 'nt' else 0 + process = subprocess.Popen([os.environ['LLMDEPLOY_CADDY'], 'run', '--config', str(config), + '--adapter', 'caddyfile'], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=environment, creationflags=flags) + try: + deadline = time.monotonic() + 5 + while not synthetic_caddy_owns_listener(process, port): + if process.poll() is not None or time.monotonic() >= deadline: + self.fail('synthetic Caddy listener ownership could not be verified') + time.sleep(.05) + connection = HTTPConnection('127.0.0.1', port, timeout=5) + connection.request('POST', '/v1/chat/completions', b'{}', { + 'Authorization': 'Bearer synthetic-placeholder', + 'X-LLMDeploy-Client-IP': '192.0.2.99'}) + response = connection.getresponse() + self.assertEqual((response.status, response.read()), (200, self.upstream.last_body)) + connection.close() + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual(data['clients'][0]['ip'], '127.0.0.1') + self.assertEqual(self.upstream.calls[-1][1:], ('Bearer synthetic-placeholder', None)) + connection = HTTPConnection('127.0.0.1', port, timeout=5) + connection.request('POST', '/v1/chat/completions', b'{}', { + 'Authorization': 'Bearer synthetic-placeholder', 'X-Session-Id': b'synthetic-\xe4\xb8\xad'}) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.wait_recorded() + gateway_headers = parse_headers(BytesIO(b'Authorization: Bearer synthetic-placeholder\r\n' + b'X-Session-Id: synthetic-\xe4\xb8\xad\r\n\r\n')) + expected = affinity_key(gateway_headers, b'{}', hashlib.sha256(b'synthetic-placeholder').hexdigest()) + self.assertIn(expected, [row['session'] for row in report(self.path, 'total', now=time.time() + .01)['requests']]) + large_headers = {f'X-Synthetic-{i}': 'x' * 5000 for i in range(3)} + connection = HTTPConnection('127.0.0.1', port, timeout=5) + connection.request('POST', '/v1/chat/completions', b'{}', large_headers) + response = connection.getresponse() + self.assertEqual(response.status, 200) + response.read() + connection.close() + self.upstream.mode = 'slow_length' + self.upstream.stream_started.clear() + self.upstream.stream_release.clear() + client = socket.create_connection(('127.0.0.1', port), timeout=1) + client.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n' + 'Content-Length: 2\r\n\r\n{}').encode()) + try: + self.assertTrue(self.upstream.stream_started.wait(1)) + received = b'' + while b'data:' not in received: + received += client.recv(4096) + self.assertIn(b'data:', received) + client.shutdown(socket.SHUT_RDWR) + client.close() + self.wait_recorded() + data = report(self.path, 'total', now=time.time() + .01) + self.assertEqual(data['totals']['interrupted'], 1) + finally: + self.upstream.stream_release.set() + client.close() + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=5) + + +class ReportTests(unittest.TestCase): + def test_overview_phase_estimates_use_measurable_success_samples(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('fixture', now - 1000, now, now, 0)) + statement = '''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) VALUES(?,?,?,?,?,?,?,?,?,?,?)''' + db.execute(statement, ('fixture', now - 10, '192.0.2.4', 2, 200, 'known', 100, 7, 50, None, None)) + db.commit() + empty = report(path, 'total', now=now + .01)['totals'] + self.assertIsNone(empty['first_mean']) + self.assertIsNone(empty['prefill_estimate_tps']) + self.assertIsNone(empty['decode_estimate_tps']) + for elapsed, prompt, output, cached, first in ((2.5, 100, 11, 40, .5), + (4.5, 50, 6, 0, 1.5), + (2, 50, 1, 10, 1)): + db.execute(statement, ('fixture', now - 5, '192.0.2.4', elapsed, 200, 'known', + prompt, output, cached, None, first)) + db.execute(statement, ('fixture', now - 5, '192.0.2.4', 2, 500, 'failed', + None, None, None, None, None)) + db.commit() + db.close() + totals = report(path, 'total', now=now + .01)['totals'] + self.assertEqual((totals['known'], totals['first_samples'], totals['prefill_samples'], totals['decode_samples']), + (4, 3, 3, 2)) + self.assertAlmostEqual(totals['first_mean'], 1) + self.assertAlmostEqual(totals['prefill_estimate_tps'], 50) + self.assertAlmostEqual(totals['decode_estimate_tps'], 3) + self.assertAlmostEqual(totals['hit_rate'], 1 / 3) + + def test_subsecond_coverage_gap_is_incomplete_everywhere(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = datetime(2026, 1, 3, 12, tzinfo=timezone(timedelta(hours=8))).timestamp() + start = now - 12 * 3600 + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('first', start, now - 300.4, now - 300.4, 0)) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('second', now - 299.6, now, now, 0)) + db.commit() + db.close() + value = report(path, 'today', now=now) + self.assertFalse(value['totals']['complete']) + self.assertFalse(value['days'][-1]['complete']) + bucket = next(point for point in value['series_24h'] if point['start'] == now - 600) + self.assertFalse(bucket['complete']) + self.assertAlmostEqual(bucket['covered_seconds'], 599.2, places=4) + + def test_stale_active_snapshot_is_unknown_even_with_live_heartbeat(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('live', now - 60, now - 1, None, 0)) + db.execute('INSERT INTO active_samples VALUES (?,?)', ('live', now - 4)) + db.execute('INSERT INTO active_requests VALUES (?,?,?,?,?,?)', + ('fixture', 'live', now - 30, '192.0.2.4', None, None)) + db.commit() + db.close() + value = report(path, 'total', now=now) + self.assertTrue(value['collector_live']) + self.assertFalse(value['active_ready']) + self.assertEqual(value['active'], []) + lightweight = active_report(path, now=now) + self.assertTrue(lightweight['collector_live']) + self.assertFalse(lightweight['active_ready']) + self.assertEqual(lightweight['active'], []) + + def test_lightweight_active_report_matches_full_report(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('live', now - 60, now - 1, None, 0)) + db.execute('INSERT INTO active_samples VALUES (?,?)', ('live', now - 1)) + db.execute('INSERT INTO active_requests VALUES (?,?,?,?,?,?)', + ('fixture', 'live', now - 30, '192.0.2.4', 'synthetic-session', None)) + db.commit() + db.close() + full = report(path, 'total', now=now) + lightweight = active_report(path, now=now) + for key in ('collector_live', 'active_ready', 'active_sampled_at', 'active'): + self.assertEqual(lightweight[key], full[key]) + self.assertTrue(lightweight['active_ready']) + self.assertEqual(len(lightweight['active']), 1) + self.assertFalse(active_report(path, now=now + 3)['active_ready']) + + def test_session_and_request_pagination_do_not_drop_history(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('fixture', now - 1000, now, now, 0)) + for i in range(205): + db.execute('''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) + VALUES(?,?,?,?,?,?,?,?,?,?,?)''', + ('fixture', now - 205 + i, '192.0.2.4', 2, 200, 'known', 100, 7, 60, + f'session-{i % 105}', .5)) + db.commit() + db.close() + value = report(path, 'total', now=now + .01, request_offset=100, session_offset=100) + self.assertEqual((value['request_count'], len(value['requests'])), (205, 100)) + self.assertEqual((value['session_count'], len(value['sessions'])), (105, 5)) + self.assertEqual(value['totals']['input'], 20500) + + def test_sorting_is_global_before_independent_pagination(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('fixture', now - 1000, now, now, 0)) + for i in range(73): + known = i % 11 != 0 + db.execute('''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) + VALUES(?,?,?,?,?,?,?,?,?,?,?)''', + ('fixture', now - 100 + i, None if i == 5 else f'192.0.2.{i % 15 + 1}', 2, 200, 'known' if known else 'missing', + 1000 - i if known else None, i + 1 if known else None, 0 if known else None, + f'session-{i % 17:02}', .5 if known else None)) + db.commit() + db.close() + for size in (10, 20, 50): + for direction in ('asc', 'desc'): + pages = [report(path, 'total', now=now + .01, request_offset=offset, request_limit=size, + request_sort='input', request_dir=direction, session_limit=10) + for offset in range(0, 73, size)] + records = [item for page in pages for item in page['requests']] + self.assertEqual(len({item['id'] for item in records}), 73) + inputs = [item['input'] for item in records if item['input'] is not None] + self.assertEqual(inputs, sorted(inputs, reverse=direction == 'desc')) + self.assertTrue(all(item['input'] is None for item in records[len(inputs):])) + self.assertEqual(pages[0]['request_limit'], size) + self.assertEqual(pages[0]['session_limit'], 10) + for direction in ('asc', 'desc'): + ip_page = report(path, 'total', now=now + .01, request_limit=200, + request_sort='ip', request_dir=direction)['requests'] + self.assertIsNone(ip_page[-1]['ip']) + octets = [int(item['ip'].split('.')[-1]) for item in ip_page if item['ip'] is not None] + self.assertEqual(octets, sorted(octets, reverse=direction == 'desc')) + session_pages = [report(path, 'total', now=now + .01, request_limit=20, session_offset=offset, + session_limit=10, session_sort='output', session_dir='asc') + for offset in (0, 10)] + sessions = [item for page in session_pages for item in page['sessions']] + self.assertEqual(len({item['session'] for item in sessions}), 17) + self.assertEqual([item['output'] for item in sessions], sorted(item['output'] for item in sessions)) + clamped = report(path, 'total', now=now + .01, request_offset=10000, request_limit=20, + session_offset=10000, session_limit=10) + self.assertEqual((clamped['request_offset'], clamped['session_offset']), (60, 10)) + for bad in ({'request_sort': 'bad'}, {'session_sort': 'bad'}, {'request_dir': 'bad'}, + {'session_dir': 'bad'}, {'request_limit': 0}, {'session_limit': 201}): + with self.assertRaises(ValueError): + report(path, 'total', now=now + .01, **bad) + + def test_session_sort_puts_unknown_source_last_in_both_directions(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('fixture', now - 100, now, now, 0)) + for i, ip in enumerate((None, '192.0.2.1', '192.0.2.20')): + db.execute('''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) + VALUES(?,?,?,?,?,?,?,?,?,?,?)''', + ('fixture', now - 10 + i, ip, 2, 200, 'known', 100, 7, 0, f'session-{i}', .5)) + db.commit() + db.close() + for direction in ('asc', 'desc'): + sessions = report(path, 'total', now=now + .01, session_sort='ips', session_dir=direction)['sessions'] + self.assertEqual(sessions[-1]['ips'], ['未知来源']) + + def test_page_snapshot_excludes_new_and_late_commits_but_live_totals_advance(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('fixture', now - 200000, now, now, 0)) + statement = '''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) VALUES(?,?,?,?,?,?,?,?,?,?,?)''' + for i in range(40): + db.execute(statement, ('fixture', now - 100 + i, '192.0.2.4', 2, 200, 'known', + 100, 7, 0, f'session-{i:02}', .5)) + db.commit() + db.close() + first = report(path, 'total', now=now, request_limit=20, session_limit=20) + db = _open(path) + for i, finished in enumerate((now + 1, now - 1)): + db.execute(statement, ('fixture', finished, '192.0.2.4', 2, 200, 'known', + 100, 7, 0, f'new-session-{i}', .5)) + db.execute('UPDATE runs SET last_seen=?,ended=? WHERE id=?', (now + 5, now + 5, 'fixture')) + db.commit() + db.close() + second = report(path, 'total', now=now + 5, request_offset=20, request_limit=20, + session_offset=20, session_limit=20, + snapshot_at=first['snapshot_at'], snapshot_id=first['snapshot_id']) + self.assertEqual((second['request_count'], second['session_count'], second['totals']['requests']), (40, 40, 42)) + self.assertEqual(len({item['id'] for item in first['requests'] + second['requests']}), 40) + self.assertEqual(len({item['session'] for item in first['sessions'] + second['sessions']}), 40) + latest = report(path, 'total', now=now + 5) + self.assertEqual((latest['request_count'], latest['session_count']), (42, 42)) + + def test_snapshot_completeness_uses_snapshot_window(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('fixture', now - 86400, now, now, 0)) + db.execute('''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) VALUES(?,?,?,?,?,?,?,?,?,?,?)''', + ('fixture', now - 121, '192.0.2.4', 2, 200, 'known', 100, 7, 0, 'session', .5)) + db.commit() + db.close() + value = report(path, '24h', now=now, snapshot_at=now - 120, snapshot_id=1) + self.assertTrue(value['totals']['complete']) + self.assertFalse(value['snapshot_complete']) + self.assertEqual(value['sessions'][0]['input'], 100) + + def test_schema_one_migrates_without_losing_history(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + db = sqlite3.connect(path) + db.executescript('''CREATE TABLE stats_meta (key TEXT PRIMARY KEY,value TEXT NOT NULL); + INSERT INTO stats_meta VALUES ('application','llmdeploy-gateway-stats'); + CREATE TABLE runs (id TEXT PRIMARY KEY,started REAL NOT NULL,last_seen REAL NOT NULL, + ended REAL,dropped INTEGER NOT NULL DEFAULT 0); + CREATE TABLE requests (id INTEGER PRIMARY KEY,run_id TEXT NOT NULL,finished REAL NOT NULL, + client_ip TEXT,elapsed REAL NOT NULL,http_status INTEGER,state TEXT NOT NULL, + input_tokens INTEGER,output_tokens INTEGER,cached_tokens INTEGER); + INSERT INTO runs VALUES ('old',100,200,200,0); + INSERT INTO requests VALUES (1,'old',150,'192.0.2.4',2,200,'known',100,7,60); + PRAGMA user_version=1;''') + db.commit() + db.close() + migrated = _open(path) + self.assertEqual(migrated.execute('PRAGMA user_version').fetchone()[0], 2) + self.assertEqual(migrated.execute('SELECT input_tokens,session_id,first_visible FROM requests').fetchone(), + (100, None, None)) + migrated.execute("UPDATE runs SET started=?,last_seen=?,ended=? WHERE id='old'", + (1800000100, 1800000200, 1800000200)) + migrated.execute("UPDATE requests SET finished=1800000150 WHERE run_id='old'") + migrated.commit() + migrated.close() + value = report(path, 'total', now=1800000201) + self.assertEqual((value['totals']['input'], value['totals']['unknown_session']), (100, 1)) + + def test_backend_metric_subset_and_unavailable(self): + class Metrics(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + def do_GET(self): + body = (b'deepinfer:requests_running 2\n' b'deepinfer:requests_waiting 1\n' + b'deepinfer:workers_alive 8\n' b'deepinfer:admission_ready 1\n' + b'deepinfer:prompt_tokens_total 100\n' b'deepinfer:prompt_cache_hit_tokens_total 30\n' + b'deepinfer:prompt_cache_hit_requests_total 4\n') + if getattr(self.server, 'invalid_ratio', False): + body = body.replace(b'prompt_cache_hit_tokens_total 30', b'prompt_cache_hit_tokens_total 130') + self.send_response(200) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + server = ThreadingHTTPServer(('127.0.0.1', 0), Metrics) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + value = read_backend(server.server_port) + self.assertTrue(value['available']) + self.assertEqual((value['running'], value['waiting'], value['cached_total']), (2, 1, 30)) + self.assertEqual(value['hit_rate'], .3) + server.invalid_ratio = True + self.assertIsNone(read_backend(server.server_port)['hit_rate']) + finally: + server.shutdown() + server.server_close() + thread.join(3) + + def test_unclean_stale_run_never_claims_a_historical_day_is_complete(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + tz = timezone(timedelta(hours=8)) + now = datetime(2026, 1, 3, 12, tzinfo=tz).timestamp() + start = datetime(2026, 1, 1, 0, tzinfo=tz).timestamp() + sample = datetime(2026, 1, 2, 12, tzinfo=tz).timestamp() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('unclean', start, now - 7200, None, 0)) + db.execute('''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens) VALUES(?,?,?,?,?,?,?,?,?)''', + ('unclean', sample, '192.0.2.4', 2, 200, 'known', 100, 7, 60)) + db.commit() + db.close() + value = report(path, 'total', now=now) + day = next(item for item in value['days'] if item['date'] == '2026-01-02') + self.assertEqual(day['covered_seconds'], 86400) + self.assertEqual(day['known'], 1) + self.assertFalse(day['complete']) + + def test_cleanly_drained_run_can_cover_a_complete_historical_day(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + tz = timezone(timedelta(hours=8)) + now = datetime(2026, 1, 3, 12, tzinfo=tz).timestamp() + start = datetime(2026, 1, 1, 0, tzinfo=tz).timestamp() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('clean', start, now - 7200, now - 7200, 0)) + db.commit() + db.close() + value = report(path, 'total', now=now) + day = next(item for item in value['days'] if item['date'] == '2026-01-02') + self.assertTrue(day['complete']) + self.assertEqual(day['unclean_runs'], 0) + + def test_local_days_and_gaps_are_reported_without_zero_filling(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + now = datetime(2026, 1, 2, 1, tzinfo=timezone(timedelta(hours=8))).timestamp() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('fixture', now - 7200, now - 1, None, 0)) + for stamp, state, values in ((now - 3700, 'known', (100, 7, 60)), + (now - 100, 'missing', (None, None, None))): + db.execute('''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens) VALUES(?,?,?,?,?,?,?,?,?)''', + ('fixture', stamp, '192.0.2.4', 2, 200, state, *values)) + db.commit() + db.close() + value = report(path, '24h', now=now) + self.assertFalse(value['totals']['complete']) + self.assertEqual(value['totals']['known'], 1) + self.assertEqual(value['totals']['missing'], 1) + yesterday = next(d for d in value['days'] if d['date'] == '2026-01-01') + self.assertEqual((yesterday['input'], yesterday['output'], yesterday['cached']), (100, 7, 60)) + self.assertTrue(any(not point['complete'] and not point['known'] for point in value['series_24h'])) + + def test_existing_unrelated_database_is_not_repurposed(self): + for version in (0, 1): + with self.subTest(version=version), tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'existing.sqlite3' + with closing(sqlite3.connect(path)) as db: + db.execute('CREATE TABLE keep_me(value TEXT)') + db.execute("INSERT INTO keep_me VALUES ('preserve')") + db.execute(f'PRAGMA user_version={version}') + db.commit() + with self.assertRaises(ValueError): + Recorder(path) + with closing(sqlite3.connect(path)) as db: + self.assertEqual(db.execute('SELECT value FROM keep_me').fetchone()[0], 'preserve') + self.assertEqual(db.execute('PRAGMA journal_mode').fetchone()[0], 'delete') + + def test_full_writer_queue_is_reported_as_a_gap(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'usage.sqlite3' + recorder = Recorder(path, queue_size=1, heartbeat=.02) + with closing(sqlite3.connect(path, timeout=1)) as blocker: + blocker.execute('BEGIN IMMEDIATE') + event = (time.time(), '192.0.2.4', 1.0, 200, 'known', 10, 2, 5) + accepted = [recorder.submit(event) for _ in range(20)] + self.assertIn(False, accepted) + blocker.rollback() + self.assertTrue(recorder.flush()) + self.assertTrue(recorder.close()) + data = report(path, 'total', now=time.time() + .01) + self.assertGreater(data['totals']['dropped'], 0) + self.assertFalse(data['totals']['complete']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 0000000..0388371 --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,259 @@ +from copy import deepcopy +from http.client import HTTPConnection +import threading +import time +import unittest +from unittest import mock + +from deploy_stats import remote_observe, telemetry, web + + +HASHES = {key: 'a' * 64 for key in telemetry.SOURCE_KEYS} + + +def node(label='one', port=19001): + return {'label': label, 'backend_port': port, 'ssh_target': 'user@example.invalid', + 'container': 'example-container', 'image_fingerprint': 'c' * 64, 'driver_sha256': '9' * 64, + 'source_sha256': HASHES.copy(), 'rank_devices': [0, 1], + 'prefill_flops_per_token': 10.0, 'peak_flops_per_device': 1000.0} + + +def sample(now, *, phase=True): + rows = [{'kind': 'stats', 'at': now - 1 + rank * .01, 'rank': rank, 'device': rank, + 'step': 10, 'is_prompt': phase, 'current_tps': 100 + rank * 10, + 'current_sps': 10 + rank * .01} for rank in range(2)] + rows += [{'kind': 'cpu', 'at': now - .9 + rank * .01, 'rank': rank, 'device': rank, 'step': 10, + 'timers': {'prefill': {'mean_ms': 20 + rank * 10, 'calls': rank + 1}}} for rank in range(2)] + return {'schema': 1, 'source_id': 'b' * 64, 'host_fingerprint': 'd' * 64, 'image_fingerprint': 'c' * 64, + 'driver_sha256': '9' * 64, + 'source_sha256': HASHES.copy(), 'source_before_start': True, + 'source_started_at': now - 100, 'started_at': now - 2, 'finished_at': now, + 'hbm': [{'id': i, 'started_at': now - .55 + i * .01, 'finished_at': now - .45 + i * .01, + 'hbm_bandwidth_percent': 20 + i * 10} for i in range(2)], + 'engine_rows': rows} + + +class TelemetryTests(unittest.TestCase): + def test_log_parsing_keeps_good_fields_when_phase_share_is_nan(self): + line = ('2026-11-02T00:00:00.123456789Z INFO RANK 0, GPU 0, step 17, [stats] ' + '(is_prompt True | cur_SPS 10.000 | cur_TPS 12.5 | prompt_TPS 0.0 | ' + 'completion_TPS 0.0 | prefill/decode nan%/nan% | MFU 0.123)\n') + row = remote_observe.parse_log(line.encode())[0] + self.assertTrue(row['is_prompt']) + self.assertEqual(row['current_tps'], 12.5) + self.assertEqual(row['current_sps'], 10) + self.assertIsNone(row['phase_share']) + + def test_same_second_source_mtime_does_not_prove_runtime_identity(self): + count = len(remote_observe.SOURCES) + self.assertFalse(remote_observe.source_predates(['100'] * count, 100.9)) + self.assertTrue(remote_observe.source_predates(['99'] * count, 100.1)) + self.assertFalse(remote_observe.source_predates(['99'] * (count - 1), 100.1)) + + def test_docker_observation_does_not_activate_socket(self): + active = mock.Mock(returncode=0, stdout=b'active\n') + inactive = mock.Mock(returncode=3, stdout=b'inactive\n') + with mock.patch.object(remote_observe.subprocess, 'run', side_effect=[active, inactive]): + self.assertTrue(remote_observe.docker_ready()) + with mock.patch.object(remote_observe.subprocess, 'run', side_effect=[inactive, active]): + self.assertFalse(remote_observe.docker_ready()) + with mock.patch.object(remote_observe, 'docker_ready', return_value=False), \ + mock.patch.object(remote_observe.subprocess, 'Popen') as popen: + with self.assertRaises(ValueError): + remote_observe.bounded_command(['docker', 'inspect', 'synthetic'], 1, 100) + popen.assert_not_called() + + def test_complete_matching_step_and_device_samples(self): + now = time.time() + result = telemetry.summarize_node(sample(now), node(), now - 3, now + 1) + self.assertTrue(result['source_ok']) + self.assertEqual(result['prefill']['tokens_per_second'], 210) + self.assertAlmostEqual(result['prefill']['mfu_estimate'], 1.05) + self.assertEqual(result['hbm']['mean_percent'], 25) + self.assertAlmostEqual(result['phase_timers']['prefill']['mean_ms'], 80 / 3) + self.assertIsNone(result['decode']) + + def test_missing_duplicate_mixed_stale_and_skewed_steps_are_unknown(self): + now = time.time() + variants = [] + missing = sample(now); missing['engine_rows'] = missing['engine_rows'][1:]; variants.append(missing) + duplicate = sample(now); duplicate['engine_rows'].append(deepcopy(duplicate['engine_rows'][0])); variants.append(duplicate) + mixed = sample(now); mixed['engine_rows'][1]['is_prompt'] = False; variants.append(mixed) + stale = sample(now); stale['engine_rows'][0]['at'] = now - 70; variants.append(stale) + spread = sample(now); spread['engine_rows'][1]['current_sps'] = 20; variants.append(spread) + rounded = sample(now); rounded['engine_rows'][0]['current_sps'] = .001; rounded['engine_rows'][1]['current_sps'] = .001; variants.append(rounded) + wrong_device = sample(now); wrong_device['engine_rows'][1]['device'] = 0; variants.append(wrong_device) + for raw in variants: + with self.subTest(raw=raw): + value = telemetry.summarize_node(raw, node(), now - 3, now + 1) + self.assertIsNone(value['prefill']) + self.assertIsNotNone(value['hbm']) + + def test_unknown_source_invalid_bandwidth_and_clock_are_not_zero(self): + now = time.time() + raw = sample(now); raw['source_sha256']['stats'] = 'b' * 64 + self.assertFalse(telemetry.summarize_node(raw, node(), now - 3, now + 1)['source_ok']) + raw = sample(now); raw['source_before_start'] = False + self.assertFalse(telemetry.summarize_node(raw, node(), now - 3, now + 1)['source_ok']) + raw = sample(now); raw['image_fingerprint'] = 'e' * 64 + self.assertFalse(telemetry.summarize_node(raw, node(), now - 3, now + 1)['source_ok']) + raw = sample(now); raw['driver_sha256'] = '8' * 64 + value = telemetry.summarize_node(raw, node(), now - 3, now + 1) + self.assertTrue(value['source_ok']) + self.assertFalse(value['driver_ok']) + self.assertIsNone(value['hbm']) + self.assertIsNotNone(value['prefill']) + raw = sample(now); raw['hbm'][1]['hbm_bandwidth_percent'] = None + value = telemetry.summarize_node(raw, node(), now - 3, now + 1) + self.assertIsNone(value['hbm']) + self.assertIsNotNone(value['prefill']) + raw = sample(now); raw['finished_at'] = now + 20 + self.assertFalse(telemetry.summarize_node(raw, node(), now - 3, now + 1)['clock_ok']) + + def test_host_clock_offset_is_bounded_before_cross_node_combination(self): + now = time.time() + raw = sample(now) + for key in ('source_started_at', 'started_at', 'finished_at'): + raw[key] += 9 + for item in raw['engine_rows']: + item['at'] += 9 + for item in raw['hbm']: + item['started_at'] += 9 + item['finished_at'] += 9 + corrected = telemetry.summarize_node(raw, node(), now - 3, now + 1) + self.assertTrue(corrected['clock_ok']) + self.assertAlmostEqual(corrected['prefill']['log_at'], now - .99) + self.assertAlmostEqual(corrected['clock_uncertainty_seconds'], 1) + self.assertIsNotNone(corrected['hbm']) + self.assertEqual(telemetry.summarize([corrected], now)['prefill_tps'], 210) + + def test_distinct_device_groups_are_not_combined(self): + now = time.time() + first = telemetry.summarize_node(sample(now), node(), now - 3, now + 1) + other_node = node('two', 19002); other_node['peak_flops_per_device'] = 2000.0 + raw = sample(now); raw['host_fingerprint'] = 'e' * 64; raw['source_id'] = 'f' * 64 + second = telemetry.summarize_node(raw, other_node, now - 3, now + 1) + total = telemetry.summarize([first, second], now) + self.assertIsNone(total['prefill_tps']) + self.assertIsNone(total['prefill_mfu']) + self.assertIsNone(total['hbm_percent']) + self.assertTrue(all(n['source_ok'] for n in total['nodes'])) + + def test_duplicate_lifecycle_or_host_invalidates_both_groups(self): + now = time.time() + first = telemetry.summarize_node(sample(now), node(), now - 3, now + 1) + second = telemetry.summarize_node(sample(now), node('two', 19002), now - 3, now + 1) + values = telemetry.summarize([first, second], now)['nodes'] + self.assertTrue(all(n['duplicate_source'] and not n['source_ok'] and n['hbm'] is None for n in values)) + + def test_public_snapshot_omits_internal_identity_fingerprints(self): + now = time.time() + value = telemetry.summarize([telemetry.summarize_node(sample(now), node(), now - 3, now + 1)], now) + public = telemetry.public_snapshot(value) + self.assertNotIn('source_id', public['nodes'][0]) + self.assertNotIn('host_fingerprint', public['nodes'][0]) + self.assertIn('source_id', value['nodes'][0]) + + def test_combination_checks_full_rank_and_device_intervals(self): + now = time.time() + first = telemetry.summarize_node(sample(now), node(), now - 3, now + 1) + second = telemetry.summarize_node(sample(now), node('two', 19002), now - 3, now + 1) + first['clock_uncertainty_seconds'] = second['clock_uncertainty_seconds'] = 0 + first['prefill']['log_at'], first['prefill']['log_skew_seconds'] = now - 5, 5 + second['prefill']['log_at'], second['prefill']['log_skew_seconds'] = now, 5 + first['hbm']['sample_at'], first['hbm']['span_seconds'] = now - 5, 5 + second['hbm']['sample_at'], second['hbm']['span_seconds'] = now, 4.8 + result = telemetry.summarize([first, second], now) + self.assertIsNone(result['prefill_tps']) + self.assertIsNone(result['hbm_percent']) + + def test_snapshot_expires_by_oldest_rank_not_latest(self): + now = time.time() + collector = telemetry.Collector({'interval_seconds': 30, 'nodes': [node()]}) + item = telemetry.summarize_node(sample(now), node(), now - 3, now + 1) + collector.value = telemetry.summarize([item], now) + with mock.patch.object(telemetry.time, 'time', return_value=now + 59): + self.assertIsNone(collector.snapshot()['nodes'][0]['prefill']) + + def test_device_mean_rejects_long_or_slow_sampling_spans(self): + now = time.time() + raw = sample(now) + raw['started_at'] = now - 19 + raw['hbm'][0]['started_at'] = now - 18 + raw['hbm'][0]['finished_at'] = now - 17 + value = telemetry.summarize_node(raw, node(), now - 20, now + 1) + self.assertTrue(value['source_ok']) + self.assertIsNone(value['hbm']) + + def test_old_lifecycle_cpu_timer_cannot_attach_to_new_step(self): + now = time.time() + raw = sample(now) + raw['source_started_at'] = now - 5 + for row in raw['engine_rows']: + if row['kind'] == 'cpu': + row['at'] = now - 60 + result = telemetry.summarize_node(raw, node(), now - 3, now + 1) + self.assertIsNotNone(result['prefill']) + self.assertIsNone(result['phase_timers']['prefill']['mean_ms']) + + def test_newer_log_group_wins_even_if_worker_step_counter_restarts(self): + now = time.time() + raw = sample(now) + old = deepcopy(raw['engine_rows'][:2]) + for row in old: + row['step'] = 99999 + row['at'] -= 20 + raw['engine_rows'].extend(old) + result = telemetry.summarize_node(raw, node(), now - 3, now + 1) + self.assertEqual(result['prefill']['step'], 10) + + def test_config_rejects_command_injection_and_duplicate_devices(self): + config = {'interval_seconds': 30, 'nodes': [node()]} + telemetry.validate_config(config) + config['nodes'][0]['container'] = 'valid;evil' + with self.assertRaises(ValueError): + telemetry.validate_config(config) + config = {'interval_seconds': 30, 'nodes': [node()]} + config['nodes'][0]['rank_devices'] = [0, 0] + with self.assertRaises(ValueError): + telemetry.validate_config(config) + config = {'interval_seconds': 30, 'nodes': [node()]} + config['nodes'][0]['prefill_flops_per_token'] = 10 ** 400 + with self.assertRaises(ValueError): + telemetry.validate_config(config) + config = {'interval_seconds': 30, 'nodes': [node(), node('two', 19002)]} + with self.assertRaises(ValueError): + telemetry.validate_config(config) + + def test_loopback_server_rejects_untrusted_origin(self): + collector = telemetry.Collector({'interval_seconds': 30, 'nodes': [node()]}) + server = telemetry.Server(0, collector) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 5) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + connection = HTTPConnection('127.0.0.1', server.server_port, timeout=3) + connection.request('GET', '/snapshot', headers={'Origin': 'https://example.invalid'}) + self.assertEqual(connection.getresponse().status, 403) + connection.close() + + def test_dashboard_exposes_only_configured_telemetry(self): + server = web.Dashboard(0, 'unused.db', telemetry_port=19883) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join, 5) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + with mock.patch.object(web, 'read_telemetry', return_value={'available': True, 'nodes': []}) as reader: + connection = HTTPConnection('127.0.0.1', server.server_port, timeout=3) + connection.request('GET', '/api/telemetry') + response = connection.getresponse() + self.assertEqual(response.status, 200) + self.assertIn(b'"available":true', response.read()) + reader.assert_called_once_with(19883) + connection.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_telemetry_history.py b/tests/test_telemetry_history.py new file mode 100644 index 0000000..2adb16e --- /dev/null +++ b/tests/test_telemetry_history.py @@ -0,0 +1,221 @@ +import json +from contextlib import closing +from http.client import HTTPConnection +from pathlib import Path +import sqlite3 +import tempfile +import threading +import time +import unittest +from unittest import mock + +from deploy_stats import telemetry, web +from deploy_stats.telemetry_history import BUCKET_SECONDS, History, SOURCE +from tests.test_telemetry import node as engine_config, sample as engine_sample + + +def config(port=19001, target='operator@example.invalid'): + return {'backend_port': port, 'ssh_target': target, 'container': 'example-container', + 'rank_devices': [0, 1], 'source_sha256': {'engine': 'a' * 64}, + 'image_fingerprint': 'b' * 64, 'driver_sha256': 'c' * 64} + + +def node(at, port=19001, host='d' * 64, source='e' * 64, step=1): + return {'source_ok': True, 'backend_ready': True, 'clock_ok': True, 'backend_port': port, + 'host_fingerprint': host, 'source_id': source, 'clock_uncertainty_seconds': .1, + 'prefill': {'log_at': at, 'log_skew_seconds': .2, 'step': step, + 'tokens_per_second': 100, 'mfu_estimate': .5, 'valid_until': at + 40, + '_observation_id': f'{step:064x}'}, + 'decode': {'log_at': at, 'log_skew_seconds': .2, 'step': step + 1, + 'tokens_per_second': 0, 'valid_until': at + 40, '_observation_id': f'{step+1:064x}'}} + + +class TelemetryHistoryTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(dir=Path.cwd() / 'local') + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) / 'history.sqlite3' + self.history = History(self.path) + self.at = 100000 * BUCKET_SECONDS + 100 + + def test_samples_are_per_group_deduplicated_and_keep_true_zero(self): + item = node(self.at) + self.assertEqual(self.history.record(item, config(), self.at + 1), 2) + item['prefill']['log_at'] += .05 # changed clock estimate, same observed step + self.assertEqual(self.history.record(item, config(), self.at + 1), 0) + result = self.history.series(config(), item['host_fingerprint'], self.at + BUCKET_SECONDS) + self.assertEqual(result['method'], SOURCE) + self.assertEqual(len(result['points']), 144) + point = result['points'][-1] + self.assertEqual(point['prefill_tps'], 100) + self.assertEqual(point['prefill_samples'], 1) + self.assertEqual(point['decode_tps'], 0) + self.assertEqual(point['decode_samples'], 1) + self.assertIsNone(result['points'][-2]['prefill_tps']) + + def test_bucket_boundary_expiry_and_unknown_source_are_not_recorded(self): + boundary = 100000 * BUCKET_SECONDS + BUCKET_SECONDS - .05 + self.assertEqual(self.history.record(node(boundary), config(), boundary + 1), 0) + item = node(self.at) + item['prefill']['valid_until'] = self.at + item['decode'] = None + self.assertEqual(self.history.record(item, config(), self.at + 1), 0) + item = node(self.at) + item['source_ok'] = False + self.assertEqual(self.history.record(item, config(), self.at + 1), 0) + + def test_history_is_not_shared_across_host_or_configuration(self): + item = node(self.at) + self.history.record(item, config(), self.at + 1) + for other, host in ((config(19002), item['host_fingerprint']), (config(), 'f' * 64)): + result = self.history.series(other, host, self.at + BUCKET_SECONDS) + self.assertIsNone(result['points'][-1]['prefill_tps']) + item2 = node(self.at, source='1' * 64) + self.history.record(item2, config(), self.at + 1) + result = self.history.series(config(), item['host_fingerprint'], self.at + BUCKET_SECONDS) + self.assertEqual(result['points'][-1]['prefill_samples'], 2) + + def test_worker_restart_can_reuse_step_without_losing_new_observation(self): + first = node(self.at, step=7) + self.assertEqual(self.history.record(first, config(), self.at + 1), 2) + second = node(self.at + BUCKET_SECONDS, step=7) + second['prefill']['_observation_id'] = '9' * 64 + second['decode']['_observation_id'] = 'a' * 64 + self.assertEqual(self.history.record(second, config(), self.at + BUCKET_SECONDS + 1), 2) + result = self.history.series(config(), first['host_fingerprint'], self.at + 2 * BUCKET_SECONDS) + self.assertEqual(result['points'][-1]['prefill_samples'], 1) + self.assertEqual(result['points'][-2]['prefill_samples'], 1) + + def test_observation_identity_uses_original_rank_logs_not_clock_estimate(self): + raw = engine_sample(self.at) + first = telemetry.summarize_node(raw, engine_config(), self.at - 3, self.at + 1) + same = telemetry.summarize_node(raw, engine_config(), self.at - 2.9, self.at + 1) + self.assertNotEqual(first['prefill']['log_at'], same['prefill']['log_at']) + self.assertEqual(first['prefill']['_observation_id'], same['prefill']['_observation_id']) + later = telemetry.summarize_node(engine_sample(self.at + 10), engine_config(), + self.at + 7, self.at + 11) + self.assertEqual(first['prefill']['step'], later['prefill']['step']) + self.assertNotEqual(first['prefill']['_observation_id'], later['prefill']['_observation_id']) + + def test_unidentified_existing_file_is_not_modified(self): + other = Path(self.tmp.name) / 'not-history.sqlite3' + with closing(sqlite3.connect(other)) as db, db: + db.execute('CREATE TABLE unrelated (value TEXT)') + db.execute('INSERT INTO unrelated VALUES (?)', ('keep',)) + before = other.read_bytes() + with self.assertRaisesRegex(ValueError, 'INVALID_TELEMETRY_HISTORY'): + History(other) + self.assertEqual(other.read_bytes(), before) + + def test_reopen_keeps_previous_samples_and_output_has_no_source_identity(self): + item = node(self.at) + self.history.record(item, config(), self.at + 1) + result = History(self.path).series(config(), item['host_fingerprint'], self.at + BUCKET_SECONDS) + self.assertEqual(result['points'][-1]['prefill_tps'], 100) + serialized = json.dumps(result) + self.assertNotIn(item['host_fingerprint'], serialized) + self.assertNotIn(item['source_id'], serialized) + + def test_collector_records_and_exposes_only_sanitized_history(self): + conf = engine_config() + first = node(self.at) + collector = telemetry.Collector({'interval_seconds': 30, 'nodes': [conf]}, + sample=lambda *_: first, history=self.history) + self.addCleanup(collector.close) + with mock.patch.object(telemetry.time, 'time', return_value=self.at + 1): + collector.collect() + self.assertTrue(collector.history_worker.changed.wait(3)) + collector.history_worker.changed.clear() + next_now = (self.at // BUCKET_SECONDS + 1) * BUCKET_SECONDS + 10 + second = node(next_now - 1, step=5) + collector.sample = lambda *_: second + with mock.patch.object(telemetry.time, 'time', return_value=next_now): + collector.collect() + self.assertTrue(collector.history_worker.changed.wait(3)) + result = telemetry.public_snapshot(collector.snapshot()) + self.assertTrue(result['history_available']) + point = result['nodes'][0]['history']['points'][-1] + self.assertEqual(point['prefill_tps'], 100) + self.assertNotIn('host_fingerprint', result['nodes'][0]) + self.assertNotIn('source_id', result['nodes'][0]) + self.assertNotIn('_observation_id', result['nodes'][0]['prefill']) + + def test_history_failure_does_not_clear_live_observations(self): + conf = engine_config() + item = node(self.at) + broken = mock.Mock() + broken.record.side_effect = sqlite3.OperationalError('synthetic') + collector = telemetry.Collector({'interval_seconds': 30, 'nodes': [conf]}, + sample=lambda *_: item, history=broken) + self.addCleanup(collector.close) + with mock.patch.object(telemetry.time, 'time', return_value=self.at + 1): + collector.collect() + self.assertTrue(collector.history_worker.changed.wait(3)) + result = collector.snapshot() + self.assertFalse(result['history_available']) + self.assertEqual(result['nodes'][0]['prefill']['tokens_per_second'], 100) + + def test_history_flows_through_the_loopback_dashboard(self): + now = 100001 * BUCKET_SECONDS + 1 + conf = engine_config() + item = node(now - 3) + collector = telemetry.Collector({'interval_seconds': 30, 'nodes': [conf]}, + sample=lambda *_: item, history=self.history) + self.addCleanup(collector.close) + sidecar = telemetry.Server(0, collector) + dashboard = web.Dashboard(0, Path(self.tmp.name)/'unused.sqlite3', telemetry_port=sidecar.server_port) + threads = [threading.Thread(target=server.serve_forever, daemon=True) for server in (sidecar, dashboard)] + for thread in threads: + thread.start() + try: + with mock.patch.object(telemetry.time, 'time', return_value=now): + collector.collect() + self.assertTrue(collector.history_worker.changed.wait(3)) + connection = HTTPConnection('127.0.0.1', dashboard.server_port, timeout=3) + connection.request('GET', '/api/telemetry') + response = connection.getresponse() + self.assertEqual(response.status, 200) + result = json.loads(response.read()) + connection.close() + self.assertTrue(result['available']) + self.assertTrue(result['history_available']) + self.assertEqual(result['nodes'][0]['history']['points'][-1]['prefill_tps'], 100) + self.assertEqual(result['nodes'][0]['history']['points'][-1]['decode_tps'], 0) + self.assertNotIn('host_fingerprint', result['nodes'][0]) + self.assertNotIn('source_id', result['nodes'][0]) + finally: + for server in (dashboard, sidecar): + server.shutdown() + server.server_close() + for thread in threads: + thread.join(3) + + def test_slow_history_io_does_not_block_live_sampling_or_snapshot(self): + gate = threading.Event() + started = threading.Event() + slow = mock.Mock() + def slow_record(*_): + started.set() + gate.wait(3) + slow.record.side_effect = slow_record + slow.series.return_value = None + collector = telemetry.Collector({'interval_seconds': 30, 'nodes': [engine_config()]}, + sample=lambda *_: node(self.at), history=slow) + self.addCleanup(collector.close) + self.addCleanup(gate.set) + with mock.patch.object(telemetry.time, 'time', return_value=self.at + 1): + start = time.monotonic() + collector.collect() + self.assertTrue(started.wait(1)) + result = collector.snapshot() + self.assertLess(time.monotonic() - start, 1.5) + self.assertFalse(result['history_available']) + self.assertEqual(result['nodes'][0]['prefill']['tokens_per_second'], 100) + + def test_history_initialization_failure_does_not_raise(self): + missing = Path(self.tmp.name) / 'absent' / 'history.sqlite3' + self.assertIsNone(telemetry.open_history(missing)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_trend_tooltip.py b/tests/test_trend_tooltip.py new file mode 100644 index 0000000..488ab00 --- /dev/null +++ b/tests/test_trend_tooltip.py @@ -0,0 +1,106 @@ +import shutil +import subprocess +import unittest +from pathlib import Path + + +class TrendTooltipTests(unittest.TestCase): + def test_hover_reports_exact_bucket_value_and_preserves_gaps(self): + node = shutil.which('node') + if node is None: + self.skipTest('Node.js is required to validate dashboard interactions') + script = r''' +const assert=require('node:assert/strict'),fs=require('node:fs'),vm=require('node:vm'); +const source=fs.readFileSync(process.argv[1],'utf8'); +function element(tag,attrs={}) {return {tag,attrs,children:[],textContent:'', + replaceChildren(){this.children=[];},appendChild(x){this.children.push(x);}, + classList:{add(){},remove(){}}, + setAttribute(k,v){this.attrs[k]=v;},removeAttribute(k){delete this.attrs[k];}, + getBoundingClientRect(){return {left:100,top:100,width:800,height:68};}};} +const tooltip={hidden:true,textContent:'',style:{},getBoundingClientRect:()=>({width:180,height:60})}; +const elements={'trend-tooltip':tooltip}; +for(const id of ['input','output','cache-rate','device-prefill','device-decode']){ + elements[`chart-${id}`]=element('label');elements[`chart-${id}-svg`]=element('svg'); +} +const ctx={$:(id)=>elements[id]||null,svgElement:element,setTextIfPresent:()=>{}, + document:{documentElement:{clientWidth:1200,clientHeight:800}},window:{innerWidth:1200,innerHeight:800}, + number:{format:String},integer:{format:String},count:String,speed:(v)=>`${v} tok/s`,percent:(v)=>`${v*100}%`, + clock:{format:(v)=>`time${v.getTime()/1000}`},dateTime:{format:(v)=>`date${v.getTime()/1000}`}}; +function extract(name){const a=source.indexOf(`function ${name}(`),b=source.indexOf('\n}\n',a); + assert(a>=0&&b>a,`missing ${name}`);return vm.runInNewContext('('+source.slice(a,b+2)+')',ctx);} +ctx.attachTrendTooltip=extract('attachTrendTooltip'); +const render=extract('renderPlot'),renderDevice=extract('renderDeviceHistory'); +const points=[{start:0,end:600,input:120,known:true,complete:true}, + {start:600,end:1200,input:43,known:true,complete:false}, + {start:1200,end:1800,input:null,known:false,complete:false}]; +render({id:'input',field:'input',kind:'tokens',label:'输入'},points); +let svg=elements['chart-input-svg']; +svg.onpointermove({clientX:100,clientY:130}); +assert.equal(tooltip.hidden,false);assert.match(tooltip.textContent,/输入.*120 Token/); +assert.match(tooltip.textContent,/date0.*date600/); +points[0].input=121;render({id:'input',field:'input',kind:'tokens',label:'输入'},points); +assert.equal(tooltip.hidden,false);assert.match(tooltip.textContent,/121 Token/); // Refresh while hovered. +svg.onpointermove({clientX:500,clientY:130});assert.match(tooltip.textContent,/至少 43 Token/); +svg.onpointermove({clientX:900,clientY:130});assert.equal(tooltip.hidden,true); // Gap is not zero. +svg.onpointermove({clientX:100,clientY:130});svg.onpointerleave();assert.equal(tooltip.hidden,true); +svg.onfocus();assert.equal(tooltip.hidden,false);assert.match(tooltip.textContent,/121 Token/); +svg.onkeydown({key:'ArrowRight',preventDefault(){}});assert.match(tooltip.textContent,/43 Token/); +points[1].input=null;points[1].known=false; +render({id:'input',field:'input',kind:'tokens',label:'输入'},points); +assert.equal(tooltip.hidden,true); // Focused bucket disappeared during refresh. +render({id:'input',field:'input',kind:'tokens',label:'输入'},points); // Remains missing through another refresh. +svg.onkeydown({key:'ArrowLeft',preventDefault(){}});assert.match(tooltip.textContent,/121 Token/); +svg.onblur();assert.equal(tooltip.hidden,true); +const output=[{start:0,end:600,output:456,known:true,complete:true}]; +render({id:'output',field:'output',kind:'tokens',label:'输出'},output); +svg=elements['chart-output-svg'];svg.onpointermove({clientX:100,clientY:130}); +assert.match(tooltip.textContent,/输出.*456 Token/); +const hit=[{start:0,end:600,known:true,complete:true,mean_request_hit_rate:.25,request_hit_sum:.5,request_hit_samples:2}]; +render({id:'cache-rate',field:'mean_request_hit_rate',kind:'hit',label:'平均请求命中率',samples:'request_hit_samples'},hit); +svg=elements['chart-cache-rate-svg'];svg.onpointermove({clientX:100,clientY:130}); +assert.match(tooltip.textContent,/平均请求命中率.*25%.*2 个样本/); +const device=Array.from({length:144},(_,i)=>({start:i*600,end:(i+1)*600, + prefill_tps:null,prefill_samples:0,prefill_groups:0,decode_tps:null,decode_samples:0,decode_groups:0})); +device[20].decode_tps=200;device[20].decode_samples=1;device[20].decode_groups=1; +device[20].prefill_tps=1200;device[20].prefill_samples=2;device[20].prefill_groups=2; +renderDevice({available:true,history_available:true},{history:{method:'deepinfer-log-step-approx-v1', + bucket_seconds:600,aggregate:true,group_count:2,points:device}}); +svg=elements['chart-device-prefill-svg'];svg.onpointermove({clientX:100+20/143*800,clientY:130}); +assert.match(tooltip.textContent,/Prefill.*1200 tok\/s[\s\S]*2\/2 组/); +elements['chart-input-svg'].onfocus(); // Keyboard focus may remain while another chart is hovered. +svg=elements['chart-device-decode-svg'];svg.onpointermove({clientX:100+20/143*800,clientY:130}); +assert.match(tooltip.textContent,/Decode.*200 tok\/s[\s\S]*1\/2 组.*部分数据/); +render({id:'input',field:'input',kind:'tokens',label:'输入'},points); +assert.match(tooltip.textContent,/Decode.*200 tok\/s/); // Unrelated refresh cannot steal the tooltip. +svg.onpointermove({clientX:100+21/143*800,clientY:130});assert.equal(tooltip.hidden,true); +renderDevice({available:true,history_available:false},null); +assert.equal(tooltip.hidden,true);assert.equal(svg.onpointermove,null); +// A failed statistics refresh must clear the previous tooltip and handlers. +render({id:'input',field:'input',kind:'tokens',label:'输入'},points); +svg=elements['chart-input-svg'];svg.onpointermove({clientX:100,clientY:130});assert.equal(tooltip.hidden,false); +ctx.$=(id)=>elements[id]||(elements[id]=element('div')); +Object.assign(ctx,{revision:0,pendingRefresh:null,activeExpiry:null,backendExpiry:null,lastSnapshot:null,selected:'24h', + tables:{requests:{offset:0,size:20,sort:'finished',direction:'desc'}, + sessions:{offset:0,size:20,sort:'last_finished',direction:'desc'}}, + charts:[{id:'input'},{id:'output'},{id:'cache-rate'}],empty:()=>{},updatePager:()=>{}, + AbortController,URLSearchParams,setTimeout:()=>0,clearTimeout:()=>{},fetch:async()=>{throw Error('synthetic failure');}}); +const start=source.indexOf('async function refresh('),end=source.indexOf('\n}\n',start); +assert(start>=0&&end>start); +const refresh=vm.runInNewContext('('+source.slice(start,end+2)+')',ctx); +refresh().then(()=>{assert.equal(tooltip.hidden,true);assert.equal(svg.onpointermove,null); + assert.equal(elements['chart-input'].textContent,'暂无数据'); + renderDevice({available:true,history_available:true},{history:{method:'deepinfer-log-step-approx-v1', + bucket_seconds:600,aggregate:true,group_count:2,points:device}}); + svg=elements['chart-device-decode-svg'];svg.onpointermove({clientX:100+20/143*800,clientY:130}); + return refresh(); +}).then(()=>{assert.equal(tooltip.hidden,false);assert.match(tooltip.textContent,/Decode.*200 tok\/s/); +}).catch((error)=>{console.error(error);process.exitCode=1;}); +''' + source = Path(__file__).resolve().parents[1] / 'deploy_stats/static/app.js' + result = subprocess.run([node, '-e', script, str(source)], capture_output=True, text=True, + encoding='utf-8', timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_trends.py b/tests/test_trends.py new file mode 100644 index 0000000..bb8d0f3 --- /dev/null +++ b/tests/test_trends.py @@ -0,0 +1,66 @@ +from datetime import datetime, timedelta, timezone +from pathlib import Path +import tempfile +import time +import unittest + +from deploy_stats.report import report +from deploy_stats.store import _open + + +class TrendTests(unittest.TestCase): + def test_24h_rates_are_computed_from_eligible_requests_and_include_partial_bucket(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'series.sqlite3' + now = datetime(2026, 1, 3, 12, 2, 3, tzinfo=timezone(timedelta(hours=8))).timestamp() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('fixture', now - 2400, now, now, 0)) + statement = '''INSERT INTO requests(run_id,finished,client_ip,elapsed,http_status,state, + input_tokens,output_tokens,cached_tokens,session_id,first_visible) VALUES(?,?,?,?,?,?,?,?,?,?,?)''' + for elapsed, prompt, output, cached, first in ((3, 100, 11, 90, 1), + (5, 10, 3, 0, 2), + (1, 0, 1, 0, 0), + (2, 20, 2, 20, None)): + db.execute(statement, ('fixture', now - 900, '192.0.2.4', elapsed, 200, 'known', + prompt, output, cached, None, first)) + db.execute(statement, ('fixture', now - 900, '192.0.2.4', 1, 500, 'failed', + None, None, None, None, None)) + db.execute(statement, ('fixture', now - 100, '192.0.2.4', 3, 200, 'known', 20, 5, 0, None, 1)) + db.commit() + db.close() + points = report(path, 'today', now=now)['series_24h'] + bucket_start = int((now - 900) // 600) * 600 + bucket = next(point for point in points if point['start'] == bucket_start) + self.assertAlmostEqual(bucket['mean_request_hit_rate'], 1.9 / 3) + self.assertEqual(bucket['request_hit_samples'], 3) + self.assertAlmostEqual(bucket['prefill_estimate_tps'], 20 / 3) + self.assertEqual((bucket['prefill_samples'], bucket['prefill_tokens'], bucket['prefill_seconds']), (2, 20, 3)) + self.assertAlmostEqual(bucket['decode_estimate_tps'], 12 / 5) + self.assertEqual((bucket['decode_samples'], bucket['decode_tokens'], bucket['decode_seconds']), (2, 12, 5)) + empty = next(point for point in points if point['start'] == bucket_start - 600) + self.assertTrue(empty['complete']) + self.assertIsNone(empty['mean_request_hit_rate']) + self.assertIsNone(empty['prefill_estimate_tps']) + self.assertIsNone(empty['decode_estimate_tps']) + partial = next(point for point in points if point['start'] <= now - 100 < point['end']) + self.assertLess(partial['end'] - partial['start'], 600) + self.assertAlmostEqual(sum(point['request_hit_sum'] for point in points) / + sum(point['request_hit_samples'] for point in points), 1.9 / 4) + + def test_historical_overload_detail_is_not_treated_as_zero(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'overload.sqlite3' + now = time.time() + db = _open(path) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('old', now - 20, now - 10, now - 10, 1)) + db.execute('INSERT INTO runs VALUES (?,?,?,?,?)', ('new', now - 10, now, now, 3)) + db.execute('INSERT INTO stats_meta VALUES (?,?)', ('overload:new', '3')) + db.commit() + db.close() + totals = report(path, 'total', now=now + .01)['totals'] + self.assertEqual((totals['dropped'], totals['overload_rejected'], + totals['overload_tracked_runs'], totals['overload_untracked_runs']), (4, 3, 1, 1)) + + +if __name__ == '__main__': + unittest.main()