From 67eb151aa88acade1fae9bc90cd7873c828ca0d9 Mon Sep 17 00:00:00 2001 From: hx_v5 Date: Tue, 10 Nov 2026 23:52:34 +0800 Subject: [PATCH] Add authenticated multi-model routing --- deploy_gateway/auth.py | 21 ++ deploy_gateway/server.py | 15 +- deploy_model_router/__init__.py | 1 + deploy_model_router/__main__.py | 3 + deploy_model_router/caddy.py | 66 ++++ deploy_model_router/credentials.py | 41 +++ deploy_model_router/server.py | 531 +++++++++++++++++++++++++++++ docs/gateway.md | 27 ++ tests/test_model_router.py | 286 ++++++++++++++++ tests/test_model_router_caddy.py | 51 +++ 10 files changed, 1029 insertions(+), 13 deletions(-) create mode 100644 deploy_gateway/auth.py create mode 100644 deploy_model_router/__init__.py create mode 100644 deploy_model_router/__main__.py create mode 100644 deploy_model_router/caddy.py create mode 100644 deploy_model_router/credentials.py create mode 100644 deploy_model_router/server.py create mode 100644 tests/test_model_router.py create mode 100644 tests/test_model_router_caddy.py diff --git a/deploy_gateway/auth.py b/deploy_gateway/auth.py new file mode 100644 index 0000000..b17d147 --- /dev/null +++ b/deploy_gateway/auth.py @@ -0,0 +1,21 @@ +"""Authentication against the existing version 1 gateway key store.""" +import hashlib +import hmac +import json +from pathlib import Path +import re + + +def authorized(key_store, value): + data = json.loads(Path(key_store).read_text()) + if data.get('version') != 1 or not isinstance(data.get('records'), dict): + raise ValueError('Unsupported key store') + digest = hashlib.sha256(value.encode()).hexdigest() + accepted = False + for record in data['records'].values(): + expected = record['sha256'] + if not re.fullmatch(r'[0-9a-f]{64}', expected) or not isinstance(record['active'], bool): + raise ValueError('Invalid key record') + matches = hmac.compare_digest(digest, expected) + accepted = accepted or (matches and record['active']) + return accepted diff --git a/deploy_gateway/server.py b/deploy_gateway/server.py index d0dffed..fca7416 100644 --- a/deploy_gateway/server.py +++ b/deploy_gateway/server.py @@ -5,7 +5,6 @@ """ import argparse import hashlib -import hmac from http.client import HTTPConnection, HTTPException, IncompleteRead from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json @@ -18,6 +17,7 @@ import time from urllib.parse import urlsplit +from .auth import authorized from .routing import Busy, SessionRouter, Unavailable, affinity_key, prefix_rank HOP_HEADERS = {'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', @@ -141,18 +141,7 @@ def server_close(self): super().server_close() def authorized(self, value): - data = json.loads(self.key_store.read_text()) - if data.get('version') != 1 or not isinstance(data.get('records'), dict): - raise ValueError('Unsupported key store') - digest = hashlib.sha256(value.encode()).hexdigest() - accepted = False - for record in data['records'].values(): - expected = record['sha256'] - if not re.fullmatch(r'[0-9a-f]{64}', expected) or not isinstance(record['active'], bool): - raise ValueError('Invalid key record') - matches = hmac.compare_digest(digest, expected) - accepted = accepted or (matches and record['active']) - return accepted + return authorized(self.key_store, value) def context_route(self, headers): values = headers.get_all('x-model-context-window', []) diff --git a/deploy_model_router/__init__.py b/deploy_model_router/__init__.py new file mode 100644 index 0000000..40f97c6 --- /dev/null +++ b/deploy_model_router/__init__.py @@ -0,0 +1 @@ +"""Model alias router for an existing authenticated inference gateway.""" diff --git a/deploy_model_router/__main__.py b/deploy_model_router/__main__.py new file mode 100644 index 0000000..bc40eef --- /dev/null +++ b/deploy_model_router/__main__.py @@ -0,0 +1,3 @@ +from .server import main + +main() diff --git a/deploy_model_router/caddy.py b/deploy_model_router/caddy.py new file mode 100644 index 0000000..45edc6b --- /dev/null +++ b/deploy_model_router/caddy.py @@ -0,0 +1,66 @@ +"""Prepare an atomic Caddy servers update without altering other routes.""" +from copy import deepcopy + + +def proxies(value): + if isinstance(value, dict): + if value.get('handler') == 'reverse_proxy': + yield value + for child in value.values(): + yield from proxies(child) + elif isinstance(value, list): + for child in value: + yield from proxies(child) + + +def prepare_servers(servers, *, front_port, fallback_port, router_port): + """Return a copy; reject a topology whose safety cannot be established.""" + if not all(type(x) is int and 1 <= x <= 65535 for x in (front_port, fallback_port, router_port)): + raise ValueError('Invalid port') + if len({front_port, fallback_port, router_port}) != 3 or not isinstance(servers, dict): + raise ValueError('Ports must be distinct') + result = deepcopy(servers) + + def find(port): + found = [s for s in result.values() if isinstance(s, dict) + and any(isinstance(x, str) and x.rsplit(':', 1)[-1] == str(port) + for x in s.get('listen', []))] + if len(found) != 1 or len(found[0].get('listen', [])) != 1: + raise ValueError('Expected one dedicated server per port') + return found[0] + + front, fallback = find(front_port), find(fallback_port) + if front is fallback or fallback.get('trusted_proxies'): + raise ValueError('Unsupported trusted-proxy topology') + front_proxies = list(proxies(front.get('routes', []))) + fallback_proxies = list(proxies(fallback.get('routes', []))) + if not front_proxies or len(front_proxies) != len(fallback_proxies): + raise ValueError('Unexpected proxy routes') + old = [p.get('upstreams') for p in fallback_proxies] + if any(not isinstance(x, list) or not x for x in old): + raise ValueError('No fallback upstreams') + if [p.get('upstreams') for p in front_proxies] != old: + raise ValueError('Frontend and fallback routes differ') + forbidden = {f'127.0.0.1:{p}' for p in (front_port, fallback_port, router_port)} + for items in old: + if any(not isinstance(x, dict) or not isinstance(x.get('dial'), str) + or x['dial'] in forbidden for x in items): + raise ValueError('Unexpected fallback upstream') + for proxy in front_proxies: + proxy['upstreams'] = [{'dial': f'127.0.0.1:{router_port}'}, + {'dial': f'127.0.0.1:{fallback_port}'}] + proxy['load_balancing'] = {'selection_policy': {'policy': 'first'}, + 'try_duration': 5_000_000_000, 'try_interval': 200_000_000} + # A model-specific HTTP error must not take down the shared frontend. + proxy['health_checks'] = {'passive': {'fail_duration': 5_000_000_000, 'max_fails': 1}} + response_set = proxy.get('headers', {}).get('response', {}).get('set', {}) + for name in tuple(response_set): + if name.lower() == 'x-llmdeploy-service': + del response_set[name] + fallback['trusted_proxies'] = {'source': 'static', 'ranges': ['127.0.0.1/32', '::1/128']} + for proxy in fallback_proxies: + request_set = proxy.setdefault('headers', {}).setdefault('request', {}).setdefault('set', {}) + if any(k.lower() == 'x-forwarded-for' for k in request_set): + raise ValueError('Existing client-IP override needs review') + request_set['X-Forwarded-For'] = ['{http.vars.client_ip}'] + return result diff --git a/deploy_model_router/credentials.py b/deploy_model_router/credentials.py new file mode 100644 index 0000000..70ea6a6 --- /dev/null +++ b/deploy_model_router/credentials.py @@ -0,0 +1,41 @@ +"""Read an upstream credential without placing it in arguments or files.""" +import ctypes +import os +from ctypes import wintypes + + +class _FileTime(ctypes.Structure): + _fields_ = [('low', wintypes.DWORD), ('high', wintypes.DWORD)] + + +class _Credential(ctypes.Structure): + _fields_ = [ + ('flags', wintypes.DWORD), ('type', wintypes.DWORD), + ('target', wintypes.LPWSTR), ('comment', wintypes.LPWSTR), + ('last_written', _FileTime), ('blob_size', wintypes.DWORD), + ('blob', ctypes.POINTER(ctypes.c_ubyte)), ('persist', wintypes.DWORD), + ('attribute_count', wintypes.DWORD), ('attributes', ctypes.c_void_p), + ('target_alias', wintypes.LPWSTR), ('username', wintypes.LPWSTR), + ] + + +def read_windows_credential(target): + if os.name != 'nt': + raise OSError('Windows Credential Manager is unavailable') + api = ctypes.WinDLL('advapi32', use_last_error=True) + api.CredReadW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, + ctypes.POINTER(ctypes.POINTER(_Credential))] + api.CredReadW.restype = wintypes.BOOL + api.CredFree.argtypes = [ctypes.c_void_p] + ptr = ctypes.POINTER(_Credential)() + if not api.CredReadW(target, 1, 0, ctypes.byref(ptr)): + raise OSError('Upstream credential is unavailable') + try: + blob = ctypes.string_at(ptr.contents.blob, ptr.contents.blob_size) + encoding = 'utf-16-le' if len(blob) > 1 and blob[1] == 0 else 'utf-8' + value = blob.decode(encoding).rstrip('\x00') + if not value or '\r' in value or '\n' in value: + raise ValueError('Invalid upstream credential') + return value + finally: + api.CredFree(ptr) diff --git a/deploy_model_router/server.py b/deploy_model_router/server.py new file mode 100644 index 0000000..b2c55eb --- /dev/null +++ b/deploy_model_router/server.py @@ -0,0 +1,531 @@ +"""Route OpenAI-compatible model aliases alongside an existing gateway. + +Client and upstream credentials, request bodies and response bodies are never +logged. The default route remains responsible for its own model scheduling. +""" +import argparse +from collections import OrderedDict +import hashlib +from http.client import HTTPConnection, HTTPSConnection, HTTPException +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import re +import select +import socket +import threading +import time +from urllib.parse import quote, urlsplit + +from deploy_gateway.auth import authorized +from deploy_gateway.routing import affinity_key +from deploy_gateway.server import HOP_HEADERS +from .credentials import read_windows_credential + +MAX_BODY = 64 * 1024 * 1024 +MAX_MODELS = 1024 * 1024 + + +def origin(value, *, local=False): + url = urlsplit(value) + if (url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password + or url.path not in ('', '/') or url.query or url.fragment): + raise ValueError('An HTTP origin without a path or credentials is required') + if local and (url.scheme != 'http' or url.hostname not in ('127.0.0.1', '::1')): + raise ValueError('The existing gateway must use a loopback HTTP origin') + return url.scheme, url.hostname, url.port or (443 if url.scheme == 'https' else 80) + + +def connection(target, timeout): + scheme, host, port = target + return (HTTPSConnection if scheme == 'https' else HTTPConnection)(host, port, timeout=timeout) + + +def alias_response(raw, backend_model, alias): + try: + value = json.loads(raw) + if isinstance(value, dict) and value.get('model') == backend_model: + value['model'] = alias + return json.dumps(value, ensure_ascii=False, separators=(',', ':')).encode() + except (ValueError, UnicodeError): + pass + return raw + + +class ModelRoute: + def __init__(self, spec): + if not isinstance(spec, dict) or set(spec) != {'alias', 'backend_model', 'max_context', 'backends'}: + raise ValueError('Invalid model route') + self.alias, self.backend_model, self.max_context = spec['alias'], spec['backend_model'], spec['max_context'] + if (not isinstance(self.alias, str) or not re.fullmatch(r'[A-Za-z0-9._-]+', self.alias) + or not isinstance(self.backend_model, str) or not self.backend_model + or type(self.max_context) is not int or self.max_context <= 0): + raise ValueError('Invalid model mapping') + if not isinstance(spec['backends'], list) or not spec['backends']: + raise ValueError('At least one model backend is required') + self.backends = [] + for item in spec['backends']: + if not isinstance(item, dict) or set(item) - {'origin', 'credential_target'} or 'origin' not in item: + raise ValueError('Invalid model backend') + target = item.get('credential_target') + if target is not None and (not isinstance(target, str) or not target): + raise ValueError('Invalid upstream credential target') + self.backends.append((origin(item['origin']), target)) + if len({entry[0] for entry in self.backends}) != len(self.backends): + raise ValueError('Duplicate model backend') + self.lock = threading.Lock() + self.inflight = [0] * len(self.backends) + self.failed_until = [0.0] * len(self.backends) + self.failure_code = [0] * len(self.backends) + self.bindings = OrderedDict() + self.cursor = 0 + + def acquire(self, key, excluded): + with self.lock: + good = [i for i in range(len(self.backends)) if i not in excluded and self.failed_until[i] <= time.monotonic()] + if not good: + return None + preferred = self.bindings.get(key) if key is not None else None + if preferred in good: + chosen = preferred + else: + chosen = min(good, key=lambda i: (self.inflight[i], (i - self.cursor) % len(self.backends))) + if key is not None: + self.bindings[key] = chosen + self.bindings.move_to_end(key) + if len(self.bindings) > 16384: + self.bindings.popitem(last=False) + self.inflight[chosen] += 1 + self.cursor = (chosen + 1) % len(self.backends) + return chosen + + def release(self, index, failure=0): + with self.lock: + self.inflight[index] -= 1 + if failure: + self.failure_code[index] = failure + self.failed_until[index] = time.monotonic() + (1 if failure == 429 else 5) + + def unavailable_status(self): + with self.lock: + now = time.monotonic() + return 429 if all(t > now and c == 429 for t, c in zip(self.failed_until, self.failure_code)) else 503 + + def metadata(self): + return {'id': self.alias, 'object': 'model', 'created': 0, + 'owned_by': 'model-router', 'max_model_len': self.max_context} + + +class ModelRouter(ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + request_queue_size = 256 + + def __init__(self, address, *, key_store, fallback_origin, routes, key_provider=None, + upstream_timeout=10800, max_requests=256, max_body_bytes=512 * 1024 * 1024): + self.key_store = Path(key_store) + authorized(self.key_store, 'startup-validation-placeholder') + self.fallback = origin(fallback_origin, local=True) + if not isinstance(routes, list) or not routes: + raise ValueError('At least one model route is required') + models = [ModelRoute(route) for route in routes] + self.routes = {route.alias: route for route in models} + if len(self.routes) != len(models): + raise ValueError('Duplicate model alias') + if upstream_timeout <= 0: + raise ValueError('Invalid upstream timeout') + if type(max_requests) is not int or max_requests < 1 or type(max_body_bytes) is not int or max_body_bytes < 1: + raise ValueError('Invalid request admission limits') + self.slots = threading.BoundedSemaphore(max_requests) + self.body_limit = max_body_bytes + self.body_bytes = 0 + self.body_lock = threading.Lock() + self.key_provider = key_provider or read_windows_credential + self.upstream_timeout = upstream_timeout + super().__init__(address, Handler) + if self.fallback[2] == self.server_port: + super().server_close() + raise ValueError('The fallback cannot use the router listener port') + + def handle_error(self, request, client_address): + pass + + def reserve_body(self, size): + with self.body_lock: + if self.body_bytes + size > self.body_limit: + return False + self.body_bytes += size + return True + + def release_body(self, size): + with self.body_lock: + self.body_bytes -= size + + +class Handler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + server_version = 'ModelRouter' + sys_version = '' + + def log_message(self, format, *args): + pass + + def error(self, status, message): + body = json.dumps({'error': {'message': message, 'type': 'gateway_error'}}).encode() + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.send_header('Connection', 'close') + if status == 429: + self.send_header('Retry-After', '1') + self.end_headers() + self.close_connection = True + if self.command != 'HEAD': + self.wfile.write(body) + + def do_GET(self): + self.proxy() + + def do_POST(self): + self.proxy() + + def do_HEAD(self): + self.proxy() + + def authenticate(self): + values = self.headers.get_all('Authorization', []) + if len(values) != 1: + self.error(401, 'A bearer API key is required') + return False + fields = values[0].split() + if len(fields) != 2 or fields[0].lower() != 'bearer' or len(fields[1]) > 256: + self.error(401, 'Invalid bearer API key') + return False + try: + good = authorized(self.server.key_store, fields[1]) + except (OSError, ValueError, KeyError, TypeError): + self.error(503, 'Gateway credential store is unavailable') + return False + if not good: + self.error(401, 'Invalid bearer API key') + else: + self.key_digest = hashlib.sha256(fields[1].encode()).hexdigest() + return good + + def request_headers(self, *, backend=False, credential_target=None): + dynamic = {x.strip().lower() for v in self.headers.get_all('Connection', []) for x in v.split(',')} + excluded = HOP_HEADERS | dynamic | {'host', 'content-length', 'authorization', 'accept-encoding'} + if backend: + allowed = {'content-type', 'accept', 'user-agent'} + headers = {name: value for name, value in self.headers.items() if name.lower() in allowed} + else: + headers = {name: value for name, value in self.headers.items() if name.lower() not in excluded} + headers.update({'Accept-Encoding': 'identity', 'Connection': 'close'}) + if not backend: + headers['Authorization'] = self.headers['Authorization'] + elif credential_target is not None: + headers['Authorization'] = 'Bearer ' + self.server.key_provider(credential_target) + return headers + + def send_headers(self, response, *, length=None, chunked=False): + self.send_response(response.status) + dynamic = {x.strip().lower() for x in (response.getheader('Connection') or '').split(',')} + for name, value in response.getheaders(): + if name.lower() not in HOP_HEADERS | dynamic | {'server', 'date', 'content-length'}: + self.send_header(name, value) + if length is not None: + self.send_header('Content-Length', str(length)) + elif chunked: + self.send_header('Transfer-Encoding', 'chunked') + self.send_header('Connection', 'close') + self.end_headers() + self.close_connection = True + + def proxy(self): + if not self.server.slots.acquire(blocking=False): + self.error(429, 'Service request capacity reached') + return + try: + self.proxy_request() + finally: + self.server.slots.release() + + def proxy_request(self): + self.connection.settimeout(30) + if not self.authenticate(): + return + url = urlsplit(self.path) + if url.scheme or url.netloc or not (url.path.startswith('/v1/') or url.path == '/health'): + self.error(404, 'API route is unavailable') + return + lengths = self.headers.get_all('Content-Length', []) + if self.headers.get('Transfer-Encoding') or len(lengths) > 1: + self.error(400, 'A single Content-Length is required') + return + try: + length = int(lengths[0]) if lengths else 0 + if not 0 <= length <= MAX_BODY: + raise ValueError() + except ValueError: + self.error(413, 'Invalid or oversized request body') + return + if not self.server.reserve_body(length): + self.error(429, 'Request body capacity reached') + return + try: + self.proxy_body(url, length) + finally: + self.server.release_body(length) + + def proxy_body(self, url, length): + try: + body = self.rfile.read(length) + except (OSError, TimeoutError): + self.error(408, 'Request body timed out') + return + if len(body) != length: + self.error(400, 'Incomplete request body') + return + if self.command == 'GET' and url.path == '/v1/models': + self.models() + return + if self.command == 'GET': + for route in self.server.routes.values(): + if url.path == '/v1/models/' + quote(route.alias, safe=''): + self.model_detail(route) + return + route = None + affinity = None + if self.command == 'POST' and body: + try: + value = json.loads(body) + route = self.server.routes.get(value.get('model')) if isinstance(value, dict) and isinstance(value.get('model'), str) else None + if route: + if len(route.backends) > 1: + affinity = affinity_key(self.headers, body, self.key_digest) + value['model'] = route.backend_model + body = json.dumps(value, ensure_ascii=False, separators=(',', ':')).encode() + except (ValueError, UnicodeError): + if route: + self.error(400, 'Invalid session identifier') + return + if route: + windows = self.headers.get_all('x-model-context-window', []) + if windows and (len(windows) != 1 or not re.fullmatch(r'[0-9]{1,10}', windows[0]) + or not 0 < int(windows[0]) <= route.max_context): + self.error(503, 'No backend supports this context window') + return + excluded = set() + while True: + index = route.acquire(affinity, excluded) + if index is None: + status = route.unavailable_status() + self.error(status, 'Model backend capacity reached' if status == 429 else 'No eligible model backend is available') + return + failure = 0 + try: + target, credential = route.backends[index] + retry, failure = self.forward(target, body, route, credential, retryable=len(route.backends) > 1) + finally: + route.release(index, failure=failure) + if not retry: + return + excluded.add(index) + else: + self.forward(self.server.fallback, body) + + def models(self): + fallback = connection(self.server.fallback, 2) + sent = False + try: + fallback.request('GET', self.path, headers=self.request_headers()) + response = fallback.getresponse() + raw = response.read(MAX_MODELS + 1) + if len(raw) > MAX_MODELS or response.length: + raise HTTPException('Invalid model list') + if response.status >= 500: + self.standalone_catalog() + return + if response.status == 200: + try: + models = json.loads(raw) + if not isinstance(models.get('data'), list): + raise ValueError('Invalid model list') + models['data'] = [m for m in models['data'] if not isinstance(m, dict) or m.get('id') not in self.server.routes] + models['data'].extend(route.metadata() for route in self.server.routes.values()) + raw = json.dumps(models, ensure_ascii=False, separators=(',', ':')).encode() + except (ValueError, TypeError, AttributeError): + raise HTTPException('Invalid model list') + self.send_headers(response, length=len(raw)) + sent = True + self.wfile.write(raw) + except (OSError, HTTPException, TimeoutError): + if not sent: + self.standalone_catalog() + finally: + fallback.close() + + def standalone_catalog(self): + data = {'object': 'list', 'data': [route.metadata() for route in self.server.routes.values()]} + raw = json.dumps(data, ensure_ascii=False, separators=(',', ':')).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(raw))) + self.send_header('X-LLMDeploy-Legacy-Status', 'unavailable') + self.send_header('Connection', 'close') + self.end_headers() + self.close_connection = True + self.wfile.write(raw) + + def model_detail(self, route): + raw = json.dumps(route.metadata(), ensure_ascii=False, separators=(',', ':')).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(raw))) + self.send_header('Connection', 'close') + self.end_headers() + self.close_connection = True + self.wfile.write(raw) + + def forward(self, target, body, route=None, credential_target=None, *, retryable=False): + backend = route is not None + upstream = connection(target, 5) + stopped = threading.Event() + disconnected = threading.Event() + watcher = None + sent = False + try: + headers = self.request_headers(backend=backend, credential_target=credential_target) + upstream.connect() + transport = upstream.sock + + def watch_disconnect(): + while not stopped.wait(.1): + try: + ready, _, _ = select.select([self.connection], [], [], 0) + if ready and not self.connection.recv(1, socket.MSG_PEEK): + disconnected.set() + transport.shutdown(socket.SHUT_RDWR) + return + except OSError: + disconnected.set() + try: + transport.shutdown(socket.SHUT_RDWR) + except OSError: + pass + return + + watcher = threading.Thread(target=watch_disconnect, daemon=True) + watcher.start() + upstream.sock.settimeout(self.server.upstream_timeout) + upstream.request(self.command, self.path, body=body if self.command == 'POST' else None, headers=headers) + response = upstream.getresponse() + if backend and retryable and (response.status == 429 or response.status >= 500): + status = 429 if response.status == 429 else 503 + return True, status + content_type = (response.getheader('Content-Type') or '').lower() + if backend and response.status in (401, 403): + self.error(502, 'Model backend authentication failed') + return False, 0 + if backend and self.command != 'HEAD' and 'application/json' in content_type: + raw = response.read(MAX_BODY + 1) + if len(raw) > MAX_BODY or response.length: + raise HTTPException('Invalid model response') + raw = alias_response(raw, route.backend_model, route.alias) + try: + self.send_headers(response, length=len(raw)) + except OSError: + disconnected.set() + raise + sent = True + try: + self.wfile.write(raw) + except OSError: + disconnected.set() + raise + else: + original_length = response.getheader('Content-Length') + length = int(original_length) if not backend and original_length and original_length.isdecimal() else None + chunked = self.command != 'HEAD' and length is None + try: + self.send_headers(response, length=length, chunked=chunked) + except OSError: + disconnected.set() + raise + sent = True + if self.command != 'HEAD': + def write_chunk(chunk): + try: + if chunked: + self.wfile.write(f'{len(chunk):x}\r\n'.encode() + chunk + b'\r\n') + else: + self.wfile.write(chunk) + self.wfile.flush() + except OSError: + disconnected.set() + raise + + if backend and 'text/event-stream' in content_type: + done = False + while True: + line = response.readline() + if not line: + break + if line.startswith(b'data:'): + ending = b'\r\n' if line.endswith(b'\r\n') else b'\n' if line.endswith(b'\n') else b'' + data = line[5:].strip() + done = done or data == b'[DONE]' + line = b'data: ' + alias_response(data, route.backend_model, route.alias) + ending + write_chunk(line) + if (urlsplit(self.path).path in ('/v1/chat/completions', '/v1/completions') and not done): + raise HTTPException('Incomplete model stream') + else: + while True: + chunk = response.read1(65536) + if not chunk: + break + write_chunk(chunk) + if response.length: + raise HTTPException('Incomplete upstream response') + if chunked: + try: + self.wfile.write(b'0\r\n\r\n') + self.wfile.flush() + except OSError: + disconnected.set() + raise + except (OSError, HTTPException, TimeoutError, ValueError, UnicodeError): + if disconnected.is_set(): + return False, 0 + if not sent and retryable and not disconnected.is_set(): + return True, 503 + if not sent: + try: + self.error(502, 'Model route is unavailable') + except OSError: + pass + self.close_connection = True + return False, 503 if backend else 0 + finally: + stopped.set() + if watcher is not None: + watcher.join(.3) + upstream.close() + return False, 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--listen', default='127.0.0.1') + parser.add_argument('--port', type=int, required=True) + parser.add_argument('--key-store', type=Path, required=True) + parser.add_argument('--fallback-origin', required=True) + parser.add_argument('--routes-file', type=Path, required=True) + args = parser.parse_args() + server = ModelRouter((args.listen, args.port), key_store=args.key_store, + fallback_origin=args.fallback_origin, + routes=json.loads(args.routes_file.read_text(encoding='utf-8'))['routes']) + try: + server.serve_forever() + finally: + server.server_close() diff --git a/docs/gateway.md b/docs/gateway.md index efba0fe..343c119 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -27,3 +27,30 @@ DeepInfer 等自行调度 DP rank 的后端使用 `--backend-managed-affinity` TCP 监听队列为 256,用于吸收反向代理的瞬间建连,避免请求进入 HTTP 准入前被重置;它不扩大推理并发和等待配额。 可选 `--control-file local/admission.json` 支持滚动排空。文件内容为 `{"enabled_ports":[19001,19002,19008]}`,只准入列表内的已配置后端;采用临时文件加原子替换更新。空列表、文件缺失或无效内容会关闭新请求准入,现有流继续执行。健康探测循环刷新此控制,每轮结束后间隔两秒,`backend-state.json` 的各后端 `draining` 和 `inflight` 字段用于确认排空。只在配额、原生请求指标与客户端连接均排空后停止远端服务;保留旧容器与配方用于回退。 + +## 在既有入口增加模型别名 + +`python -m deploy_model_router` 在现有认证网关前增加 OpenAI 兼容模型别名。客户端仍使用既有 key;被选中的模型请求由路由器改写为上游真实模型名。有认证的上游使用 Windows 凭据管理器中的凭据,没有认证的上游不接收客户端 key。其他模型继续转发原网关,不混入原网关的健康池。上游地址、模型名称和凭据条目由操作者自行提供;凭据本身不放在命令、配置或日志中。 + +私有路由文件示例,地址均为占位回环端口: + +```json +{"routes":[ + {"alias":"example-model","backend_model":"upstream-model","max_context":524288, + "backends":[{"origin":"http://127.0.0.1:19030","credential_target":"LLMDeploy/example"}]}, + {"alias":"example-pool","backend_model":"pool-model","max_context":1048576, + "backends":[{"origin":"http://127.0.0.1:19031"},{"origin":"http://127.0.0.1:19032"}]} +]} +``` + +```powershell +python -m deploy_model_router --listen 127.0.0.1 --port 19020 ` + --key-store local/keyring.json --fallback-origin http://127.0.0.1:19000 ` + --routes-file local/model-routes.json +``` + +路由器只接受 loopback 的原网关入口,不能将自身配置为回退入口。上游 origin 不含 `/v1` 路径。`GET /v1/models` 在原模型列表中增加配置的别名,实际模型名不作为新条目暴露;别名详情可通过 `/v1/models/<别名>` 查询。列表声明的是配置,不是上游健康证明。原网关不可用时,聚合列表只返回新增别名,原网关的健康检查应访问其独立入口,不能把聚合目录 200 当作原网关健康。`max_context` 是保守声明的输入与输出总上限,不代替模型端的 token 校验或实测。 + +同一别名的多个后端需要服务相同模型及兼容能力。会话优先保留实例;连接失败、429 或 5xx 时在响应开始前尝试同池其他实例,流开始后的失败不会重放当前请求,但后续请求会迁移。绑定保存在进程内,重启后重建。入口最多 256 个在途请求、单请求体最多 64 MiB、在途请求体总额最多 512 MiB;超出时返回 429。该额外上限可能使多条接近 64 MiB 的并发请求比原入口更早排队或被拒绝,部署前应评估现有工作负载。 + +前后使用两个反向代理时,须在接入前验证客户端 IP 的传递与可信代理设置,避免破坏用量统计;还须验证流式输出、客户端取消、原模型会话亲和、实际模型能力和上下游认证。改变入口可能影响正在运行的故障切换守护;不能仅重载代理,还须先核对并保持现有守护的配置及健康检查合同。命令只启动路由器,不切换现有入口。 diff --git a/tests/test_model_router.py b/tests/test_model_router.py new file mode 100644 index 0000000..e72a383 --- /dev/null +++ b/tests/test_model_router.py @@ -0,0 +1,286 @@ +import hashlib +from http.client import HTTPConnection, IncompleteRead +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import socket +import tempfile +import threading +import unittest + +from deploy_model_router.server import ModelRouter + + +class FakeService(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def log_message(self, *args): + pass + + def reply(self, status, body, content_type='application/json'): + self.send_response(status) + self.send_header('Content-Type', content_type) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self.server.calls.append(('GET', self.path, self.headers.get('Authorization'), None)) + if self.path == '/health' and not self.server.up: + self.reply(503, b'{"error":"unavailable"}') + return + if self.path == '/v1/models': + if self.server.model_status: + self.reply(self.server.model_status, b'{"error":"authentication"}') + return + if not self.server.up: + self.reply(503, b'{"error":"unavailable"}') + return + data = {'object': 'list', 'data': [{'id': self.server.model, + 'object': 'model', 'max_model_len': self.server.context}]} + self.reply(200, json.dumps(data).encode()) + return + self.reply(200, b'{"ok":true}') + + def do_POST(self): + raw = self.rfile.read(int(self.headers.get('Content-Length', '0'))) + self.server.calls.append(('POST', self.path, self.headers.get('Authorization'), raw)) + self.server.seen_headers.append({name.lower(): value for name, value in self.headers.items()}) + value = json.loads(raw) + if self.server.status_code: + self.reply(self.server.status_code, b'{"error":"busy"}') + return + if not self.server.up: + self.reply(503, b'{"error":"unavailable"}') + return + if value.get('truncate_stream') or self.server.truncate_stream: + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Transfer-Encoding', 'chunked') + self.end_headers() + chunk = b'data: {"model":"actual-model"}\n\n' + self.wfile.write(f'{len(chunk):x}\r\n'.encode() + chunk + b'\r\n') + self.wfile.flush() + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + if value.get('stream'): + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Connection', 'close') + self.end_headers() + event = json.dumps({'model': self.server.model, 'choices': [{'delta': {'content': 'ok'}}]}).encode() + self.wfile.write(b'data: ' + event + b'\n\n') + self.wfile.flush() + if value.get('wait_for_disconnect'): + self.connection.settimeout(3) + try: + if not self.connection.recv(1): + self.server.disconnected.set() + except OSError: + pass + return + self.wfile.write(b'data: [DONE]\n\n') + self.wfile.flush() + return + self.reply(200, json.dumps({'model': self.server.model, 'echo': value}).encode()) + + +class ModelRouterTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.key = 'client-test-placeholder' + self.backend_key = 'backend-test-placeholder' + self.store = Path(self.temp.name) / 'keys.json' + self.store.write_text(json.dumps({'version': 1, 'records': {'test': { + 'sha256': hashlib.sha256(self.key.encode()).hexdigest(), 'active': True}}})) + self.fallback = self.service('original-model', 1048576) + self.backend = self.service('actual-model', 524288) + self.flex = [self.service('flex-model', 1048576) for _ in range(3)] + routes = [ + {'alias': 'example-alias', 'backend_model': 'actual-model', 'max_context': 524288, + 'backends': [{'origin': f'http://127.0.0.1:{self.backend.server_port}', 'credential_target': 'synthetic'}]}, + {'alias': 'flex-alias', 'backend_model': 'flex-model', 'max_context': 1048576, + 'backends': [{'origin': f'http://127.0.0.1:{s.server_port}'} for s in self.flex]}, + ] + self.router = ModelRouter(('127.0.0.1', 0), key_store=self.store, + fallback_origin=f'http://127.0.0.1:{self.fallback.server_port}', + routes=routes, key_provider=lambda target: self.backend_key, + upstream_timeout=3) + self.start(self.router) + + def start(self, server): + worker = threading.Thread(target=server.serve_forever, kwargs={'poll_interval': .02}, daemon=True) + worker.start() + def cleanup(): + server.shutdown() + server.server_close() + worker.join(3) + self.addCleanup(cleanup) + + def service(self, model, context): + server = ThreadingHTTPServer(('127.0.0.1', 0), FakeService) + server.daemon_threads = True + server.model, server.context, server.up, server.truncate_stream, server.status_code, server.model_status = model, context, True, False, None, None + server.calls, server.seen_headers, server.disconnected = [], [], threading.Event() + self.start(server) + return server + + def request(self, method, path, body=None, token=None, headers=None): + conn = HTTPConnection('127.0.0.1', self.router.server_port, timeout=5) + hdr = {'Authorization': 'Bearer ' + (self.key if token is None else token), + 'Content-Type': 'application/json'} + hdr.update(headers or {}) + raw = json.dumps(body).encode() if body is not None else None + try: + conn.request(method, path, raw, hdr) + response = conn.getresponse() + return response.status, response.read() + finally: + conn.close() + + def test_auth_and_model_specific_credential_and_name(self): + status, _ = self.request('POST', '/v1/chat/completions', {'model': 'example-alias'}, token='wrong') + self.assertEqual(status, 401) + self.assertFalse(self.backend.calls) + payload = {'model': 'example-alias', 'messages': [{'role': 'user', 'content': 'hello'}]} + status, raw = self.request('POST', '/v1/chat/completions', payload) + self.assertEqual(status, 200) + self.assertEqual(json.loads(raw)['model'], 'example-alias') + self.assertEqual(json.loads(raw)['echo']['model'], 'actual-model') + self.assertEqual(self.backend.calls[-1][2], 'Bearer ' + self.backend_key) + self.assertFalse(self.fallback.calls) + + def test_existing_model_is_forwarded_unchanged(self): + payload = {'model': 'original-model', 'stream': False, 'messages': [{'role': 'user', 'content': 'hi'}]} + status, body = self.request('POST', '/v1/chat/completions', payload) + self.assertEqual(status, 200) + self.assertEqual(json.loads(body)['echo'], payload) + self.assertEqual(self.fallback.calls[-1][2], 'Bearer ' + self.key) + self.assertFalse(self.backend.calls) + + def test_backend_does_not_receive_other_client_credentials(self): + headers = {'Cookie': 'synthetic-cookie', 'X-Api-Key': 'synthetic-api-key', + 'X-Other-Token': 'synthetic-token'} + self.assertEqual(self.request('POST', '/v1/chat/completions', + {'model': 'example-alias'}, headers=headers)[0], 200) + seen = self.backend.seen_headers[-1] + self.assertEqual(seen['authorization'], 'Bearer ' + self.backend_key) + self.assertNotIn('cookie', seen) + self.assertNotIn('x-api-key', seen) + self.assertNotIn('x-other-token', seen) + + def test_model_list_and_context(self): + status, raw = self.request('GET', '/v1/models') + self.assertEqual(status, 200) + models = {m['id']: m for m in json.loads(raw)['data']} + self.assertEqual(set(models), {'original-model', 'example-alias', 'flex-alias'}) + self.assertEqual(models['original-model']['max_model_len'], 1048576) + self.assertEqual(models['example-alias']['max_model_len'], 524288) + self.assertFalse(self.backend.calls) + status, detail = self.request('GET', '/v1/models/example-alias') + self.assertEqual((status, json.loads(detail)['id']), (200, 'example-alias')) + self.assertEqual(self.request('POST', '/v1/chat/completions', {'model': 'example-alias'}, + headers={'x-model-context-window': '524289'})[0], 503) + + def test_unavailable_original_does_not_block_new_models(self): + self.fallback.up = False + status, raw = self.request('GET', '/v1/models') + self.assertEqual(status, 200) + self.assertEqual({x['id'] for x in json.loads(raw)['data']}, {'example-alias', 'flex-alias'}) + self.assertEqual(self.request('GET', '/health')[0], 503) + self.assertFalse(self.backend.calls) + self.assertEqual(self.request('POST', '/v1/completions', {'model': 'example-alias'})[0], 200) + self.assertEqual(self.request('GET', '/v1/models/example-alias')[0], 200) + + def test_existing_gateway_auth_error_is_not_masked(self): + self.fallback.model_status = 401 + self.assertEqual(self.request('GET', '/v1/models')[0], 401) + + def test_catalog_is_configured_and_does_not_wait_for_backend(self): + def unavailable(target): + raise OSError('credential unavailable') + self.router.key_provider = unavailable + status, raw = self.request('GET', '/v1/models') + self.assertEqual(status, 200) + self.assertIn('example-alias', [x['id'] for x in json.loads(raw)['data']]) + self.assertNotIn('actual-model', raw.decode()) + self.assertEqual(self.request('POST', '/v1/completions', {'model': 'example-alias'})[0], 502) + self.assertFalse(self.backend.calls) + self.assertEqual(len(self.fallback.calls), 1) + + def test_flex_balances_sessions_and_rebinds_on_failure(self): + body = {'model': 'flex-alias', 'messages': [{'role': 'user', 'content': 'hello'}]} + for i in range(3): + status, raw = self.request('POST', '/v1/chat/completions', body, headers={'x-session-id': f'session-{i}'}) + self.assertEqual((status, json.loads(raw)['model']), (200, 'flex-alias')) + self.assertEqual([len(s.calls) for s in self.flex], [1, 1, 1]) + self.assertTrue(all(s.calls[0][2] is None for s in self.flex)) + self.assertTrue(all(json.loads(s.calls[0][3])['model'] == 'flex-model' for s in self.flex)) + self.flex[0].up = False + status, raw = self.request('POST', '/v1/chat/completions', body, headers={'x-session-id': 'session-0'}) + self.assertEqual((status, json.loads(raw)['model']), (200, 'flex-alias')) + counts = [len(s.calls) for s in self.flex] + self.request('POST', '/v1/chat/completions', body, headers={'x-session-id': 'session-0'}) + self.assertEqual(len(self.flex[0].calls), counts[0]) + + def test_alias_sse_stream_and_response_name(self): + status, raw = self.request('POST', '/v1/chat/completions', {'model': 'example-alias', 'stream': True}) + self.assertEqual(status, 200) + lines = raw.splitlines() + self.assertEqual(json.loads(lines[0][5:])['model'], 'example-alias') + self.assertIn(b'data: [DONE]', raw) + + def test_client_cancel_closes_upstream(self): + conn = HTTPConnection('127.0.0.1', self.router.server_port, timeout=5) + body = json.dumps({'model': 'example-alias', 'stream': True, 'wait_for_disconnect': True}).encode() + conn.request('POST', '/v1/chat/completions', body, + {'Authorization': 'Bearer ' + self.key, 'Content-Type': 'application/json'}) + response = conn.getresponse() + self.assertEqual(response.status, 200) + self.assertTrue(response.readline().startswith(b'data:')) + response.close() + conn.close() + self.assertTrue(self.backend.disconnected.wait(3)) + + def test_truncated_sse_is_not_a_successful_eof(self): + conn = HTTPConnection('127.0.0.1', self.router.server_port, timeout=5) + body = json.dumps({'model': 'example-alias', 'stream': True, 'truncate_stream': True}).encode() + conn.request('POST', '/v1/chat/completions', body, + {'Authorization': 'Bearer ' + self.key, 'Content-Type': 'application/json'}) + response = conn.getresponse() + self.assertEqual(response.status, 200) + with self.assertRaises(IncompleteRead): + response.read() + conn.close() + + def test_stream_failure_rebinds_only_the_next_request(self): + self.flex[0].truncate_stream = True + body = {'model': 'flex-alias', 'stream': True, 'messages': [{'role': 'user', 'content': 'hi'}]} + with self.assertRaises(IncompleteRead): + self.request('POST', '/v1/chat/completions', body, headers={'x-session-id': 'sticky'}) + self.assertEqual([len(s.calls) for s in self.flex], [1, 0, 0]) + status, data = self.request('POST', '/v1/chat/completions', body, headers={'x-session-id': 'sticky'}) + self.assertEqual(status, 200) + self.assertIn(b'data: [DONE]', data) + self.assertEqual([len(s.calls) for s in self.flex], [1, 1, 0]) + + def test_flex_retries_explicit_capacity_rejection_and_reports_full_pool(self): + body = {'model': 'flex-alias', 'messages': [{'role': 'user', 'content': 'hello'}]} + self.flex[0].status_code = 429 + self.assertEqual(self.request('POST', '/v1/chat/completions', body, headers={'x-session-id': 'one'})[0], 200) + self.assertEqual([len(s.calls) for s in self.flex], [1, 1, 0]) + self.flex[1].status_code = self.flex[2].status_code = 429 + self.assertEqual(self.request('POST', '/v1/chat/completions', body, headers={'x-session-id': 'two'})[0], 429) + self.assertEqual(self.request('POST', '/v1/chat/completions', body, headers={'x-session-id': 'three'})[0], 429) + + def test_request_admission_rejects_before_reading_or_forwarding(self): + self.router.body_limit = 1 + self.assertEqual(self.request('POST', '/v1/chat/completions', {'model': 'example-alias'})[0], 429) + self.assertEqual(self.router.body_bytes, 0) + self.assertFalse(self.backend.calls) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_model_router_caddy.py b/tests/test_model_router_caddy.py new file mode 100644 index 0000000..1b9abe2 --- /dev/null +++ b/tests/test_model_router_caddy.py @@ -0,0 +1,51 @@ +from copy import deepcopy +import unittest + +from deploy_model_router.caddy import prepare_servers, proxies + + +def proxy(): + return {'handler': 'reverse_proxy', 'upstreams': [{'dial': '127.0.0.1:20001'}, + {'dial': '127.0.0.1:20002'}], + 'health_checks': {'passive': {'unhealthy_status': [503]}}, + 'headers': {'response': {'set': {'X-LLMDeploy-Service': ['engine'], + 'X-LLMDeploy-Gateway': ['caddy']}}}} + + +def site(port): + return {'listen': [f'0.0.0.0:{port}'], 'routes': [ + {'match': [{'header': {'Incremental': ['*']}}], 'handle': [{'handler': 'subroute', + 'routes': [{'handle': [proxy()]}]}]}, {'handle': [proxy()]}]} + + +class CaddyTransformationTests(unittest.TestCase): + def test_existing_routes_remain_with_isolated_model_errors(self): + source = {'front': site(19000), 'fallback': site(19001)} + before = deepcopy(source) + result = prepare_servers(source, front_port=19000, fallback_port=19001, router_port=19002) + self.assertEqual(source, before) + front, fallback = list(proxies(result['front'])), list(proxies(result['fallback'])) + self.assertEqual(len(front), 2) + for p in front: + self.assertEqual(p['upstreams'], [{'dial': '127.0.0.1:19002'}, {'dial': '127.0.0.1:19001'}]) + self.assertNotIn('unhealthy_status', p['health_checks']['passive']) + self.assertNotIn('retry_match', p['load_balancing']) + self.assertNotIn('X-LLMDeploy-Service', p['headers']['response']['set']) + for p in fallback: + self.assertEqual(p['upstreams'], proxy()['upstreams']) + self.assertEqual(p['headers']['request']['set']['X-Forwarded-For'], ['{http.vars.client_ip}']) + self.assertEqual(result['fallback']['trusted_proxies']['ranges'], ['127.0.0.1/32', '::1/128']) + + def test_rejects_mismatched_routes_and_unknown_trust(self): + source = {'front': site(19000), 'fallback': site(19001)} + source['fallback']['routes'][1]['handle'][0]['upstreams'][0]['dial'] = '127.0.0.1:29999' + with self.assertRaises(ValueError): + prepare_servers(source, front_port=19000, fallback_port=19001, router_port=19002) + source = {'front': site(19000), 'fallback': site(19001)} + source['fallback']['trusted_proxies'] = {'source': 'static', 'ranges': ['10.0.0.0/8']} + with self.assertRaises(ValueError): + prepare_servers(source, front_port=19000, fallback_port=19001, router_port=19002) + + +if __name__ == '__main__': + unittest.main()