From 8b0f5965282ac6713b4df77e7bf8fa4bc1f73459 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 10:01:33 +0900 Subject: [PATCH] examples: HTTP validation and benchmark harness for PD serving (#240, #248, #263) Add opt_prime/examples/pd/http_matrix with the reusable parts of the HTTP work from #243, #252 and #264: - validate.py: HTTP lifecycle validation through serve_batched.build - benchmark.py, benchmark_server.py, atom_server.py: warm native/dp2/pd HTTP benchmark with token parity checks and external-server configs - heldout_quality.py and frozen fixtures for four models (fixtures/) - tests moved to opt_prime/tests so CI collects them Adjustments: atom_server.py module overrides are optional (default to the in-tree modules); heldout_quality.py reads .json.gz fixtures and accepts --model for the request name; the Qwen3-0.6B fixture records the Hub name instead of a local cache path (manifest hash updated; input IDs and per-case hashes unchanged). Run outputs are git-ignored. Co-Authored-By: Claude Opus 5.5 --- opt_prime/examples/pd/http_matrix/.gitignore | 6 + opt_prime/examples/pd/http_matrix/README.md | 67 ++++ .../examples/pd/http_matrix/atom_server.py | 76 +++++ .../examples/pd/http_matrix/benchmark.py | 264 +++++++++++++++ .../pd/http_matrix/benchmark_server.py | 18 + .../fixtures/fixture-manifest.json | 74 +++++ .../quality263-gemma-fixtures.json.gz | Bin 0 -> 23984 bytes .../fixtures/quality263-koni-fixtures.json.gz | Bin 0 -> 31333 bytes .../quality263-qwen06b-fixtures.json.gz | Bin 0 -> 24091 bytes .../quality263-qwen4b-fixtures.json.gz | Bin 0 -> 24108 bytes .../pd/http_matrix/heldout_quality.py | 139 ++++++++ opt_prime/examples/pd/http_matrix/validate.py | 310 ++++++++++++++++++ opt_prime/tests/test_benchmark.py | 52 +++ opt_prime/tests/test_heldout_quality.py | 134 ++++++++ 14 files changed, 1140 insertions(+) create mode 100644 opt_prime/examples/pd/http_matrix/.gitignore create mode 100644 opt_prime/examples/pd/http_matrix/README.md create mode 100644 opt_prime/examples/pd/http_matrix/atom_server.py create mode 100644 opt_prime/examples/pd/http_matrix/benchmark.py create mode 100644 opt_prime/examples/pd/http_matrix/benchmark_server.py create mode 100644 opt_prime/examples/pd/http_matrix/fixtures/fixture-manifest.json create mode 100644 opt_prime/examples/pd/http_matrix/fixtures/quality263-gemma-fixtures.json.gz create mode 100644 opt_prime/examples/pd/http_matrix/fixtures/quality263-koni-fixtures.json.gz create mode 100644 opt_prime/examples/pd/http_matrix/fixtures/quality263-qwen06b-fixtures.json.gz create mode 100644 opt_prime/examples/pd/http_matrix/fixtures/quality263-qwen4b-fixtures.json.gz create mode 100644 opt_prime/examples/pd/http_matrix/heldout_quality.py create mode 100644 opt_prime/examples/pd/http_matrix/validate.py create mode 100644 opt_prime/tests/test_benchmark.py create mode 100644 opt_prime/tests/test_heldout_quality.py diff --git a/opt_prime/examples/pd/http_matrix/.gitignore b/opt_prime/examples/pd/http_matrix/.gitignore new file mode 100644 index 00000000..05f8f031 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/.gitignore @@ -0,0 +1,6 @@ +# Run outputs stay local; only the frozen held-out inputs are tracked. +evidence*/ +*.json +*.jsonl +*.log +!fixtures/fixture-manifest.json diff --git a/opt_prime/examples/pd/http_matrix/README.md b/opt_prime/examples/pd/http_matrix/README.md new file mode 100644 index 00000000..d6b047bf --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/README.md @@ -0,0 +1,67 @@ +# HTTP validation and benchmark harness for PD serving + +Scripts that drive the OpenAI-style HTTP stage server (`opt_prime.accel.serve.llmd.serve_stage`) +as built by `examples/pd/serve_batched.py`, for aggregated and prefill/decode (PD) serving. +They write a JSON or JSONL record per run; outputs are git-ignored (see `.gitignore`). +Models are loaded with `local_files_only=True`, so pass a local model directory or a cached name. + +| Script | Purpose | +| --- | --- | +| `validate.py` | HTTP lifecycle check for one configuration in one process: readiness, 128/2K/8K-token completions, concurrency 4 and 8 against a serial baseline, stream cancel and client disconnect with drain checks, invalid-request rejection, prefix-cache and host-tier metrics. | +| `benchmark.py` | Warm HTTP comparison of `native` (one GPU), `dp2` (two replicas) and `pd` (prefill `cuda:0`, decode `cuda:1`). Starts isolated server processes, checks token parity against warmup references, then times streamed batches (first SSE event, e2e, estimated TPOT). | +| `benchmark_server.py` | Server launcher used by `benchmark.py`: runs `serve_batched.py` with the given flags, optionally loading an HTTP module from a file first. | +| `atom_server.py` | Single-device Rebellions ATOM decode server for `benchmark.py --server-config`, optionally fed by a subprocess prefill worker (`--worker-python` + `--engine-factory`). Requires `optimum-rbln` and exactly one visible `/dev/rbln*` device. | +| `heldout_quality.py` | Held-out structured-answer check: `fixtures` freezes 36 synthetic cases per model (addition, passkey, Korean table lookup at short/2K/8K tokens); `run` sends the frozen token IDs to an endpoint and scores exact JSON answers; `compare` (Python API) pairs two runs case by case. | +| `fixtures/` | The frozen inputs for four models (`*-fixtures.json.gz`) and `fixture-manifest.json` with the SHA-256 of each uncompressed file. | + +Tests: `opt_prime/tests/test_benchmark.py` and `opt_prime/tests/test_heldout_quality.py` (no GPU needed). + +## Running + +Paths below are relative to this directory. + +```bash +# HTTP lifecycle validation (torch backend; add --pd-prefill-device cuda:0 for PD) +python validate.py --model MODEL_DIR --device cuda:1 --output evidence/validate.json +python validate.py --model MODEL_DIR --device cuda:1 --prefix-cache --host-kv-gib 4 --output evidence/validate-host.json + +# Benchmark: native first, then reuse its prompts and token references for the other modes +python benchmark.py --model MODEL_DIR --mode native --output evidence/native.json +python benchmark.py --model MODEL_DIR --mode pd --reference evidence/native.json --output evidence/pd.json +python benchmark.py --model MODEL_DIR --mode dp2 --reference evidence/native.json --output evidence/dp2.json +``` + +`benchmark.py` defaults: input lengths 128 and 8192, 32 output tokens, concurrency 1 and 4, 3 repeats, +4 slots, bfloat16. It fails (and keeps the raw record) when output parity breaks. + +External backends: `--server-config FILE` takes JSON with `servers`, a list of argv arrays each +containing exactly one `"{port}"` argument, and `metadata` with resource facts (`physical_gpus`, +`physical_atom`, `physical_rngd`, `slots_per_decoder`, `producer_cache`, `decoder_cache`, +`producer_chunk`, `local_chunk`, `context`). For example: + +```json +{"servers": [["python", "atom_server.py", "--model", "MODEL_DIR", "--cache-dir", "ATOM_CACHE", "--port", "{port}"]], + "metadata": {"physical_gpus": 0, "physical_atom": 1, "physical_rngd": 0}} +``` + +`--http-module` (validate, benchmark) and the module options of `atom_server.py` load a trusted +local file in place of the in-tree module, for testing a candidate change in isolation. + +## Held-out structured answers + +```bash +# Use the frozen inputs as they are (gzip is read directly): +python heldout_quality.py run --fixtures fixtures/quality263-qwen4b-fixtures.json.gz \ + --endpoint http://localhost:8000 --route native-gpu --model SERVED_MODEL_NAME --output results.jsonl +# Or freeze new inputs for another model, before any inference: +python heldout_quality.py fixtures --model MODEL_DIR --output new-fixtures.json +``` + +The fixtures record the model path used when they were frozen (the Qwen3-0.6B file records the +Hub name `Qwen/Qwen3-0.6B` instead of a local cache path); the server rejects a request +whose `model` differs from its own, so pass `--model` with the served name (input IDs and hashes +are unchanged). Serve with greedy sampling and normal EOS; the runner sends `max_tokens` 128, +refuses to overwrite an existing output, and stops at the first HTTP failure. +`answer_correct` needs a parsed JSON object whose `answer` string matches exactly; `format_correct` +needs exactly one `answer` key. This measures structured-answer adherence on 36 synthetic cases, +not general model quality. diff --git a/opt_prime/examples/pd/http_matrix/atom_server.py b/opt_prime/examples/pd/http_matrix/atom_server.py new file mode 100644 index 00000000..7687f69f --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/atom_server.py @@ -0,0 +1,76 @@ +"""Single assigned ATOM HTTP backend, optionally fed by the established RNGD subprocess adapter.""" +import argparse +import importlib.util +import signal +import sys +from pathlib import Path + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--model', required=True) + p.add_argument('--cache-dir', required=True) + p.add_argument('--port', required=True, type=int) + p.add_argument('--batching', help='optional trusted ATOM batching module file (default: the in-tree module)') + p.add_argument('--http-module', help='optional trusted HTTP module file (default: the in-tree module)') + p.add_argument('--pd-module', help='optional trusted subprocess worker module file (default: the in-tree module)') + p.add_argument('--worker-python') + p.add_argument('--engine-factory') + p.add_argument('--width', type=int, default=8192) + p.add_argument('--kv-blocks', type=int, default=3) + p.add_argument('--max-output', type=int, default=128) + a = p.parse_args() + if bool(a.worker_python) != bool(a.engine_factory): + p.error('worker-python and engine-factory must be supplied together') + for name, path in [('opt_prime.accel.pd.pd_batching', a.pd_module), + ('opt_prime.accel.platforms.atom.batching', a.batching), + ('opt_prime.accel.serve.llmd', a.http_module)]: + if not path: + continue + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + from opt_prime.accel.pd.pd_batching import SubprocessPrefillWorker + from opt_prime.accel.platforms.atom.engine import AtomRblnEngine + from opt_prime.accel.serve.batched import BatchedOwner, BatchedService + from opt_prime.accel.serve.llmd import serve_stage + from transformers import AutoTokenizer + + nodes = list(Path('/dev').glob('rbln[0-9]*')) + if len(nodes) != 1: + raise RuntimeError('benchmark requires exactly one assigned ATOM device') + engine = AtomRblnEngine(devices=[int(nodes[0].name[4:])], max_seq_len=9216, + batch_size=1, kv_blocks=a.kv_blocks, cache_dir=a.cache_dir) + worker = owner = server = http = None + try: + engine.start(a.model) + c = engine.model.config.get_text_config() + if a.worker_python: + worker = SubprocessPrefillWorker(a.worker_python, + ['--model', a.model, '--width', str(a.width), '--engine-factory', a.engine_factory], + slots=1, layers=c.num_hidden_layers, kv_heads=c.num_key_value_heads, + head_dim=getattr(c, 'head_dim', None) or c.hidden_size // c.num_attention_heads, + max_context=a.width) + tok = AutoTokenizer.from_pretrained(a.model, local_files_only=True) + batcher = engine.batcher(prefix_cache=False, prefill_worker=worker, local_prefill=worker is None) + service = BatchedService(batcher, model=a.model, vocab_size=c.vocab_size, + max_output=a.max_output, tokenizer=tok, + render=lambda ids: tok.decode(ids, skip_special_tokens=True)) + owner = BatchedOwner(service) + # The benchmark uses fixed-length ignore_eos requests; no EOS-dependent timing comparisons. + signal.signal(signal.SIGTERM, lambda *_: owner.submit('stop')) + server, http = serve_stage(owner, 'decode', a.port) + owner.run() + finally: + if server: + server.shutdown() + server.server_close() + http.join(5) + if worker: + worker.close() + engine.close() + + +if __name__ == '__main__': + main() diff --git a/opt_prime/examples/pd/http_matrix/benchmark.py b/opt_prime/examples/pd/http_matrix/benchmark.py new file mode 100644 index 00000000..8c273ba2 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/benchmark.py @@ -0,0 +1,264 @@ +"""Warm HTTP native/DP/PD comparison; isolated server processes, exact controls and SSE timings.""" +import argparse +import hashlib +import json +import os +import signal +import socket +import statistics +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + + +def timing_summary(rows, wall_s): + return { + "requests": len(rows), "output_tokens": sum(r["output_tokens"] for r in rows), + "wall_s": wall_s, + "output_tok_s": sum(r["output_tokens"] for r in rows) / wall_s, + "median_first_sse_s": statistics.median(r["first_sse_s"] for r in rows), + "median_tpot_est_s": statistics.median(r["tpot_est_s"] for r in rows), + "median_e2e_s": statistics.median(r["e2e_s"] for r in rows), + "all_text_equal": all(r["text_equal"] for r in rows), + } + + +RESOURCE_FIELDS = {"physical_gpus", "physical_atom", "physical_rngd", "slots_per_decoder", + "producer_cache", "decoder_cache", "producer_chunk", "local_chunk", "context"} + + +def load_server_config(path): + """Trusted executable argv arrays; no shell expansion. Resource facts are supplied by the experiment.""" + config = json.loads(Path(path).read_text()) + if not isinstance(config, dict): + raise TypeError("server config must be an object") + servers, metadata = config.get("servers"), config.get("metadata") + if not isinstance(servers, list) or not servers or not isinstance(metadata, dict): + raise ValueError("server config needs nonempty servers and metadata object") + if set(metadata) - RESOURCE_FIELDS: + raise ValueError("unsupported resource metadata key") + for argv in servers: + if not isinstance(argv, list) or not argv or not all(isinstance(x, str) and x for x in argv): + raise ValueError("each server must be a nonempty argv array") + if argv.count("{port}") != 1: + raise ValueError("each server needs exactly one standalone {port} argument") + return config + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--model", required=True) + p.add_argument("--mode", choices=["native", "dp2", "pd"], required=True) + p.add_argument("--output", required=True) + p.add_argument("--startup-timeout", type=int, default=180, help="server readiness deadline in seconds; excluded from timing") + p.add_argument("--server-config", help="trusted JSON argv templates and resource metadata for external backends") + p.add_argument("--reference", help="native benchmark JSON with identical token-ID prompts") + p.add_argument("--http-module", help="trusted local HTTP candidate") + p.add_argument("--lengths", nargs="+", type=int, default=[128, 8192]) + p.add_argument("--outputs", nargs="+", type=int, default=[32]) + p.add_argument("--concurrency", nargs="+", type=int, default=[1, 4]) + p.add_argument("--repeats", type=int, default=3) + p.add_argument("--slots", type=int, default=4) + p.add_argument("--dtype", choices=["bfloat16", "float32"], default="bfloat16") + a = p.parse_args() + if a.startup_timeout <= 0: + p.error("startup timeout must be positive") + external = load_server_config(a.server_config) if a.server_config else None + if min(*a.lengths, *a.outputs, *a.concurrency, a.repeats, a.slots) < 1: + p.error("sizes must be positive") + if max(a.lengths) + max(a.outputs) > 9216: + p.error("input plus output exceeds context9216") + import torch + import transformers + from transformers import AutoTokenizer + + tok = AutoTokenizer.from_pretrained(a.model, local_files_only=True) + head = [tok.bos_token_id] if tok.bos_token_id is not None else [] + fill = tok.encode("This is background context. The red box contains apples. The blue box contains books.\n", + add_special_tokens=False) + tail = tok.encode("\nQuestion: What is the capital of France?\nAnswer:", add_special_tokens=False) + prompts = {} + for n in a.lengths: + left = n - len(head) - len(tail) + if left < 0 or not fill: + p.error("prompt length too short") + prompts[str(n)] = head + (fill * ((left + len(fill) - 1) // len(fill)))[:left] + tail + assert len(prompts[str(n)]) == n + reference = json.loads(Path(a.reference).read_text()) if a.reference else None + if reference: + assert reference["prompts"] == prompts and reference["dtype"] == a.dtype + refs = {} if reference is None else reference["references"].copy() + output = Path(a.output) + output.parent.mkdir(parents=True, exist_ok=True) + procs, logs, ports = [], [], [] + result = {"passed": False, "model": Path(a.model).name, "mode": a.mode, "dtype": a.dtype, + "torch": torch.__version__, "transformers": transformers.__version__, + "replicas": 2 if a.mode == "dp2" else 1, "slots_per_decoder": a.slots, + "physical_gpus": 1 if a.mode == "native" else 2, "cpu_threads_per_server": 4, + "producer_cache": False, "decoder_cache": False, "producer_chunk": 256, + "local_chunk": 256, "context": 9216, "prompts": prompts, + "warmups": [], "correctness_batches": [], "batches": [], + "harness_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "startup_timeout_s": a.startup_timeout, + "timing_scope": "Client first SSE event, full response e2e and estimated TPOT=(e2e-first_event)/(output_tokens-1). SSE can coalesce tokens; this is not per-token kernel timing. Loading/warmup/correctness checks excluded."} + devices = ["cuda:0", "cuda:1"] if a.mode == "dp2" else ["cuda:1"] + if external: + devices = [None] * len(external["servers"]) + for key in RESOURCE_FIELDS: + result.pop(key, None) + result.update(external["metadata"]) + result["replicas"] = len(devices) + result["server_config"] = external + started = time.perf_counter() + try: + for endpoint, device in enumerate(devices): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + log = output.with_suffix(f".server{endpoint}.log").open("w") + logs.append(log) + cmd = [sys.executable, str(Path(__file__).with_name("benchmark_server.py")), + "--model", a.model, "--device", device, "--dtype", a.dtype, + "--slots", str(a.slots), "--max-context", "9216", "--max-output", str(max(a.outputs)), + "--kv", "paged", "--page-size", "64", "--chunk", "256", "--port", str(port)] + if a.http_module: + cmd += ["--http-module", a.http_module] + if a.mode == "pd": + cmd += ["--pd-prefill-device", "cuda:0", "--pd-prefill-no-prefix-cache", + "--pd-prefill-chunk", "256"] + if external: + cmd = [str(port) if arg == "{port}" else arg for arg in external["servers"][endpoint]] + proc = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, + env=dict(os.environ, OMP_NUM_THREADS="4", MKL_NUM_THREADS="4")) + procs.append(proc) + ports.append(port) + end = time.monotonic() + a.startup_timeout + while True: + if proc.poll() is not None: + raise RuntimeError(f"server{endpoint} exited{proc.returncode}; see server log") + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/ready", timeout=1) as response: + if response.status == 200 and json.load(response)["ready"]: + break + except (OSError, urllib.error.HTTPError): + pass + if time.monotonic() >= end: + raise TimeoutError("server startup") + time.sleep(.05) + result["load_s"] = time.perf_counter() - started + + def request(n, count, rid, endpoint, stream=False, gate=None): + body = {"model": a.model, "prompt": prompts[str(n)], "max_tokens": count, + "temperature": 0, "ignore_eos": True, "stream": stream} + req = urllib.request.Request(f"http://127.0.0.1:{ports[endpoint]}/v1/completions", + data=json.dumps(body).encode(), headers={"x-request-id": rid}) + if gate is not None: + gate.wait(timeout=30) + start = time.perf_counter() + key = f"{n}:{count}" + with urllib.request.urlopen(req, timeout=180) as response: + assert response.status == 200 + if not stream: + out = json.load(response) + choice = out["choices"][0] + assert choice["finish_reason"] == "length" and len(choice["token_ids"]) == count + return {"id": rid, "endpoint": endpoint, "tokens": choice["token_ids"], + "text": choice["text"], "tokens_equal": key not in refs or choice["token_ids"] == refs[key]["tokens"]} + first_event = None + usage = finish = None + done = False + text = [] + for line in response: + if not line.startswith(b"data: "): + continue + if first_event is None: + first_event = time.perf_counter() - start + if line[6:].strip() == b"[DONE]": + done = True + break + event = json.loads(line[6:]) + if "usage" in event: + usage = event["usage"] + for choice in event.get("choices", []): + text.append(choice.get("text", "")) + if choice.get("finish_reason"): + finish = choice["finish_reason"] + elapsed = time.perf_counter() - start + assert done and finish == "length" and usage["completion_tokens"] == count + text = "".join(text) + return {"id": rid, "endpoint": endpoint, "input_tokens": n, "output_tokens": count, + "text": text, "text_equal": text == refs[key]["text"], "first_sse_s": first_event, + "tpot_est_s": (elapsed - first_event) / (count - 1) if count > 1 else 0, + "e2e_s": elapsed} + + def batch(n, count, concurrency, rid, repeat, stream): + gate = threading.Barrier(concurrency + 1) + with ThreadPoolExecutor(max_workers=concurrency) as clients: + futures = [clients.submit(request, n, count, f"{rid}-{i}", + (repeat + i) % len(ports), stream, gate) for i in range(concurrency)] + start = time.perf_counter() + gate.wait(timeout=30) + rows = [future.result(timeout=240) for future in futures] + wall = time.perf_counter() - start + return rows, wall + + warm_start = time.perf_counter() + for endpoint in range(len(ports)): + for n in a.lengths: + for count in a.outputs: + row = request(n, count, f"warm-{endpoint}-{n}-{count}", endpoint) + refs.setdefault(f"{n}:{count}", {"tokens": row["tokens"], "text": row["text"]}) + result["warmups"].append(row) + result["references"] = refs + for n in a.lengths: + for count in a.outputs: + for concurrency in a.concurrency: + rows, _ = batch(n, count, concurrency, f"check-{n}-{count}-{concurrency}", 0, False) + result["correctness_batches"].append({"input_tokens": n, "output_tokens": count, + "concurrency": concurrency, "rows": rows}) + result["warmup_and_checks_s"] = time.perf_counter() - warm_start + for repeat in range(a.repeats): + for n in (a.lengths if repeat % 2 == 0 else list(reversed(a.lengths))): + for count in a.outputs: + for concurrency in a.concurrency: + rows, wall = batch(n, count, concurrency, f"r{repeat}-n{n}-o{count}-c{concurrency}", repeat, True) + result["batches"].append({"repeat": repeat, "input_tokens": n, + "output_tokens_per_request": count, "concurrency": concurrency, + "summary": timing_summary(rows, wall), "rows": rows}) + result["metrics"] = [] + for port in ports: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/metrics", timeout=10) as response: + result["metrics"].append(response.read().decode().replace(a.model, result["model"])) + result["parity_passed"] = all(r["tokens_equal"] for r in result["warmups"]) and all( + r["tokens_equal"] for b in result["correctness_batches"] for r in b["rows"]) and all( + b["summary"]["all_text_equal"] for b in result["batches"]) + result["passed"] = True + except Exception as error: + result["error"] = repr(error) + raise + finally: + for proc in procs: + try: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait(timeout=10) + except ProcessLookupError: + proc.wait(timeout=10) + for log in logs: + log.close() + output.write_text(json.dumps(result, indent=2) + "\n") + if not result["parity_passed"]: + raise AssertionError("output parity failed; raw data retained, no equivalent-output speedup claim") + print(json.dumps({"passed": result["passed"], "parity_passed": result["parity_passed"], + "batches": len(result["batches"]), "mode": a.mode}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/opt_prime/examples/pd/http_matrix/benchmark_server.py b/opt_prime/examples/pd/http_matrix/benchmark_server.py new file mode 100644 index 00000000..e37fa5fa --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/benchmark_server.py @@ -0,0 +1,18 @@ +"""Launch the production serving CLI with an optional trusted HTTP candidate.""" +import argparse +import importlib.util +import runpy +import sys +from pathlib import Path + +p = argparse.ArgumentParser(description=__doc__, add_help=False) +p.add_argument("--http-module") +a, rest = p.parse_known_args() +if a.http_module: + spec = importlib.util.spec_from_file_location("opt_prime.accel.serve.llmd", a.http_module) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) +path = Path(__file__).resolve().parents[1] / "serve_batched.py" +sys.argv = [str(path), *rest] +runpy.run_path(str(path), run_name="__main__") diff --git a/opt_prime/examples/pd/http_matrix/fixtures/fixture-manifest.json b/opt_prime/examples/pd/http_matrix/fixtures/fixture-manifest.json new file mode 100644 index 00000000..4bdc5408 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/fixtures/fixture-manifest.json @@ -0,0 +1,74 @@ +{ + "quality263-gemma-fixtures.json": { + "sha256": "0c24b1e759148dd7de68e15fa435e03c77a85b947fd6e923eee46af8c87edf44", + "cases": 36, + "length_ranges": { + "0": [ + 38, + 68 + ], + "2048": [ + 2034, + 2048 + ], + "8192": [ + 8183, + 8187 + ] + } + }, + "quality263-koni-fixtures.json": { + "sha256": "22f6e5b117913530ef6d58ed5352bb47684d55409535157b75f0462d64797eaf", + "cases": 36, + "length_ranges": { + "0": [ + 150, + 182 + ], + "2048": [ + 2040, + 2045 + ], + "8192": [ + 8186, + 8191 + ] + } + }, + "quality263-qwen06b-fixtures.json": { + "sha256": "5fd8133ab6026cf757ae005d671e7778c69e141660d1771627cf1df7070e6a83", + "cases": 36, + "length_ranges": { + "0": [ + 40, + 77 + ], + "2048": [ + 2037, + 2046 + ], + "8192": [ + 8183, + 8192 + ] + } + }, + "quality263-qwen4b-fixtures.json": { + "sha256": "701be76546e6fa3c7410475bc026a50c21c677abb7725565fb650b514fc71c98", + "cases": 36, + "length_ranges": { + "0": [ + 40, + 77 + ], + "2048": [ + 2037, + 2046 + ], + "8192": [ + 8183, + 8192 + ] + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/fixtures/quality263-gemma-fixtures.json.gz b/opt_prime/examples/pd/http_matrix/fixtures/quality263-gemma-fixtures.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..4cb6f6c0558df5072b3189bdd53caa0f93eaf2d8 GIT binary patch literal 23984 zcmeHvdtB1@{{Q!!v(0U-TsK{oa@L%7F)cGSZ~R(w&J{XWYNSXPZ#WgNC@S#VY*~_8 zTAGr0rllnYDJCiiGg4E$q@<{bq^PKfHv|O)epI_vJNs^K-*X;)f9L%lpU3AdpNEeZ z&-e57d_G?v0QSxQdb0kinc|7DTNb~1Jhv#{nTn=n6Y&85kN$pgW)0Ns4SSbf$$K6U z$Zc51&Dsg#r4KLEzqWQwg$v^3$+p68%=%E5KCrkSd4{%IsdL*z!}QEB8si9BKOVgA zo1`93?|#IlpbMaJ?nedFx7_LLoyj^PXN(Pkk~z~c*bk@>DH~!wJ{}0WmhF0NOrHR#%$QCSWubdPo3kqE$MzeM&Su@R zy?wl@gsMIjz?oL4+yIAHc`P(HXizl?z)S?yAOo-XnMUl|j=cH#b8QIKiM`Vtqtz}<-QsV^jOJoC+DNK!&_$WIIN0?4Zij_b7r8Sb(h(|(c+=zc?TBp zf_<YpnoS%*4n}|>4eh31>$yVC~ z^@rT~3Ki)dh{^@I&+;|vl4|A%balltP}8l;0&o~Jo=Zgnrw}TUFE;n{W=X9>D7?AJ zlYoZK5dAtknM^&Z5C{+hq%CmU@kIU^PDK70i&8r`QevjQ=j>E#MPX}r(wvv)>^sW` z?V+d}fF#_5E~_m~>e*ttoO(D93Z~C(cDWPTPpJv)bza(`s5J=z_V&l!)s*nN6%~+1 z)Nz^Xl`2U^OCFMU6wtLf9@~LIh4s*J+&OBeE;$H(I4U7`h83isK81=|< zAIV&;=wk~t^Ug8qDQHIlI#NAC5o%`R_~$S^*TK9kbQ;d2rlO8MZ+MG7Z?`L1AX=Vv zmfgrn@=Q5+tEI4J{tNr~KIkn|*CU>Ayt~ATrR(Pf@@j+*#YC>>Ko1-ddgd~Q;`k)H zInOKEnEUsS!j^7ZgV;N^-y5*nqm$utg)1{zp=EOoeb66o@Q>1L8`Cqbt z(hna?G*SXzwakGhy>)dNf9N|H&)Jyv+Tp>eb-N?qS~{Eaf9E)fZ^a)(-!ty3kC@yM zuC6}x-O}dhx89?^wyX#J>lkD+CG46tw${CPPpgo9UKmC@tQVW552^9(0p&EXh*!^6 z9?o(0X;39;GG zcYwYHcETqb*VG+l)IqQ<2#T$^ss{19Q91WfJ^KQ@<3M<+Byg>}ep*|_c_EM1QZd

KBCzhOj|OSXHPHsa070u)z_@MAse7+xkfp1QQ3$vFVMu+TVs$p?HBbG$i5~mC+jD{tRb_ba~f0qB&b5) zBR*|WM}y;E(VSSi5oa8~QO!VYG*9OP3 z6WtWw%h>B-{nqBiS?w$uh^%fiU#b8Oz0OziH3W@1M>Ao;n8Zj}G7f^2j$*H(Px+wg z<7Pht(iA)HX<*^=vrV?1St*+SQUUxSW?I?hv%y!T0-_Jy+q!FU<}`e=7(7K%D?&W< zkU3QivHHAVt%rqW)9-adWu^>;azf*v0cr$kBy|Np zTAG7e)YhkhH8?eC5iMQ(#2W)uAY}!z$(hFG2F&20HT*fGbY4t=sCZ@a=voEU@(XVa zze4ly-9=9iZ+YP{ukMPLG1p5mSl8;Clg_*#=c-`QXiS}V463hcbm2A%zlfb6MBx#x z^Nrq^x!YFlA`Jia$La^9*^~7%6kfARtOz|ea30a5nqZ>!4N30$oe9)qtHrWND=5hw z0e9C7UsLz(FDp6J5bk(E9MWsyti0rGLGN_q=QZ4aY0=N`A)CCsWUK3>|0gqP;^4_4 zPYxJ(tt9l`;$MZZrLQilaM$C@!asrR&C?P3%9qeX{_Sgbgao1EzGv3#;%*v+;FW)nC%vOs*2a+Zwyr$yF%U= zD1JwmMUI%36u&7n3+_4jLEAl0fC1hNcy$pQsnfRg5NXqAd7P8Q+rs1DE0&n*D6`u@ zM~?W}xDTaSNTwbm^}U}S^LlyAu}~VtKAc^6jdwm!Y)K*%i*sO5^OhvAjhH54`49HM zy1iWeVo}{y0d8H$r~!9QyHh=9;Wm33V7 z#`P~QrAZ??X22zkBc(!Ox`64~qOeEdR6#e>DYa-~)*HCds_xWFi~q2bOtYOWQ;l5zqn;6sz)DGXV#t<(iys_fM{LoTc=D5g8ObT97`}U(RNou`wjz!WWrzv&ydpK?h*=LVr8OQD}q-|$1=gqEZXxr~0R{@Eetk)Pmt0?7* z6}9b$ipubL!W${st>k2(pKsl=g2}=iU4tXf4Ng~suu%xpJSoI$AB_5GX%*BM3Hi|T zN}ME#eys$|YXtH1Sn`T|j@)t=*$Ujs9j$%`ol z!QOUvrO#6sVJQQqXNHlnB@rSS8aTL_W2(BtAdRVeIh&O(IQnK~gL+oNPeQ911%5`= z&&o>A=EyIW6&{#~P_r=#9^a{MeZFQMD`cp{7T_IDScY<$1hm&QWKp2E6V+KAf*JbK z+~d*O0+pna4|lTHH1CbJcnNQ0tJ7Wwe&C={GWKKUt(L!iEC-^E95wPRNf4uUNfhK> zpke8E%E=f4r*S_yF;&vXd!WC}SA%ATFw+wjVl&q@3Z9a?BBdP7ZvDHfER)^BNlrZ);e| zlHRT9Qqgs6x`OjYT4c4gMs&sppKRI!6{b+gOI>aIO6``%Pk8Eq5GsG)+wK`3dDbDF z+SpSOL|oYg#=VJQ6WYFJz@taVjp*H7tG>amWQjx(-5Ry-XI`Gi)oTI%WnFf@Zcm7* zZ`Ku^0&>~B-2~yqpv?c&5>|V~tZ%--e-s>#Rh~ zj&o;mEmjU-^z0qK)-N8KRV?IvdmU*Qm^%E1I7U5kf?rVs3uzIJZKoXw#2Y%?7EF$E z0DglZ(Y@Ew77axro;i=AD)^jvht~t(VsuY(KJK*ICwrQsM zm}NQ}bg;LA+p$|+705l*H!+GrO*4TUBPQ}WTycm4`e6oP+AC^jnnW>=FHqWFHkw`+ z&K;a-N)#xr17vY;@Bs3J)0$v*+-XKHz&9vIC#*`OgNtPJ>^k-GsYjRo;-P1~Wi{{^ zu=g5BITE{|@8NY=^y|o%;ihQn?3{8Q4YnTDW8@d91=!q^JWxrk3HgfW) z-Tnc;lKLTN-erOgw9n5J?l1&4a66RwDB!!Wr@gSy37#6!{`Jp#;bbH~gZy@$bGF~! z%T^@jVUK}-Jl_l3(4Sx$!L%NVXp0Qc21WSud`n2-Y{^u*fMjRg31*BgY_)I{f(=G|sZiV%-2D%O>BjF_)bwDcA+!WuIY)65$?@Yjh{?R=;5bN6N}WHW0i*ek#~*4q4c4M zySyXBoQ^mvG7F!zYD??X>3TE;%z5u3&eFlG@X+IlvXWfGcrWgu***7YLVC&Z}w zClW~$CvvUc%W){xf$M}8Ja2nE>RU!z=NaM{%bY=0Y6s zJ&RAu-7c@rc2rxIi@#o$ha$CQc}rWCHc5Vx^mc7o!YL~*%Lz0;c{9uk6cvyNf#~x2 zYm44VCrOxGaMHw>fu|ul=FSeR2^DQg>?v{iwsg<|crtV9gl+6bo2L@YVBUys$Ruq? zw8aMVl`JBOsw}Cxja+jow<)4X87{l_sT-T1}l{qY(IDLh4z=b z7JGGj>gLL)rOX;DzO|i%E17*G7m1jyOef?(G4V)3+sSC_L9QVM=FvX=G1x@Pq9%bRIj;_LOsK+T2#&rRy$3ksPYdiy3hxEf&B}-UJIFyuy6DLPOPX8 z93aJFkL`JJmBUf3WiLXW_VW_~w@24Hi0uS6P&r>VX4n2E7Ur=(4^;E(By6uhOTzeC z5;mh#H@3O}rZ>36++4S^2BDf5*KK66lFBeQCw}Q$-y(RD(7dWd{un2tjMx!4J!rpX7b>j4bV*(HE>);RzYw`Gxyj zC2XvlwktOzmsTpYva7xB^hpS~Lawg!V+$rS*r!(Gq%HU{;3e$&#;&yC@8wG>-!U}>P0pS^3kGla$~Xht zvNvNToE(c4@da-;W?zYoZgT-(tCtnP(EYnO-=C1iu7(%3Ju@GKKrP`Kyq9tDY}!GW z+_R=E-7)p4f#*QQD0O1Fxt<;n&3<+~ip_s^JT~S&cRYeKjh(;nXF0g(Nh)!sNU(Wn zrH`H~c^mF}nGjanDhW&&pESWWL3$7n+$eJqbgDlbE_Q0dM0q0%5150Jt9`BVZY5*@B#rv^F28W9SUwIFZL0z|60{Sdtwh$%!=Di)Kl8* zLr2KZc6>S5nHv}|E$&$%o5Jnn%_K&AkXGzU3?&gnLrVKom+R+Z(ed_J3!ZOLDKle@ zS3svf$Gx#zv9RrEH%`bdD=nRhsCn?Tha@t-E^a4SE`Bt$q6R~e0Fg2@$(c``4V_Jd z$~B*47Q?P=I7w`7N$vo9Aq__)PppmOSB8wQvtO&56YNiOSU%^!4o%++|LF9!prL~} z8|S5queB*=_ZAKnTdsD`1~`=IlQ&pl&FTWT!TeHk3#$*<=L7fmQbNP?^5$!iZ^JL< z2I;tqsuSeWlpIR`7V98E9WN56H#+5J^<`=?xRroBE(BME=D_eKl7PHNk@*3r&ap5? z=;ZlsFW4n-SjAxlvAxy=Vm==QZhIaSe>xm_zwvO)`J;y;Oiz0_HvKO@93yHsg8JTnhZY5?YlF)yaXEZg75N z{O>$ZlNxihG>L|NMw3=OPm?M?HF49osBpR^G2YH5PL(=AVjzyt!}hTdyt_R|Tmh~O zE6}kqM+P&v7czNg~Qm-8|xM7I)O#$_t z%_-?mVWm|y=<#~n*5!{!>YsV@k<5d8z6X|sfv3`xi@#QyS{!~RO@&20Cr#yAP?P)u zT`S5&rD9gCPhdb#yC5)E7MZyL#nzJ$JM>XNN8NtA3|8$-)^#|cY$ix+&qEi0$k=Qm zPpVWjCvYjuy7K%28KuGpc8!x`Lm%L$OD5fhdc-O;&4%c%IEhJaRh5%$WEs{d8yQwL zq&@Qrgc>2n_eYU6&iVI=tStHqA}jenBI|!6vZ4)=HUb8Ch0Udko!896hZQ@6|J3$z zHW>>${-N{LFDvFBM`wHb>BwoR*yM7r0Nb}FtPa4J?&u1Hk}ur*XWeN4xsry z3hP@0_s6>Sv}2QUWwAUxNw~rM^F$9$ATu|!WZ_RsAQTcEG+Smyue1H{0z` z(svVOJ#mFtAi4XL4Rt+gHfm357sU^gd%t1yqr5A~>RWH?Wtybx%E~5BM0<_GB7%=! ziH!(7UQ)lEWd&bz2&5N!`)df+VUH zo>42^T}3#_*dW%?zssB!pc9qB@BSY2_zm|eB^OlV)fxRnQ5<#rI%dl*4)-Px(L_i= zZE#@^kU@`bhq?{Jjc-6h2Z?VkvIz>&mlZ7V4wSZZHK}Ho{gn{MzsRFk+WKbsGME3 z^s*dhy%JB}h&EgEyk+<^gJ&bZG|MEv&Zwn#+Ko89A)NPveT-m-D^Ek=Q-Cl<_>{hq z*@2PBWs z0I@1-a+>U6_7cykzNiF1AFZKgD8=HU120RXf9T4TyI!Jts!#5&nTW$E7u;ghEDO9w zLFr^TD$Uf>5{7z8E5Z$;?@r6zoiZ58VQ-87Ry})JPx~EL`02-HSz9`^`QG*dht-%w z!M`%Ufv=P1j}Lc*-tBFh{un^>y->Jtn*%k8jI6mB7+@4$`t>*0&kK?r33k2LJdp-l z^1K2Jx8QnU5iOQ&IOZ2wuls1DY+F0-fg;ymtJt2&`oX! zq@aO{O~1VRsUD1Rf$N0LiOt!^{@|fE+jmjEHD6uYpX3qqg3|v}1K9ke0qhffbwNBm z5|I^vcfS z*k2l6`H8@}XvN0yU|C_56)p-3>)=&GRxcK%h$p}p`&w$c!Ue}h1UGdjR68b$>^mC@ z`a;Tayz@@+HSak;Kg06+D%bT*+7Xu6JNpUqR<>5RTkLGDZ?hj?4#=}fxnZ)Z%B!B} zjQ<8VY+@U5#5iXa(myzeRTcb#-aYC)q(d9n_}=49TnA>p34WtY9R7{HZ<|H^Qqp zUO7<#sxSw;WhCa`aQ52=sk>?e*;l|}S-O%(X)}7Do*-4schAd9{7Nd%RkP$3MR3cD zktJ+6O|^fkDrNJJ;RIe~(|;S>S|JJZ{U0M?9eZUz3}O{(=hP~F;YA3pA<0PQu@02N zP)O5$$r67VSAP4IDehYPXMQ@LWji>VxS0`WMdM@yao<0#Tar;^ogh@j&3qz~yP#}7 z0gxI6Fcc=TSDQ~%1Y75y-XpHrOd$-R7w$MZir_AB91l%!bpIN0^>JNF5FBM%yR_G5 z7ebrY)UnVn+J+KIb6kCW&98%VQ*fmx-iHk@o#-fCV;$s)e}w z&LjW&o*lh+ACM!CJedVZ9f^A1UGR_G?zSuC*>Al{hy043m9x2e()hjL<4v%5-|EvI zuYb_29k5%|aS;k8e!X_fw6BS4R0Ds@;h6R9kS}sl$euyBgj-kiMX%5%EnC`nVb@O4 zv)#0EefD!NZ6*w>|d=1wSiKP4mv(+Ugu9GJ|Mm}>3HCne& z^29BS9N+ddQZ)d5Zu0Uacjp;u)%`MRp*mYx&UNECk^3B5!PKjbm)za&%aarul(#i# zP{?+J^u%~lw1ZZi9c=6NEW30O0Q80cxNO1+N2xdNrFz`3ucsK@VhkvVj!~R0?6de;VI^DbN3V zF37~^UT3t_BWM zPsT>>u5hrAA8hDmk$_2eqpA#k9^S)B=*O(X21g|g6<38=>o6+uPz8~YdMY;-nb$5F zwcM@`*%TC;9f)L{qdHAzI$se!k1aohwkDp*QXX1a%DC4ky!R*Z-bl&x`%%3A0J99m zEHM`c@nK%|vhS*EJ7$`sNW?=ih%1q4^E>%7mN#*}zfSlfanb6ZX#E@MnI< zEn1dkxEkmb-lUod*aBawOZ&6tdNq`MD(upT?Yp~^IHXS}Yxf=u9En?1{*MN;@!fxg z)lY@ozX-Xq4>IZHb>xa8;LWezO$J}o=bpH;`}Cc1)b6F-DGlShmvoT*nv;YCC%X!e z?z-{E&hCKgEve^hhR>JQ^h`DSG&k_I>VEs0%nUy#6W8iFML>3Tj_&)U)cBj}Mitf| z)4d`SgLcu@j}IM$?Cb1*cqW*&V}rQz(O74&A!rxY!RRw?FsrK79FThPlQtTRbnR}- zD2Z3D(^GWOa;C{YHBCa`_?VC7( zJV!EN<&A%=eQgLU)q$=l+$BiF=79#T+LgWw_c}bS9v<)@w=(y+yyd(;sAYaNpdnln zYDo#nhQ5g}@uGmC`2q;^Pt@eN&NVXiR1Xc#ZX0UJvLo3dEWb zKJuPQuL{P|ru0Dtlu^r762zb$l~KFxqx^l}f*)b&EJC=pCtY^dkiDs`)G!7KK(ayE z3>Z?GS?a**KL4<*bxtk4VHqU1R2LiVB+hiT({zeBydNzV)vJeKK!CCVEtkUq1sZkV z=sk9ug3XQu&WGCo=@f9mYHYH9;-i9r8A@}f!v9j44YvDXn9+wRBI%q=?C)?qEbFAB ztU1Z?0xzT&_@PqrA@2_AAv~P7=?|*&FKwq&dLd-8M<|!r6 zGjVv$0#?FzLMyuyI~dD$G|6y@z6D=7sY^v-hZ&jDgpk#+%4(7t2zzTwHT# zi$h9EmUPHem2}vstg(ovL`4k%Kbvm#3q=N1aTCjNHHJ2XA-^bw{eBVrLkh=LGrMqo zbM2Gq&$e*d72oAHFRXa#@AC1{)0t;Blc0exr2O6a(sL=3jX}cp&(;GOa3j zs)2WblrTg9?G`B|2pLQWMjf&0jzREz!G^peUfuy07zvxaMO621i42s}NGwFi7FH~+ z;ob=APtpM$dE_|+_}}ki>hikpCsS+QM~8FE@aattYu9*QZaL`lP4lj9X^b$T0&JK! zZv6fkyPX1GoOicBy7Oasp&P3iEZRXobBFp5xC%M#hHuh=7oxD%*2T_7m`Z1#Mu6+1 z(7fh8!M8i^<2GhxNQbiv19MZpOkbC3(tde#b0t+pZXPH_vAd7{0FF@6KLJN1xs_Z? zP$0`~%leC<{;b+v3dE)C?jhRSkl=nKbQcy;a~Q|8{xY@Vss+w!S4(Mu_B|??PR&%e zxg?xBBjN?_GeY%e5aeykfmwG?$`pS;O>xA#oo}-K2^_o7qd$RTKf*5``r>KI)uT)SQjat}1f!NKsCP>$-3n zI`_ut=)nhstwyCqDPg|)d3 zHgk^4%p1Po0e|n|SXr2bgYMj2fAA6KfvVJ;jmV@$Ew;qz?ZZ{CosYly_FQ1Y$e=~L z;XamwZ}jEQts;xf__KSaWF02a1R2Qt2qexO7E`gcAZeycF)GS{)@d`aB|!q7*h9Z1 z1aYyoHWYd6ItqLR)_J4tWcd{JROe`H)!`foy*)2!@EbdpB&u&8Y_RHmk&uM|=X`rB z?@2;Jj#USd5V>(&?JT6YiYrie{s1f+w7{Zm8NX_)L(}Yu8OoDXTsc&ha0e6L-_e9F zhsuYj8aWiGSOj$E9CEkO{@|Kay;R=v(qN!g>ROX2_P|6j_$Dr*Ea!?rGp~abG__sCLjBFeH&bI!#d*=QE&)&WbX6+PkgEZ5c1E@ScQy%5=YBC8&@iizcTxzyL>@< zhE`UvbDW0tEIT__e2%9R@7t?RHgjPhlbnD}`8Re=2A|^Tb^ayKB;W67mJambzY-n) z9p6O+)M}y1Z(rxViqxu|>>a7v|4!B%?{aMAjtA zw`-^IUE}`FPV(s+QWgimNQka7aO-HKa*NC_cTfF2cb9}oY#T#yM|Z# z+dlaP2>7EPdl$9o*M$kWJ9=>u>ahRQc8q39Uchiw%NDT9`P%Wi;blq&A1xh}STNf8 z$T>DPX9gur;n5gLk?PSSaWB7q?04vBFW)L-o^u9{#j6Pg1wiyVO@crUr@P@bQ*&}Q z5VAZ&FzwJRneE3YI1wIs8=_y`-D z`yFApZuZ=J$2s4oexYrxch}rVsHhGprg){8*tx#>ZF`F?XUub7F4W3{ev=&))l-{l z@&1(gxDN(T%@WSl^YZp`*49#vhO+FXw3OkZ{et& z|Jb@dXU_}_L<#iN%&{4p2QLgYcG}{3WJQM`hP2~Vw7*c{LdP|IPo)x5>Ctf5>`~Cm z!_oUjzVo{H`R$Y+zC9NoB>*|55HY+vLE~iVCJ&a{Q5OtV6^XkKw2B>~i<xOY`_M(91C!9!oQE-#dP(g85b!WCs_yz?VAr*Rh+IA|sGkh%y8HmZYqC=C~ zSJ`;~9}@%Nq16KIP_zQ;ap^)Z>AcjTa|d97$IGJ33L-5f6-i|~z0{))4uX##N(k{z z6$X4E++c}7&y!C;(h{Vs08og?qXjbsk@y%3`x;)F|6}Ww0Vf1twmPY%T)d?&3!Qqy z93dc9Al+hn@2hOpDutkciM07s$EzGleD;e2zTn@iC-H-frXU$_o^dPf$}!Cc?%brpoD=-tyN~uoci>FTOw$v%%9LBWcZ7N3l$7L( z6Rr?-65U8YnD{X6QMYRc057O@l=z~is0BCY&Ii+<^y%rJp)_i*)E~^E9L@I5c+(pt zekXV%+Ryl*m8A&2z%;k&;5b1Oxu7$cm^(^7Wp@W%t;I3SHLx;_3cO4 zeGhx0s_hD@$5DmI%1E$*0+Y_b=Zc{~7S!1mdoc&)DC=hCs< z%VX0`F$4B4!M)goXZPnOq8#)3zlmX2`3{1{W;&&29Bz(f(M)!_=VSVv3X_lJ_WN8& z>JqQjVI%vKeu8DpOg&f=q$rT)_~`07-=a zsd4Lb^Q)+`IZ7<>ZA%C#nzd^fRC0}AM$~!vZhSTeuFhNtZDbtTx~W;71Iw5jvY(0$ zh2C)m?2ubGn-p`e3`EQr``q>qAesS-6>rJvXM~Y#tklY!Z&xF+3+;gt+XCd70DpO+ zbg9;Q7Vf1FuKJ`~2Ea$9Zm!o5 zxWd#K_`FqN)>PEQ@{1vZxiHVbxh2BT=7UU?MO$2u)LdzG&!R135!*xcY;NHquC3Mv z2j#OTzF%1pK6n3N4K%JWg(9aC9Z{eRBJ4fS2hUqOzb!D>Fn@48k=|^B+*M@#J@n08 z!s5Qfm;Qp6s?EcQ`&K%%*$yE>*+#^+lXG_#<_;&oDGT4?ka^Z3h#Q#!smqm6eXQmQEC6??1wh^~_nZ*|St&6Dcme zxVb(zGrBv7*%oS_Ib$ATKiA3H9fXgP1GHBwE{(Qs-3-kFW|}ba))Tvf1Svdz0Sy~P zp$(P?4nmyy;70Wi3wIiK9|hbPYWA3^>sqC# zy9BJ7tcDhZ${v|MzHSr@6kc;@i3IL)^KlO;KkNa$K;3i@DtM3{S6MRP&=C`M##hK& zWpM6Ax%({S2FB4_%P&-k6~d2g^jq41vD{fBZ03?iC=03=1}p-sxrc7fNS+Vn+^!B( zPhbKDuLj(u>`ew*3|?J#uRV8 z=1a2@*wpgeb(a6Zg~8S!xw*D#;1cI%shojWcaGll3h%@v2fX0C6cE$MOR0EdS8mgY9&gxPC_ zK-Od&c7kuoi7mL95CUK{RF6aj*y8=w##W+xe(}ZiM*rF zcx#HJYm1g@f`du5(70d#o}WSD$DnL%6Or``Y!XZi?M=$-l!g`-dxETA*-V=r9_o)e zx_V{FsJa}!cd!-LKdl!A@W7JhD0Swc01wwC=ED4ZU#p8;QCqdb66c2VdV>OJa5-hQ z>x1WQi*2*a3l@%%EeE6$VxqblPYc@-CkG(=j3sv;be*j-gSCUqHzoJ%X|xt9CCnNUNx!R;${_lRIMg&e*>8KRdRArn24^ zoLWJFbJbq4H3dM3B3A|u00xHF>;0Ai9n&}3MOrcI)uvka3-n}iPDP46AjrVuBGPA3bA# zo#%MaVN%^IAuW3K=JBA$ZSjEF@AjJEq6p1HW9sUelna7)P<(MuJyCvxud?NZnk|?j z@;+dq`Bfj#j}bh=>cMOCBX$4I4V$ol(7C9%@o#?F&(FY~4VCw8xF8`$%g?itksWGo zz6+bDR7sp|DCCF|_Vu#rRl5xDiG|D|mCUAsp|m|f--yBmd*8yxQo+Tzc1e>~ zJ*WdY5NJpBG#q%G@&E;w8kDk}gFb;V-egl-{C%@cce~w5P2Ul=e3D%!R_IMb&Jch! zcR!O^+bibLkf~sWQl_`NLydJYM~w2%BR!pm(QJRS4=I7M@iLiJz^#XiIkBxg#bwT^G2xm{yL8BzjwwMlHFdjo$+sPy@mb0R~*w=xa}{-H=5v2-vHiPJ3>-YL3@CW*!wn8^`$AKTAbQw^^E@ zZ7*w`in^K&6=`(9++aFBFu9Pjgc>I@Bym0o^7-)e7}RwTuFpvV5tEL1@VLghCFz8I z*Jya;=U1m4J&U2urbBLJ8>8)3q#)RJ;`45hehI=fme-8W5peE%)E8v(j93daTNd@L zm6h0|MTh4v4AW{Ji6j|xUp5yc9us>~qT7vXdh<@=s(|(hrBz$PCBCjW5j3ptbJ{VW z%s@xstAYB`BiRK(er2bq?TbH}X;(@YK&2*6NK_%xqu~ZwuDCC9;AAlqF^gEK*FG{e zs{b%hut}GvrGJlCHwns*%%y&|jTqSjiMgU|=snYLfcWP2g8P>cJ&C4^7Klqe60hQPooSdsps`Oroy!i{l$D2bqBXYO-$mZijEpS9Dg1=sXl+ibK57> zQ(Xjm|I#&NI7>ucDP8N~U45g|350uVkqc%kW~4l4?}t#$WcWVotUB2?h}w&Pr&50| zxsfo5H8TTm2gI%@>WG+WQ|;6v>A{w7?eJ~c==78?U*$MYWL@?#Hh?7-JYH@I@I@7h z_B~FykuQ5aAQ_z|nwzTA9PVsz!cM=$5uhOxlLosH$XyIDv{>4b;P~6!9tPq3n7Zz@ z)0pnobHF%dzq6$`N0_RmyD3n&KEp!!Oh6K9H$Rl*OX32?wfh65B{wSypl;wLrke zRZk@8m`v6Rn!v6Is~vN~)RrEhyQCoEK2kNDN~(>q=Mh#_WKPYo7jTT8RDwqE!aN`o z7#22P97r3%tE)2yYRof(>GKKrE|<7aReu&KFHE4EG2N7xsXE-t`;kVy)x5cC)^3kY z6>Ul2+Ra^KQyv;FLafcfsk<7P%t-=+hrN8kYh&0WZxyRpv${Uz5i0*hwP|>Q+fV^x z45Lr3UZBHzU@M!<&b@71h>=^H)n~LR?p~*Zy2f9)x(3O(SMrx3SWaFTHFa@w*z|O6 zKS`iGM5UmcNnDC=B0AU|TX(nF3C~67pp%IRzQmZ8te%s*eGx-0GZxlvG&1N@w#FzxrkWm!-UJ0B2oRt%RV$T^=o zj&8Ncs7cCaw?5oWP2)~HGpJ5>iS7)@x(3L7)@c21HC6I|Y3T^JI;*ZWel&AjAGFux zbXf~@SaPm?L5U(w=#tC+I<2=^B7oQ0=!#k{)_L>W_%XE%#*g5W-2e6jHWo#SSW)~b`g^?q`68?fU$g+^0e@JU(% z#+ui*@+&|GS03db$O9#D*7#2wqeEZT7J^R&ux=d{4p{~$3_C{a5)VU9GM<$x3z1_M~ zI8?~#%FAl~hEiJpczf&E8um)wdi8-ZlZB;)Ax9JUeZgNIlb(FhhqRJ*qrl|-w6fN{CB4m)TwdoqV5e3TZ7oH=kT}=vljU61 zduDnAx)hcVH|ef#fnNjQ0- z*Rr;G=_T%0=Avv$Hm*LcA#3Gkxw=*@jG?@-aC^QnEmRHOe{cton|+`gX`|LZj^T&%NyUr0K{Rz<7&(%adN$q`Qj&>yB z@Y2V{4yR`9oWY^nRar|xle~{tU zp|D%8v42=uX*A!{K?g!k^OG&2t!Be%a4f0cJp^->51El$w}-drOD@BaC{rWzjV~-u3IMNEbOR z(Auo4X;(Vsb|I&%KwD*H=KW7)W&Ojn!~10w@{Df8t4}=s1)R4CaqM&5$;jB6l&qs_ zrw8>R+wYV?9^NZ$F)6ZIGX8TYlpDl$`XdPIQ>PpG+R zUyFm}rjq2SoINmKP9|q-50nHe9lInCj^AtA6C$6JT(f!L4CGMO6zOHHs(jm+2G&oV z48kfMai8I~8mpVJl$(BO1TfBvcOlA_Mdfg(_vp#>fG`b^*+Ye(dtEv0WsU3Okm8#e z2{Fo`Lc)yXTDJDOu=>p$7eB|$(2~LAGOSF=(ir4|olHy4Qe+RE@Inpq$QtwOF4#wB zsQfJ?Zo1~^ZEM}FOfENl~B(0H1aa?!}#&X;kIE=ZroWLaz^v_%3+X|iefluR> zDsu&3FgF0zuvcZ$z-@J$?Kot963q?{WRGOl-Y^i}q%E`f@>bO~m~BOk-<7xALeXG8 zVZr4$VZg!Ar9gBB)~@Qw`c-*>*i7ievwk;jr?^zrlo&Taqm*BrFLB7#y}U_`;_MYN zkf;J1EAxmTR$&8e7b`4hBu)$!?mkQOmcQAJ3S7B^x&m50T2r8wjGY_Yz#YhWaC*`) z60r(Oo)mR4GL?`Bi>o1`-jH%7B|;IWz*~-q4sU6>eb#VKAn4MJBZTM&E3B^IX;sm$OeytAfC;o34JsqjGsTJ<^c3Q}U$5vSDr<*c(K`@nvi0&SicYVgAIGfJ}3^^)4v_aENHRuthC zzOs4N_@n-r(;1(O0F9N+u32Uf?wCv0>v_MpsT3-)gLWJ$_bJ*~m{oqqV>(hdLnF0O zTbLOY<98%GLv6iO!D}di$N5NCSSdox1n|*x z84!2tnuTyus75+D6rU%7OV^(YNA=Fl%&Z&yG zN6L$H*F&>c)<}QIcJ#K;5r^)GbD?G2|8P@mdBxcj&vQ0KF?ep2J{lPYWJBS)j`{@! z%5+p3&@$|Cs_DLo>Vof9)N#~->xfg7;cUt5Zq#63%h0nqYtySViV^a}&8#!POI_Qn z04^~#myjFoXd<8AV9Ox+00KT;AlgSf)wBx{h2lF_e=-i)7Mhi+bn1B2SZUJ1epA!3 zy`iHsF_QV-sHaubcMc}1R$a?n=T50{#8C6+3&Z0?Bb(1&h>*5gJ|w1KTE3 z-AeYGTuXyMZ^FM6W?1WseDm`V-3z$5ayemAb^geX?VrYXAb%Dyb-JQsU|@Y>k{b?8 zFwQk;YV09wF;k~4W*U6b#9wKN!%Pn=E8*jkj5*A-ElEzsd}~(rJ1PTjchza*ng8e1sr~P%Qwfebbt^;lanxz@C#%!jzpqXm|21_=e@~s7 z|BgDf{J1(D+rd$%zr+LJ9oxnR4vZ(8aX)%)_Lyq+>zL#FaLnD03pw-~acc5boK}3A zcKl~?3OeVVcGTlwyuj^mng~Y@#t6_&Vot|1M?X=Vavor2aMJ;i9|^xLO6->(T1z?P z;rK%lc%*}5aA{-ANM8HOvnp}1i@M#H0x8kTBr{{`rAzrHM^LIbkikylw#S@7#duVd z+r=4FceL8iol&#uh34hHr6^#P4+fPC8M*sZi58&kn;?v2`UAJ_3zX{9&NrKkr%cQY zjue$Hs`p(`5ypROQL@|k_qplBTW(rimdX7aZu+BMKmU^%K#l!wP2Yp8`FM zNC@RwwB&eqx6ae*!>;qZV2|rMISKLvQ*EDDIwuR`FU-nR>FzH`SKc%*`;JSjD4F`@ zPU&VLe`CQ*px8p=RJeB$vh8qJV2@;7eB z*$>?gjfdm@aa(T3gHLih*pm}3yroIFmcvRr&a2HzcV|av`PBMn?fIxUmA@<&Xt#GD zz;@ePZ2BDmvS#*WpElCz(=d_8j-l?QMAfj&e}eEhSrTFs`q+9d75QXL|Mn^saP^#TmGF6IQ73eOIzHs&neub%v@(m`w34tK!LA@n&BnHV2*ASP|-nS*way-oIMhayMswC)J<&Gzm zWtt)f&m+l$l65imAtoD4S9gY5CQ{N*A&0-k0qgaAQpK%|13m58;)IQl%ge`AuJ7?#wHF5&Ae z18JP;_k}(mp#mQ?;sdg0e8JpbQ)rb*)B~>UuxLPPnEc`lC2ohtsn0w~n z&m>aV4H$tMND+s)f<7|JO|*F-};M&*FW?ZTQ7hG#~g&n z?nrWl#WMk`$tY7jtHecH7er~C=Mg>WN*l>e>uKr4w8HfFi{=+Ezxb_}uUyIu{?N-m z@`0C+FSlc_vRAU%^lbmy>38Phbc*rsnTt#I8cLG-WFNZKX_->2;UB^fd&J1Kpw%WnSXsar4Oj%6UT5U+ zr}CK9jjRPIk!e@yCpS1VuJ{jUocK6r#-(#++#b%1%K_n>yLY!-blm5+g6RsV3Uk(BZ{6FTdwqQJHFPzushf=}L^y;kn$e%S(O>O^;Syb>)~5 zpuzMpuNIr!{6LF)ehy+qeK3)1)c|Ew*yXfeySZDLy$@ z`kVAjTsOIs4h;7VwMO>7wfAz<8JPUl@uhIdte(h7!6#V0&$_S*z=@?@&sS zeW?1!`6fdlTrGHhcsUQtZUW5NxfrnL8|tvZgPY73P4vPlMBB{<$9bKfh zu`s+2&SZJ^&lU-1hAw`0&G2_Ca+E9fr_k%7SacqiSrD+c6qLuq3Z`co&S`8C_}ZrK zV$%oNqsxPndDwO0;rWjR&h>z2gbTv!eomavTuI=BaAgd$I8nfmbc+EeZLg0+}pw(7x+T=Q01>C;wNC! z8;Ja$K@A92G(8&di-~BX&wu4z;qL!n8eWtBX&S29zdsENzBqOG15P8fV3TAzYd9 z?&|Pv#B5AmK>tf_ixIcewSI?w2+-C1y;Casm1w1(j;Wx{YkSg|a#m8lXqPrj93M;z z_B1gt$Ci5%2jXY9_W9SkW*ypn?16sGKJo>LZqCt_G*_#8N|_B`&Y|~3HK(39bT`hS zkO*3H+ppB#oLsmy`d|4=!1gz56+9n|^sWAY^-}CRK8W&N5wd~RsCpUrC+)M5=~IqO zs2!JemD9%dQbYk3kBI6(tFX2$f^4Ljvk6@(1eT7(rpyK_xw*81O&XjnKSnt zr|)1ZC~0Yrdp}GBWmm(pGHzI^{7iY@O?^oBC7WZvcrM?(%EP~9yrzq-@8^MTo~r%8 zarqEMk>pejH5QCzp~1s(Tqo@MKjSY*=F20<%5eo;iTK(uV&8tTpvf3j%dtn%L^~s6 zKeLb>X3!m<{Hf1k%Ll{i!=)_&0$T3RVp{#g z4PBvW>57W#&M#goo2Prr_J41xY!WDT)WA&Wd|t^L=)qi9bg!PfV=gBTQeAK!Skm9{ zMZouW{HUXQOLcT9;A8}E5Nncp+VnOf{a*cfCMQ3swtlOO`hJ?qYsgn=0yUp+-13ZK zO$h+qi@#%6(%qQmaH&wj_wj%#esZ_4$x$BHN$*ph>L2!9kG_G~XZ`PJ?^y>5|P)apl!Ovfw~J(r}}p)XC2x)&pwe zRiSQsW}4AZ!Rl5uM)Cp*PH#q4}h-)Uu(F%Vp*~fgjgtALoA` z4{3fp^ZeG${@^2H*B`~Ffv+I`zk?nB?;iWHZ?9J0{-+I24PECI%0uJf-uUCbjvnB6 z=vh2Y*Z8A6aRbk7R)8=vZ9%89&)N=1X`iT|k?l)oi`s41ZN2Dt*V5iLr2L$^t?N;@ zcq3|3pi&wD?6xWAGPaK596Y>tQej1lv(H)ZmRj6Gx$T<>J|t0%*OC-Z614^Qz2x!)EKn+ZsX-gFU%ipIh3d8Q@w4#yN)~R0Xvz zrp=@S7jML9%bJDh=nsHa4qDh8`m~Mjf9F?)HLc6Iklgo^Z|0TE#JtHhc(JqQ(SoJW z9mPshy3Rkm=vsjdwl9YrQz?OW{?yL;|M=@->HS!)nvbnbAc;$&2Ws?E#mybBvdCLw+8Hn0o=G5|#0&3Sy+a~2# zeAZXZKTmq_GVG~Is8eOt=k083!Q4}~u*QjEh8mn$prLwN;YM5Yt!sY0M}?hwQb{lU zqF#e?f+d$4WT71?A(MTIcISnmI@9Z&NyA2$m2;#mg~?x~`{HJzWj+kkZgd1%^|2aX zx5wcQ`iM-qIvI*Yhz(bj7O3F|JirSZnhs?ZO&kAa8g#YIJ>WdHpuwY2u?h0 zV;B!+uP?LeG(tMk)Yr&UOiofOByZ%E%C>{jWB{os^Oma61GQ$Z-WeOHlemW0*^ zr+lP2sm6Eb1&;DJAmg{Yh#c>S z>Ev69e#TwA`YvKHWP2*zpdKs?ujx8(+@6{*asC&`zI}c9$Xgt!n*nR01ar_YUE zpR0bP?V=iAyrJhzZHE{ANJV}3|D$HfR%!n!Yv!*yH7j`Z59&)iyjHA!6WAA?AEFEN^1%_5s`fl4irZ3 zLNzF|tlZq6s#N`uU?HU-e=X5E$2&2k%MxZHH934?(tC^fm?^0GUXC1_Ej4x;-LrlY zbWMf$=+I;7BfGvN9`nvBy-wNZt-Bj~kPASTq{mx57S~KRzihVH`pTw2Ra<#${M=G) z)D_A@Q>fJalSV>Y^C#h9G_;iylU*IJBf;M{CVjk0yKG#ecG))NNC!L$mbQO#sIk;> z-vcY}T&s%vHmysLSqLiKBgyuBO>{fH{fWX7;4kqw` z5uskP{k?)^jS-|$rwMNzPbI_3s4A-pS4?N+Q}5ycl25wT7h_RZYJHjkHmTDK%KAD! zwdb?5ZQ4KV+A!a8l~aC2ZzOcUazXIJeFKjy0Oqb9$s>)=)lV_q`yKwL(?mt(108L& zji+|jF1`_zHD8i^#Prw&^q{m^2&RVI+B2O!fM~l_(I?LgJgHwk8{tcwgQlt>Z!!kB z9D`PV5c>e`e6$VzN&R-_@gJUUx+@t0{qqsg$A6$1@&Vu1Sg&4g#*mzf>g4L{Kg;`{ zNt%-fvs=xg24*amX2%+GJ4D5EZ1bUY>yYk74YWcA0QRim-1iQ~9`(K*59F^z?GPg$ zvTp|$zht=9JD@}(8-}wjTzn3N!p_Hfp&X~TqTjNEExG4W`y>kdvIGGMh+4AiN<(@% zYE|JvfQ9!z9mN3P(|Bzi&znboaDd8&b3pLj$^J764{StsbsGqVLK}=eNCK3gN)xYd z>(IA`$plumz~Nq7$NEWC!vJ?BF|g=|vG3uNbGgsNQoDb1s6S2K*!=rn&Ij*YoRWKc z1|vxo{eCtzm5ohjxILPw;!$N;7mXSj9^bo~=EaUZ{aM8Gt0>uvpqEJPJw(&;Jn{hC zxyt#(UahaAPw#FY-Tt}E+0VGdc&gW4s9M|+{=WK$OOtA@K6=^f15YGW4qxhC&~Eh2 ztX&PoPo^=n@qvv1bm;cD1)6GbUMldh^C67_CF`#41rdMEh8ek#)xE-TVG9w*up?aT z=f}pFyHDEaER47sxyITAKf_25#tNL!x3+VLP3C9H@Q zHcs=~Ihm7d;3;qa_~skP>&X zyiEe4(jliQq?c94P8b>IRlL6CaI{L`xaY)y1(7ev>N`Ai$6SHuijHIfeb9~yFJuyf zyR2TeUOSSlo|Y7Mq|qPRq}+e^06=`eb;`|nD9oWn(N|7niMc@6_* zo=xpGmjYr8bEmoD6deAFtEj7SoAMx`G)1?#VLO0?fhL}r}K#frMxrX{b{|ap?o?pH0!J-$r*RIL3J|n@f z?0}U~uwB0@n>fH`F5~#v%ced{n|AKCo79P@q5jQ*A%V~53>kSnFlS6Z5h)OK zf2IeJ=a2{1bbyIgw43AnPZAE6D}X0PR3V2l;xOi8G8B4gHWT4FYkb%{M6tAOq`LOr z@sKa?ina?n=&OWpb5of(Eubs(^0^D+R>@PDHQZOq`n+IiH-Q*AomQx#>B-4kKa}XF z-*BFu~SvZxv|;g;ut^Vu0iF9oN<%X@1uD% zr}~cj&HebG#YrGe@{G^eSBW{?34--)mpYRmt_+w=bd{*O6k6;Dm+*=^3U~!{Jt_Q{ z!E2y)Mq91N*b^iCAwHmQ87^n4j{z2n5EZUG*@K21o;+n^gSX9e_r1$lQ&^dL)s-J< zjGwi1hCLJ}yZf4kT!-h5Uo*V;KpT?Yczid5cEDzHLtrphIs-)Y2kWJSi_UEZ)*q*R zW!IuQ1_f!{X~&J0&7Pi z^6A&9j_ag@k8`_{_AWYTQOQeJ=cWyVyKwnUG9ffgUBl z73OZ`G}!Sffi8QR`0K^5CZ?NmWAp^GrI0HtyEbz5>exYd>ZG2H-SnZ3DKLZNN_+?_ z9?$QI?zPX+g!T=NMaTW!E_UW!yYM%!LQXx2r>0>PuZ6AQXO;y9LI^p>1h}h$FE(g$ zUd2&k-GG#sCgfgdo_$)SS+Lv3)uCEj7@ywiUMCVEL<0|yg!?X)$ zA^z7tI$9b~H}hn8#1K_QP@FP=9ITwijR;gtuaZq4H+82zaW@lNd1x5!bEKs=LxFbl zXkNATN)%=uY_}S=3`r(JfCI!L{3=*jpRq(#Sx-L}6R;q49ouiyY8css%)G>CV$d2& zNP-4cHH+lCggkSCvqqr@z`X?#g+!JM+t5717@s;&%*_7Ufr7=Q*zo=Z7o%n)8x@9P zzM)Qyq{SmN;6}=uv&@_9b2XaYdC6!Dm^Zv?9N#2qI7I5i?Vc z(CG2NHw_>bQXq`J4i-^yReVpCdRpJaEnB;nkJRs<<(=d%&K0b?T_)&tm|KL$zPB_) z(X#XKrsK}^+(X^xUptOjVshU~JZRi}mUV<7Qb zSbg8J=RupERZl-$Z=*`jc8~NrBgiTJ^NkTtylIaxepfFz%Ef9QO0DwT*r#%w#RWy719OyYeluW5Z228Mw~UF*IQdjmU;cN!{RxSS7;isDxxQ#!qAOu0DC zggEQOX_9`lR}vBlE;0mSUW5e0HoUJ>yXy17ep03Suj1T*7`VEusqtd5*olVYl?G#v z?lXNKNQZ`Sdl>WKC4H*VIqhikW1(uJk9@xeIA$#13WQ%=3s&J7%Ae5BO>N+D;pQv# z3Enk$S%z_GD5S1w%x=^EmB-o(4cAxx?9X-0je-?n2VmFugsI4=^5a}OyyiU<6;=`H zGhlYSGTo^y^GDLP+kdx?78l13Y6NJOG1mTm7h7gi_8u>+w&ca5u}DqcOKqoxR0?Nf zXY|Q=KU}<-R#+UlAi$mD9`5n7UNbVNw@X|Sv7KPic*|5(LRc<(VKBzWLhAuE5gFTA zmOrNS;tXHmH*%g|<88d^>bz2|$gowMA&uQX7%L92+3XmJf{Dm!1?;%7H*+ zlV+c~FP$#rUZB6jleCWPb6qtq z?Q^H&mfC2~sv1mjZLL z5S;BoAhjAQo1s;hnU2xvUhhg@nIjUVgOnPvC&CxjV@*+W+U5r=e%O9re^4+*6S40G+j1Qmj zoc+qBPe$MdSterS^5R$FHspwGk(d{D;jeF|Dtg@>{lM`R_&!f->oDmlGl0udhlW09-+!0 zE;AE_G5_mN^7$ivkn~BJ`P$tn9+`NJJQQu;CBJaR z=t>Xa=Sd1ItEebO+eqv83~DSXo>_l&p)zBjUeSDt8c5&pM)UVKmFF93=Njmz1^V_F zk?(Ex`l1oky$bljTcjZZ zty8SzyYOVT%3lTY@QkOATP((>w>gv!$iL>%y}2t*?a<$^n}rQ;hFapJV+)Ln7> zq3*rM9&uj9{cv8wYs%qBxx63U^!Aj6d^wodNOHetD|g<5>V$K>bM55%v+{e7n`E^@ z6yNrB={O$?6HZ^3macJuZLoNag{XnSZmL0_+cH(jxuAJ^mil1zavNT_c9Sw1?6u;} z_V+7mb>q~uRPGbuRKki5+*~XV0YSmZ+aNXo2C>2tNk$ztCMS(N6%blj2QVnEY@6#Cc0OYpK z-s+>34hf~H*QA;;ZiWP{w*hJ^)+y%y%b`u!Z=g*8tG(wJXtTsxUxEp1l&}63C@-iiJeZvY_N6+wzuuG;if*PACL{$ zxwLYP(3lh3aj0iwV><^_jO-On;UkxcufX(QckB0HUu@Su`tn?abZJfAk-rXOg78H2gG?b@@K$$7pCc-K5l^byE*AGDO~OIOWE70#|uq1#(s96bW}H&b^yJjQ#b zd!fBvPq$h-|LsiS39@bqsp7|HAa4Q2-YyFbD&#zmP}V#Jz&8m) z#Zg&`KpkJcuI59t@0l(Z2z`{gqBu?$QEsdaOM znRLSz@j1@0IGUH|p$a%wC%mx*?vc+r~ zJzlnIMq;*@O~LlAUzyG9Ps~Od`GK~_po>B?+z68XL02A*tA{KtMF7Go6^E?V1~dlK zrM$>eLS|9LF5HIZVO7$`YDomAH$BsYD!eAv*${?85}d7jaJg=MgxTd2K8{JoWbZb( zI!GuZzrkh=UBu7`RKht|!rZ6LaRajsfpmia@WpGk!H8afAK!?->aF)=eIw2BJy}oT z8e-0|#|+uHOnQ-Y&AVo-M*mcNe&{dvV_BQ7@MHwBglx)3KvTVgY@~|J!IQ#7#nx(UZ7pZOx}$ho|m zqpdkH*`A`2>-+P*qzG^)mhD<9DOR-#I-*wL-5#5s6=j!M zNA7Bn(-WL?&0m@^{oXFl|Eb`p-eXP!n%15_Jk(z22M4Y`47_gM@t=TLW4W zyG6Gc9{amp?AuEWnTCLfTei;B1LnNRkHe+8LSom?+>6?{W!5}$5lOC!)7!QCCZ@v8 zCap>C-e1{V|1*C#oBbJf&qBRSM5LbZrGT>M?mHv9y2Pc=q@{#QH(g9q+WVQE!*iPP zeD0Zc-{Z@v#{!LkR!+PEn$6lZBCV>)GSoWIdWwCtUqC^f+Qo(V&W0Sne=}>-kf@iQ zj0hY^bSl*qTX{WcT9cOA-<5|;Gh;#yn@>|JszW*r3>lP-juJTb>Vpf;3u zp6I_qZ46mGZS3{T|J~T8I{3HPX8pg|=D*(Sf4$fL`+oX6*yhYV+)KQThJu?l^{|^~ zVUgi$piQ(8Z1}1{PV87`-a=dMjp^L!p{slKkDl{ z#Rs0@VLwjK$i{e7ow-*xGWnaN4b8#%KM}ME4Cy0I2No;^68)GcHJx>DrGlzZ%REyJ yX_yJe0Pq=-!k}gi%@IW9%n9#KZXayw#@5DA#|~ZV2755=3YWd=uS=i!$NvFRxf?qG literal 0 HcmV?d00001 diff --git a/opt_prime/examples/pd/http_matrix/fixtures/quality263-qwen06b-fixtures.json.gz b/opt_prime/examples/pd/http_matrix/fixtures/quality263-qwen06b-fixtures.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a3e39ecb249a7a57af72a97de8375acd9a26d69a GIT binary patch literal 24091 zcmeHvdtB0K-u6B_JKM}wT2pFS>e@j&ct|tNBcIL8oJ4CTR8k}lh?Z!YCq(ewt+XVy zB9C|&v$Q-Wn5cNzhSU&|l<|nbcuoZgP*4zgX%5>owezeq^X|L*JbV9TKYusy!Ee|1 z`d-&{|MajQ{pR)ht&+!PV|Py%eXW$QXj}^$&w<)6`UbfERiBKtvJK)JO;xZq>;)~> zj@aM1RW|)$z@eaz)_33e-EYnOe*3YDN#ygkOJ095bzS|!bvkj$#okmAe2GzmPUsfW z;3jql`ab8%&brLbTsWm@k~Z(j8967RP$$lrFBirpwJAw*cg)`AbImr$5K_$H4T>0; z$0Rrxq)$%~lf$E9!*9~Y{c47)Hh?cnvWJSA zR-z-NVWsSN2VJ&y`!es-_`UYr>1A0BYp0+(eV{U2ZMKl1zMiXyKk)6L2g~ixL9*8-Ax0)w^kK?9bEb zt~c4X8Vvi;=={3T4+F|i;`&A#GU%L6LlUB%#1lzB-U>en4v( zp6D_pZ`_hf#T)rP$J30W<$}0Dm}YBJ0$pZlYm zA0Vt;8s318J$l#4W3gy%IN@Y@kwfTk%fl3{M7oV{RdQHF0I$T50#6)GaZc!|_Jt=Z zmUvLzg#LjlXLY>-#e!71G8yxE#n(Ja@-%d?Afp?jz!K%^V?YK*9wB-;EoJDgF0WLl>)OefrPLuxYrg%` z0&mq2-n!h^S1r_Px>X9&ZY?3XXIfyA!k}uxi+{vC%7B5K-RI5k3S$o=!SmO~O)GjH z^A8`vIGu%k&PRW|+C(lFM+#_dd&%paZug@)SS3gyEXJ!Ax$=jmg{3;^j(dtDuyg%c z34dO_m+l1Ch+YQe71o}laQ0JK_8vI{laq5KtPs2&xOwax=RPvw*)oTD&sA(8X7jCH zLQ_f#JfbgW4?sLUhC+ku44+)=Y$-%2b%KLJa<5TpLpvigkI+XY%MNtWy$fXja~7aJ z+ov38KwZ!g6&$um{O`0v`fK_CV>#k^-f1Z zTSf4psQxT^#Zb#qP%^PL7nY2;?OsZ;SPvpMKc*%%UZB*n987?HiK^|dG<*YBU&+dq z3TNeHs%*V)U=n+Af{6eKoPw6pAb}oPkUGO<_55`H*vs}IhVn6C`C zZeTVYBym%&@mt{_>x1}Pb^O|kWYJ?5hu9qHN^E>omvsx0rKiTUG?umGsgXUWH_5<~ zrrBXPq*D4%V%2Y*|)D@G3btfsB25t?-^0Wyp6OC1asBS)h+WT1Qv{sQPAt5ghupwz<4h={zBr4>z50kRAz?Y*)S4!Svli)sL#rba>!a;hZ*MHN82h zDKu8DAPma?Ae=PSp9}Bb8_55pw{;72ihD!ynB}mQMeB$Up50X*1Gqv>spAb3fXPKi zcgiSy9o7@1_XflhogNSLaCc+Ox&9DYgQ>o_TJi}&=4q-=OhA1MzRw*w^;||YHH2RJ z7)g~MZ`#nhs@y@SmKA*r%XyHD?g{i8ws6$Tr63L#hP94{)p0u)rvY^{awB6oKXwU; z9;`zaN|T4CSGc#9J>x!Jo+?)^v?eY<6BnG6R)H<#a|w<7>E^3ltpQoLvLOAFsPLy3 zSerD)1dhYg68RIk-knq|HN4lev?4SJeI#Ep^P-eheUy100Cmfrh7a6DK*CO06yLE_v9qyd!4st{UY^;a3>?^rjFgrp9YUpSFZxBXh1< zV4$%r(*Bp560Z{C=dEI%wSKW1gNwfo_>G50L@Q8=xfr;mb^1} z?{@eZYt>WUP=s*yP6O1ou2*hrx{Ah76yuxUFJMq|cLuO}L@cA=>#WFjG z`@0IF4?6JmtH-a<$3MT@_VDFPkx}cj7fDMsVdM01peV{++aF^0D#4&ebcaeQC=9Bo zVp=wx=W#wSQDanwN<17I6y0q*;n=>JxPZ%F!C*%kGF@k9LT&h+lt}5-suTkT&C-<^ z?-Ar=Rp}VBEh#b^g!YwAN^^=VJC5PLE3Uq+C>E)@zov2qF#bRF_uW z_axK?n*^O>0T(qZ(>*Ixv??_MX=fCJtKnOs>LTmO+Ck$l1a)0lrNkXj#YxF6UljFU z7^ytWF35g&4Qiz+CW22k4uuYee#5HjeGJ+0F&3@%dqN28m6>!{zVEvwU{+(g#n!7S z`m0oOAt}vt3CUAe(+uA*J`;c2NXt-q#{|iz4^$h{X2_IK*ZHoFV!nd19JCJCF>{3) zqo;_O2=Y;}NF0kzow;ndiHf4ZGmvZFU&2qi9 zFXhFK>hbd)*J$v^4~OIal6niC1#H<|_?N~dyU(L|?z3mw#ebP7v{!%53Dj?ZYp%Vn zRtK^Rt%QFy2se>$@?fvpVmRepW4ldgaVeFX#)e3Q1IIKLIUP?`71gz?pyVKV1m96= zy#Ds?_^YBNia#pJ@>zP~Tio&U^o*=7OUX)8;J4&aT5uvdE(}H_iKzTo69Xsj3b&4L z$s_eiV5HXqa9?tX$uQU(#4HmyLXwK+dz(yym_^%zz}6QV8|m=2Lx?21!fSry7f}dq z6i)m~o-P;IvuUbPf-Dn8+DBn`1X$T`|E{E_;4HefbmfaUzHI6cXIoYW#osEO6~=-e zOn%MS)f8c3Vk;&=VybRp=~J0S@)!?GrOKbZyIOYCtcAWQPcDwNq_5WNNC(ur9THxS zU~jy$&~X1A28i(;tJAw*ppcAD^e_a4_ABCLsGfb?La95Bn4D;fdNnhoaPP*5%Mo9v z`q@ED?@W$O`1bCtZeErx23n(Ivs0=0%W<{Mi6_&NQPigsLKAfZ7Fx=>jgPR0j<|-! z8kSP}CS>N@=d0WrovgaEu;p5UqNNJhKZ zDKPrv_>elG?E19lb@j(aUNur2PsXEITv!4xBH4-n9fbqr()S{{OrwY_GU;RjMq z#b~AMs+6XYfZ{P4?{Tnr$&9?PB&xk38Mqq)M(zE2e61^yyo|)5m)#>WKxeX`ZHDE& zyEdZqN;-c)PW7*qd=z0E-rJPMXy>f!?6gQ{^l>QQOd-!p|JfEe!rdv#ZLZm|T(p#+ zCEVaGyu527dgbJVa6IeUlA~YG!`HCcp6<$Q$9Lc-W(q8xHO76tT;Vslcgr<lV#~ zo-CzA_0vU-Ay^wm5ZZfzSfVGZ5~Q?LHa2#VgI|&JMl&)39iJNd=4a*Ra^G zf<>B?!^%Z}_Fq2-7Oh;a6VrdjQublO${!NUZE}rH1q9UY>OMk;c|SYYd#c+D+!NIs+UxGa5AtCAS8IPp5?YJC8+W7CY}wR}W|W(l$DO$g~t;e~Q@@ zMg*S&!E!YyQS#rYBf)A2yd}~azflE>`T}gDgAp8!_t1A`hP_=6&}ckecuDYS{hbF+ zm(7^NVP&LXbdx{&^0sev4*X37X^5CXVfP){k&Gcts#mwlxGG-Bd1z`0`4;u^XC&`} z-m_;dmw2klyO$nB3Bsuw^&QHj%}hVBc$u&NF*Qb93doF(Z+p+a7FXVRVgDJcc?`#< zXBy5E@ zs9;O_QUzPKIrt@e4oVOpnNM6RY9%DBRO_^+T>)#D6h6IoVf0mwjGg%&t-1o-;Sp51rh%X{C`MzN|kYM^SP0q{voC)O2~II)u(tIG%nWQX`kB?p-Y@R?{(Cgs+^-*2N=4}#BG zXo;)&LDx{2(@imNYN`}L;=ie><>JIuF#xlgh27{mH(A+N?;Q~7@Gi%y5uD)lP?F&Q zg#v(C{`y>`B-ve0F^QWHr~^K#wv!F5gSV2EBBopyy(E;hi1i-%j!1j-NK(b(6{5PH zjnx0$#m(_rxZFpCD#v)MSC-a2g10J;A{WGRxVkc7b--C&S24q!;Ei&-bra%alsF=H z<-(5lDsk^1SQAaf?6^M-VPwD85Jpr%SkQR0i=~8ffd`bYyQm3)cXK%F= z^#Vbg8P%Kg&VxOob;mmm3YzX2Nxu|=Q<6ZSEsVO;89)$5k`pJKpKTf`Oc^7_4t&+e zvD*Y@KBE;TCOY9e)Q27t&Hq1E!mR+|1XHDc72&|FsC?->=-hqb z%=6(Uc0dLa;@g5DZ`1oMIAzm-$bde#6v{&vRJe+leE9ZxuG@9K%}sK5eRuKJcl`YcbM+GP zVsxVp4z5y!H`Pg-e;O32!b_&s*>+VsUF#aYQG=(kcq)H>BC0;u2Wp^#a~IOf_T|Wk z#W9_FF-981CYhoGE^ld_Ls1i$CmP{+?$$_oGzNTojO05hH1HUUF5 zcKP;LWFgL=CH&^Y9+|?MVEt#f5E#hf@k8nFK)QOLz1>U4DA#pU?P#WEwfdL}R-8Bv zXUE1Rd14$B=)t4QFs?G#Cu5 z+KVd+PO1RpEk>UthIpP;+yXid*F}Tj$wi&BEi*yH7G6pb%b}p?;cx{nnG(~#h|C9e zQv`tdR%w(eG3+pJwmghztve5)Rt1Li_;l6Q;b)D{=~a7iced{s7sYw;3<6jPN$BLI z*dtx-MO?&kzj8^K&W+=%R9=5(Up(^Sk@^0du)k9mm`IB~>`=>4%WwX$K2^9dd)r*D z?j56{u+@bQf?M_b_*d2q9V3p-Y&)Cb-J!K&Ct0@+Yn3F7YCf5X#i3lXnj}+m1Gvcr zwx6Xz?9_OVWOX!wc!fH?@HCK|{?kQcOi~q%?(3p4R25AJN`UU-DAryx>B=YL0}+SH zznnbmbrxZ_*ZDb-X289!wx@a=X(3UM2oJh$!K3pw7e1^tw!8dFxaQT`zS4AKoh9-z zHMxS&sEA#$^UP^;_Q@1(J*WSSco7C#p;p^G`=I=rnKbFUnG_59Ra)t1i)fA9Ms-R> zSQ+ne8RdmWNdJ9=6@mcXC$D(-n(qvK8(;u$9DxkprI6?+1@8)x%T8RLL8m*c97RnE)6bgz&{61aez(>JV+}MFK8w78 z)}v(QA6Nf0I}{FZ(^agj4p9Jw%Eg{5&WSP$Yt#~nxUR5(t(@RlwGRKDv#p)ExC?CJ zHoq4l8pc;c-7}f7$wS#AIqA+UdGbVDeb!D_U4iI4fgRhb)3AL&+(w^>iH)@J0alsf zM(1bmdq<5YG}Oln!v>M9C^3(f)iP9ngPYVW^#}+bUON&gEk%w69wBzn;~vQt1&TkG zr^I2dE&YZ@C=+ko)qy>o!{b z!Ow1^+rMk0KL1@C&3n^E8L6r^iukUL+SVROXT+*Ffd;_vm~Th=@s;HoODw}Z$^!XT zAN^j%>wYY{6Y^z2;py*HJaNMhRJ_ojw^Y1bLf!*?FC4m&d4@@T_z1 zn({yQguUmI+;J>=4o`^-@X=MrvL`N*jSDMx&Q~Q39bF)&maq61e_P%2N7fglBFOw>smmv;U_^Xn?8zI9xrrbn-e3%!fsrLLR=LWC!AsvN zUq?!2-Zg}_s2m|(`-C_NldBxhQm@+ky7l%z9b8sdUtL+ra0bY~=EX{(n0RinI-nbh zT-n`+l{?7BV@9BWqw-fpmrdWJrRCpn0(jcX7frk80oN06#YqU050IuX6_ zbx)xKuv>SP&wE-5mjbNi^y<1l|1_!7SFzH)DzAZTmmMl)hdYP0FD67BZeZuzc)37Y!`@sc&OE@5lx%IgFHL+f^*ekvUB zylhtiRNvvRgo;_a_6(pM(&=@6fU0(n1?qc7G*;yYG78ZKzD;BsQ)$V)`oLJ$ICVHX z1K!6OWHeLm(Z@S@z2h{d!7%jm5*R;Y?Zb(ZKQ#@MOaj_K9!?O9Mq$OLSPKUoFrI{} zFt4W32Ew(*9slrM?m(w7wORdcMu=`qjMfObVAx#Z)Tt*A`&Qk_|Kw&?<~ z`b!OpF>PZnY^H*!kN zApPJv9HNSwHga=rc`PWBY-_DkWiXIMy%ZbWYup&o;^ULww9q`U!U*Jq8w}hb6&e}1 z{4gw43v4o|G|420#$tSf^iws(cF$5b4P)EN9?m}eL<1Y=wLLd@9ZW;YvcSgnPn_9} z#ezK6?nbQDz4-reeXzy5XEclN71M|-pE&Lq>uCHT zB%DqvyIzAT^(UDRL%y?nN?qUBJ*Ba3Q*ROMAaLJ=-*7a9x!`npFeaq16y2wm9Kv0A z&9Nbw^jbG@5hAztp*ZXz)*v5PZZq*CJ+_P$sxtOGdk+GGiPXbwgNU@^AJVBHTmK)* z`!L@XQlczk(c^N>%(qI_oG(A9!)k!WbV@WH+PTq?lnjilwVT0BHWr#TE;a`?`sjxH zy0sTC8-R^E^mB~00<56C%sVW#G%JSdpao16F*G$ zk(Zg=#<_)*Ujt=+{2FL-z&b1`tuaDMMyfQGd5+#rqyJ`(71LFi!(4~sJIMIY>GcJs zVFn~JjyZZE5}mUl*3*!EZ^tyPt&e(&d#YRMI;9ATG)Bs){&%jLpLG)U5+_T#O*D_O zo?Yyn_1}NcG;A8C*&A%ea2)xlsM)I`r7Z2hnVRd#8AUf@PQT8T5aZxL-6Mp0FW84` zxA6b6d=dgYGm6*xr~p;|F~L zAyDSSK_ zn#`|5|1HpC36Rfqz?Gt}|E6^{4b#)YtFlV5tLF&T+HPU$_=#X-uOTfO0F9bMQKK5{ zI8Avsoy9bWT-Gi}*B`-%Q#KP58z1DG^Y8OW-(#Du?NUoj&CA$V z1qzuKT&nl1HA>1O=^3VDFiPHNAwwy9`Lq~-&dsx5VGUhzXUr!?O7Kv`2mctkibWn8 znkWB$FVGRE+Oh-qvK8K8 z;TXGbSu(LQ_fGyg8xo&@96m0;*jZcA{K@}FJZz=DW@|}ymA%30`sI%2+G(K8DRG5P zI$Ju+x&xp?7FyWomJgjPRe2)g-leEO1ZTdcv&qC1+6uX;zTd*>`b8veDmuZkb7>px z`&19YNG-aFc|hCitMBH%dm!LS-bY9sz$eI+C7Av$XJ_0ujyonW*OFt$+2by~|JsRk z++Mb;eLg>h-5DJFz2(k$lUys{Sm&7jaPXtjTJkNI=bdo<`d!ER!wc1zwY0gqf=SSD zG#JuV0)lquR(0jhNc8nO%N`wrWJMpgR&Q0-D>=;Ou15Of^4PiMxTuTUrilkfKix7M zD%xNkgCBp(i43w0CQZI^+#_(%(AsmR;g#*|=6@t9YdGGMfHTS*ztxqAt-+hdoT9pA zMq%mse(1&X`tNEOzrq#CV#@n9-1q6aYr4i2L7&Mfs+hq{jBC#xuDd7_#oa-PYxLbi zYoj*bKjn)OFV0aTW%Gx2;B3XhV3hLK>-37s7VCT(h@T_vwi=Ibsn7Y_$r|$!Nd9+6 zYYa`%rq;^CIZ{0J$pqgJ>+(tGX^pm1j7+=@+3O?3mTH%B?1PG?`ur^L$>KIU^@!b9 zCH^%B%52Jxx9!sPNHQ4y6Sz%bNgvBUg8b_Fn16=GKY!}^XS^f#5QY=@vy#jI2K+Q} z4k_{orVi3;EP>B}Q3dF4bRefM-bl-iV`NiIr1C6IQ!XQzQ!y7{&9*&=9Ci`iMYLD{Z ze)7|5e$rU0w-KefkvA8q&9n6KkVuYUn37EkaNaEuV<9b z7sqY)mDh7iz)eJhbZxLt1>v`A4^jT>?@-!yf0KR>-aipt3-WbJdvVon*gMlHZBzJJ zXL4ss31@dZ>2+!^qzN*WbLD458UFRRC=1&e__*^ptDetvbif`Yv;D_WYDk@R<-zBJ zkLfNFjdmpk7k?8qWL@8UOvkY7BhXj+x?Yn)Kh2&pgCFMg-(%|>x?6O>pT~JLSa~zX zb0B1g@1Yn$$l2s+4@=a+!V%2$Og?V;t+)-PY(K>UP(i4wR%h0h=a%FF%Z;j2{Q5VJ zKF-sGWMm~fvbDg6@srFF-NqL{T9Xlhid<53D5M4Ruzo}mL=o4Z+t`*`gD=>i$}$_& z4oz%_@hm$S>L~+N-yZ_8Wcj`2>*y)Og-W5g)hD$6Tl$mhNgfi?7e!G$&{LMq6e`~j zx6{~N7T7R9jSWMa=9px4;InYT;1(B6TExkq?~e&hpK5HI>VDi3|9I1TY@uI#Ngmez zcYUQ-oyCRdORAiQDWF- zESWX`Pd#*i%WBEWQ|5HZ=@aKf5?tG!k6{l}#vZ_FzLajDueonk%IO5je`Ufz(`Hqnxl6BT&6g4~@A-lHww@vWcxJpfdmuF#2HawrxT?@V z=mG+>bbSjfd%(f3XU0wct7pcg;}4v)3)3p-?>`B0V!K{I&XllW7r)^q;KaCHZA#O-VTV~EY`R@sO#Q|d0GI=Rc-E6r z>(ct+F_5m@TD77IY{gtsUA4S|SL!$b2lp@}5`h;>3~Wx^ye5zqJ}Q zv|$^0*`IWm7wAG79NFIQsjj)Ef$bl@8Cso9{1ryecV9wctaNGu_;AkUhjmPVMT$LF z|Kl|NVtH$AFYv#~X|(9!U-z_7oh3S3h(-x+X25nNJAUUh+W+ZYr%~_3Lh!`v^ub19 z0Q(0A%g(Dj>t}<_pVxSA=U2VwKrYkPP~0{f)l_3WOCaR>oHBqwte8f3kw>mtR>E2N zr64FAktFW~60ytKyrrplWaG=~z4+qM3CUxZv`j-Z!AB}~ zBDG|>X3@s!G-{j$cMo>*=pN#vKW_@l{)|A`%l5wv_k!AcUH`FP_pAI{fe!#!Yu^d8 z++m6TCY5S3;6f_Up@*xxWQJNL%DiLWdnowbGc0SoQWGBg{b498!O~y2Z26TdZ9gGa zDZ{6I&+V~R`&}A$asQph`L0{HZ!XA1X2$I-s`mO%& ziGMV$dwSll{d|D@{3|~-Y>(fob)wowR=pjwX33`UpOy& z6~!9mfn}HKYy%p+8}cp7g)uRu0DHZ#K&yeFO9Ec4o;>7y7p*;LaecfNY#2gx&e{Gb1Jp6pZ_z z%!VI}ysg{ce*etKF8bEi&kQ#9Q@>Pw z8w=Q7`biVI<-3~4%Vs}0@ewfh;g+*QSEKxNm@FgQqq?|etCIA}lOXDj6sLsLqV+R! zK`(zMGh5)|SawP)NmX(wY}D2)95n)niyhr~QQC3>7zZP;@d>NT<62tqav?9F@rnIB zwyZ!Wka*fujNzDQA`chpO~U$&(=`|GfwwN78JHB;&5GGt4P~sx8xU2xDq)kt*Sb+V~=f5%wF0{O#Qo7n*j$vRW$iNU_QERLYrOAzty$OYq3s zQKFYUZPLW$Qaym(a}&(s5>?PbU0r&K9m_WaERBt6sdF$9d}$orJx}A8kI8uK-qxomAyW@x7ni;Ia=wkx+Fh{n>SddGA1TVYsL@Z zs_k^H0{h9ozJ#{+PpCAVOwJxaMjqRjd&_gC-o$a4Z{-Dw-txImU)>Df6E(W9dIuKG z&U7JG4L8aphQg`w+{g&e|U%IOF4H&~XIFxn@90oMa$FaP@lZY zcA|KuUs7*c79Vqq9;0m66ch9sk@j1~ZS&=)>2r^-JK0{HBzR{kB6Vf{+ZxOE%Qk6E zHhl!)Y;b%|xtBsL9c2_Q0#Fg1_qb|Vix^^V8bGI{Z!ErvlT?-6eAQJU2~2dcAvM^$ zl1dh-hr(j9Cb2DfJLmH^Vne)q!>Y#nO1=e3(JWapHX#W@JD8{w_`*x^@WG&tIVP-* zW8@jo+E7Rzn8Wc|Ls3n`LF4h-gtquRSznxWrwI%U2otE`cO)18l-l*Pe>ceW==%H@ zL)-gaK)z0?&q)L)#>cdHfBuYDNHFI@|Uxdwp;syPX6_EL)AkKl9Y+V z@2{m6)S=mF&$P=1>}twp?3o)_n!?Ji zSDQuYqX!_3_Uuo@_8}*uXTCgrtKEQgRLCi{d#4Xq(o_RJUImFKGH>A|FE9*xjk&*SW zmPRFh$ObVbRyhu6l~!{vR@D_C`U?wyS(gP8ERKa=T&}l!Ue`gH$@jzSRJ&&l#S?u} z6;l{XXtjG?IGicGfKQzZd9FDZG3z-#xsUt|rl@bW4Gr6w_74kIYq~Kpg2zouu7DaP z@hxq)&%>SPfc@c`9p$!;nnboltF>`X`2QsEstD*%>qrF5m(sXX`J7j*W@W-r%puol zyqR};sg&q|5>TY!L37K76#vfos*-W`;^<7Bia{-mhQ_u>k|J#p9^-tTTVlM!$n&r& z;wWb*1R!Bvu;*k$PzO;mD{o^G@d~eanqssb^pGYRrbV!}dR}?)WX9DOnwl~@d$}Uz zmLX-E3xjoMqlX2eUG^2JF2yIY(PzfirYQY;+@0954AeH7 zy>;_g=p9In+s>kmX4aMaL{TO~U+r;jy9X4su6HT=biM^OIkK>mtN-tSG;98hE3Mk9 zCmnuiSx0^7GG7tgcz5d^oMUM3`FtbQ<7Qo2Uw<1|`gP`7JO+Qi1TT`lb|t}Q5HXCzUlx>QN6z6a|aHD=A*=Iv8x)QS2MD*08? z6BM-Apvin_9_XN45n{2#zPK0%_OPzh3B*t%fW562dz#8JohL;P)qreMahv%DNA%q+ z@wPc6)000B1J(Qi(u!x$B3n=y=rap`5f*=cKKz#O+*c8vi9V+p-W!NACXdM9K$K!k zu0#s~%IGJ;P~z%BXk#9d1kD@aR4_Uw|5RtM>n7O1;>DpGyZZ5y7kE}u;&MEifVRb_ zd9p%%+#Tz@(m!w2Hgz6)U^>^qXZu|qzqIvziJQ(2Y;cJWa{1$1?$LtUMKh1z38 z2Xf zXh)M4K+)57C3z-&-gjkXjz?A3^RbihnM(BHD{8ndTq}L`lJJ$V?Mk=_MuLNUeZ|j) zdvhq>uDNh=2&LDOc-Wf(*sl3`OzaUIL6!Ot^%B4~&)!y~62MUZp;W(+QYk;fTu0^` z8eYBkF)|}<&ipPHa|HylGHU}N@Ept&Ew_@CiOUcYthb`PuBt`h(zIM};4!ng+BPf} z;FLx1KO?AA5Y>&y*)7BY4z!Ck(Nx!2-rlYe}R$Ae&8Jhl7bV1&x6oUzn zO%?5Q2|N?{>BCurl9n<=brNvo&o~cJ!*iM)dZ&{X+1^6~Kf5R<`^BFsGIl(NMd;!e zLe9?Edmvp4`#5tIWhNTQ2KKdXnwfM?WuF>4de ztw>E#*=cF1X@ZH0!W5w*Dk*tDL`LzDA`+r_K;WZhX3ey6+SAPX=K9w9_U|vRm-ppG zu7~@%@B6ubF95Up53kmrR8u@2Mtl7Bw<`JaW+#4ASI=qjF8@ER=Xf_jbMW~Wt%dsv z>&p(a3r@wdy^A!jKHmDtmb_gb{fFOw{pq7)CXp>)yJvm71`FDqyR1Q-?qmY@0&QwC zSk|b;7dr|j<0n1wrPI3i^}D4F^Nl-m20Uam#)!xMMN`3pZZv1DkSnFP8Y4{iEF&lQd+BOm+G7G+a;2-9ix!9D zv>`tsq)(pAB{puTnS3FK=##rY~RHYN5N!VZvuSSLt~+0}TXKrBsE^o6C*_cjZA ztWqR?stKY6jn{3a#iCihg2`3!obk&kF>S%m>0G*2|^-A^5nA1m;-b8pypwM6Jd z@X8xFUO)243Pvq3E)gS6Gx9~#qzny)pt+X(1f)jZaY@g8&!!>NN#I%pcyHkB1Z-5vGPfu$i5;4e9BotWSM#^m{K} z%otOjK1*Hz)=0)A`p9~NbyX(1amu!*eT}Cj18mpcP%R&`npAVBPJt(9>OtbnoULYM z4qavu?|U*Ep8jX2oEKRZb69hU-KNrXNc_)KYwVPNts@TN8b{sdDiABJTEC z!Ti}4B6G0$)Q(CFON!vDlqV05?odsI?@8dt7Vbl0>1&V|a_%BjIvV7*VN$jj3TqZa zldymwWUqM`r#Fxr>y(?NI2o=Qfj(0BWC2d#Zj0Ba5L!y@qr7n%p%%|mz+6p;WLOMs zX3jm^i@3_04Oo~8_mW&TUE&F+Dgk%}e-(bNQP@dK2?i+?Ak^Zw9+9#X7l2|EJ0jKq z9fcuhi5eZKSk-7rhTYOj!E9#pbxgDUk|1UP20AEzIkae16MUBlQw>ikFUGNnd{iEA zX;DE;LEEcXM>KR56YPjLZm5<300AiTWup~%Prb>vLoX;IP|WdpfP8Y$+|cahgPUGGU~<`yF* zfidpY$mR77vkNtRtwM!Gphr)3;zea4%L%$UdJ&R`uRcl}+{577x#skajtWT_33Ltk zlVOj+JIKVRi-XKUXQ{cgk{kYvzX@#jU~u7obA1t1mY^~p0#zEuSp5z z+#EGNmrD}~lX6F$x?||oA*H1%H)XVzcR+HbYzL8{dW2~!bHQvg6qmEq4}FLxZ6sxZJKa zfrCeJ0^g0YMe}+lrX~D2XkGpEd7HLkZuns+F3`zVgbVFcUU=BA%;^bjA)B?=@atxv zJXTkHPp0=Nc;!wetsyrHHnL-XVz=h?4CN?GwXg`YpLYsaxQb~J`Hx>;P`^aMw0yf` z=7RSjS5=kf_3WvN`(=&ulQKZPWG0)MTuy=ZUdudp6n!f3KHd^hxmp_P5e`ET&&oA= zd|QHumN<(*pPMP%ar?GnBEqJfSgc!W!nf2Yn;4QwD8k&AiFX)`sL@Ax8^Gfqx!qdYm+LQ;-rpqHD@sb&SHaqDtKCYrClP4#WX zeHH;pjA-ww`CKmg`dwc!?$C0)Gk5r0#5yQBd09WnWpa|zQjgkBsQ8#nzEuO8U4+Xk zzdCzs9Xm8zC;9d?n=)ANHG$9j?9|xkvt-%%gg|Hq3kNL2|?kN`=YDy{u| z@4fasnbgN&j9+!Ug=r$3Y4m*9Z+YnJ1-_o>f?UzB0NhB!Fv7cHOUmIvNf!CCiDxBT z@j`a zcOkcQwzQ8^@|-K-uv_ifTiS@Z$Y>0r#!GVC0=Wqj~-7T9Zp&J^k(-3YSpL84|OEqNj>Es+@W-ZB~Sj~ zIyg~?%<`6bqFzegdr@~SHy(4yOG z8t|Dhf`)Imh$+>K16tC133R8RoeRO!vvZqgFW#JNw6MR3F9 z@X_1sfvThS*bi&LzSq+XXpZ#*d+`Ojhb(_Fgv9*2&OJ} zTfS^EE2z+9L`FoKPTc4nOjN`Z@+h`#O(PwNzs%L<9stK4DdE0DK8`jn);k>K8N=8qYuJOop15m*&RRd|e7JoguQGVq)FKM+eVr z%APXWH_h+6R54PizI+BhY*#LKh{gg=mBVF&j{btw7kNDi zDXycEMU&xeHN(A!T7(NZErM~=HIkGkg{NMjW*R=HIL#_Utv~ESFGowhkwR&$$Mru> zKz&|Se}en+zp4@{$8CTl0>^W4L>Ox_cEGdzM6(vI$i%3O@hISns4#wvlZ}XJ!dy~5 zobI<_mJs#w*R!EkOkwffNg!@6tdo_CdO@ZbE!9j;?ic4WkPC`t#eBhZU{6@GccUk( zxyX}t5bynwTa9Vkn4jMxp80vX<8ZVPM~i!{ZF#fj$b0b*w5KbQIR|ISX}D#-qV9O) zIg4ps3l03Ow^yHLy7jpgiqucF2)eHrn(3L0vMmm|AvQqPB~7LI0=-Rdj1G_ZbnU8a zTwIz9uuh7-mBz?ljH^ycJo*Z8ACE+geo1JbdnYQ!eyc{s_E)IruSUh0##m3Hi{ZEL zjadS;NdKb<*{^_83ALXUwi)7j_y{&I6n@q9XoB@MH6XKd{Ifc8$%%N>!k@Da(7Xj}gVP;XcRvf)_4^fAYB%&G#`PFJ7( z$UZ6iYIk&hvgTD@0^|8mU1s}Ws{xF9VP7-HFWP3XPdge41y}6C*4wr*Lp@nt=9OG{ zmMbJDh;GKSf9&X!<;X1t`*Z|3Zfx~!kr!5IFroZn$ZB>qZp9}0mx_SUPOrA=8rRH5Z8lw8fxR*yf3gj2M6-W+cdqrD=ub@)#56E$S%F$>cIDqBBi8 zIy_0=OOm2uGp4!OgN%Wv^KoKc^VBr@*N`V2YWhX!W)uA5!n1;pEFB8GEf)_roYHZb zJT;Z1G%Ps$DT`ZsEo)*h;IBT`?+uCz3ow7w;L{Iq4RuqFuF-Pw0&CrMb^FD{8;}C&F(hQ?l?ByI^xWqHKg)fIb=aVNa2^c;7G z5wrL>k#llA5m=cE?GFwEfid+O(J)9LWY)KT9oVfHV!2mdYMi21X)r!C518jC)MrCU z+hXPfqwz1cST!Ny_j`ThMyL)k8G~`#Kb~Ly?>N!YnAzag>}Hyul?xZa4K_+4m-$+* zS0JQER5Rn~{q>XZ*>cx0zFjYzCC{{@{zF?u`5i>8w=UKYS#bpq{E^fBO z1*`T-Y!hBia1<^bC}~;Px4#I$BqpzT-Tb=|jP?q_h-w5QN(OgoC6Kn0Ljmu-RNNK~ zq{m%J;wMm@t~lAna>L=p&BGrUGfO-Cy*N*?ZZ}7L4%#2@d{}yRn6ssnaKDvhztMEi zJ3m`~`c$H9I=8+bw`#|^+_*)lxH3e#u!lVRa;Ex9hr{XOKvCL%X1woGiN?mkvQf}e zh^vz~YBQkxbBwvY4ixuuU6h6Wv5PX)U39$tyDlnE$!Z=aUj|ujq3S#MG_wxrg3!>* zpiNuST4#>04O?zAe~p@y`Km{Tuf@k5-_c#tOLm~5AH+xRg`bEIcFvpPgAEUH$cjzz z?(gMlI`Xx3W58Kqun(PFiyd|9&COGsvV%C!AjD_E(WhslZQC_4w+r6HP4ZkKu-@XT zl#@BtD*Kw_)mB!6`w)RX+JH#9ur>TDQ{Y>LO9~^65;I>ryeTMB#B~_F+QT&bK@+d2 zcxdAaOa9~?@)G1V=#h_T9)Zp%c3u+szZ(-BBY^|Q*Xw5(t-^*?@Y4sb3KxfiBFVPa zIu-i8*^KkC(OrA%BPd>8jkO=d4vG;yN%Af&9Bv^?+{n#M`r?o8U4;PEIjduriSJu1 zug+I>qY!2H2>WevjWMZuxqXI*8VW;AxP?nomGl3oMmL^$Ta9Xb8gKc%`CNw(v)FV? zIcuE`1Sp2eD;;ye}^6$=X2fxVRW0kn+=w>Gioc0f%|Oz-lZ>Bm*VT(a1)DV$paIT=T|%_-7$ zb;nWZMu>xpL3rWt@<@J*~Y@feTlWoPHZtOJIU4+i94)0(I-H*O zz10oQ)Nzg8TEC+`w^~#Twpg@S`7zIxvhJ_9IbmXQ}4^>aOSmul){YwqvDN-d;&o zei^#37NC1I}Wy+H&eF1;|t??j^)K9jW;;ginG$Z}Iq(o|;(Ah;5!ygo_y2Ph(J zrz-(@Nej>FG~Y%pdB@X2d|82FNi%pJ@Dy~^PA+O@6yf=xgoSabDslGPPP{$1Lsu!9 zdR!7b53-iGSJr%Gl`M&BJeq~UCLGIdkc|m@v7@s*UrYViF+rzHqGzo;y-UsO$iQ3h zd0zMtMAKYi#DO|qe%^XFtdJQ{OSYNV!aE2vsqh(CXDg1iVo{PN+MB?MTC%|hO^E*Y z6@SgmD$5zi0@qL=?2v1Ud4b}uWvOwfuNVHnB$P_Gx>-s_HV4*ere~OLcZGm~TnC6H zWRVj*On)`>2scb+#*p}?m`09)y=pquG6P?AJT|)o9QBvn{d>=eKX`AFxg2f<3MZJV zwp9=eCuhnQydWNTBoi(D4Ltu$q_+a*ly-^x;+z8;S@)dF?5kQu!(p7x&B1K#V15ier3+!pDF z%jRZ!&JNg}-Lit$@=A7G=y}_@efm@>MSmfusa=|h80RblzDsJ9yibc3SMRfvNoO?P zIqs7nof9%5m!{3Mu(nc3hH4LFn!`gN*QA=aQ}dJ zeiR0gXR7IDI{i)lJQ)|wr_45u=~G4%#5F z|Kg}MJYN5XIg#^JZBA$;pmKjOCropGYEB${!<@L`#;(v!E`K0|u6BayxQ6m)9dDfo zj_#R*a$U$~BPmJfzUQ16q{n#ohDidyK=6;G>IX*`_RAdInRYkTB3ki+s<=n#i~4SQ z9$n0R8=p3Als~D$Yp*+Ig$%mi}#pB1r@8Y&ua%VeOJ+Rj2U$%G6>~#W?oq!RCxXJfU9zMdiC-LJzK90YpNq;9t zmkAl3fv@B!<9j*EP|MMAPHX9E$n%JuYF>8B38?25Q)iXo;<;tVk;|a46?-T{-9sDH zJ@juY_R!fkduTC{hFXpE35H@V0s;L%z|zUPB3a@H*KKUbw>vm+&>Z<9xxA$$4BECt z$cbK@eTaKU7yYiNu7p{E{-KLr{l_l4gzixCRg}K--<6a_@x%%xB}Y8fwM&eXvH2=N zwnoLSVD!RlC(QnsQu*@Ya^?|`{M)J6c^GP`uSf&b0Yff-+WphH{Ot>-D^y$l6(^9J zoqW!;V;XcBeJx(b&A=W3QPHxOtZ~)ii^oNvq})8aWiIEU3u_u3DZ|0SKKNxqAQq{v zNb|hlI4#jf6NMm zjPI*}4>OEfBTcnkVm2gsuhd76$B^s_%G01oS5cZ`z}C1Qi0Ux0qX5Ic3l9u_^!^3R zN1z4+V9L{8fsp5;ZW2am^-%%)`XeN{*-%>Fkf?~+XMg9}CcTPLaBi=J(z0*zq z!qq1+KO1_qhz`_<`1F#@ugb90rt}bfhpua~e*b5$rn~LK`4-6MEyI49o8P}={}ry_ zo2h?R9rpj0ONrhI3NQTXXD8$}l7}aQCD)I9fQhek8PhMjvsBOYj3zSmt2rY%ep8ms zZ#WV}ovM}Q<7IV!=Ten1L;+4XI#Ec?f~O~Axio!ikAhi7B#))ZTk_fOZIWSSY9Q!L z#2RM_uC--hs&J+;#~JIhzl$QyeW~|$kNYPak%gx#)S z3X*)19jLX>;hx~o=G&<@Pd_LF+PCfS=nlwEkSO~%6oh*1oTsDAIKb=1&??2;o zm1nz|oY^@&X%j|U(=^?9b@JsvYf$(owvvxr`Zj>_#>D6#JQ(5~8-3ap>~o1W`4FF# zLX7taV$&#Rag<}Y3U&0t?z#71M{W%+T5qpVWOTn$|7T}L$12Tlx9hWZ`;)9usFMvb zfA`Wk>)#!`kp6JVsk5=3JF|d`+miN;O`HvH1LRcOTy+PH*`=3L> zXSL$|x4QIV0_qOdWk)g2g;P*=&=Y)}ApgijwawJcX}7)n2PYDoxS*cs^_yXmA&}FD%6>A@3wP?7YA4{(E1c!l`}!kS||liDVlHHg^IWyf)vy zJP&=o?{@hlP?ib4P~|@EdH#8-Wn0&dG?ybiV^2jS<@{j~DRwO+}A>&-0KmK~8RTc|6Myg9FTkUS)Z#Mb9G zhIZLpg0?J)n7+9&NL~-I;d8P{8HQ6m$%L{qp$B;OqmEDu$UPwe@ek$APzM~i z;Ik&gg$B@0AJX4*knmNan#7vOTJrWF!85TXVg{4u`Z}(bFx!oYNxaYq^q%d#(cXDJ zGNEth5hm6iS|ML7^yD8eW{yh6MgxGk;Wy}!+Bbd@kwsM19VoBVHGVJp^oviX_^dn! zeLHyjo=2uUm6$kmZNnoQUizL~_1+pE1R&uTA)A-=E%=511-4&;UGF|eX>}EEmGi3x z#qaM${oYro0E|~abSf|~${d>2)D+^BckY!7?{+6UuZjRK^3{6|O&n3js?!s0xB5GN z^5E<*iiE91$pYR%$pU&T=QcdKR?MzH+Xc4G=bgLvqs-?q8RTF0$8y;*eg#^tXadW!=y*nb?A}r7cZ9c6iSp_AfhqivR9N z+W!j2|KEh8-r&9Tn9R-K#v@m$h*vwbBr!Q|>)(xF|Dvxd(c3}c!(V|14;_z=3V`yO z?racQk3?&i7k;Id(a%|g%UcW6Ug!1UpB`Qvop(>2)=!E1GP5}adZ;XSDl%LbUt@eI z{l&Ic;&*dG>QCo{CzDGrS0$AAk@oj{|FHMt)Q|Bi7eHQ{=1NRW;6&EB}`yM)d2?^@~$3{f(h>F zj{7}Lnh=z}y}N8-U3)EKljpiGe*Jb9`nMnHi#;HFF+?Ncl}Z3L@%&zEIjtl^Idg1g zJ_T^eO;<5>k;vzv7bpBP?3d%@eT(63-)$FoSJ3enQ?H*rkrGq&BKGAE5f5tA{1gdb zkby#}riSn=m9+EX5%iL|b;1IPxC&puQ;kenQTzY=Kg5kj;%c+Copr@CUlMq+>P^er zdevL8f84Y@G(GwKu4NnKL;_utABe9gdf4&5OH7YFw6HVz!6 z_abw?Huq?gZm@C)G2K#DSX%V?8RKE!z>q?nn~;HUw(%eodnC=+#1;D0?@bMUkNc^@ zM_-NJLNxTs96g$tkWW%s_TLcH5~wK?yb*v>$5&WiyxL5$z5QK^OPf|98&8k4PWvj&`#^~mZ^|Durh&X29M%E4Q8esiq^vR$MX z5Pg&xS7+$(iI{lL-vQcZyW8EA2s|&(n3tW@(ABFbM6N4yjl7c={@D?BGu~62WUejltRdJ;>)t znFFDh1y(-t5&#Td|6N|S-i^$Ww9m~%b;6EY9-%QrzF6(OE=vJ*)8iOKl4*`fb~7#; zD+!?-+ro@E8pNdcwQG{^2BaLXr;l|!q{Khmu#(^Xn~%w@no*JFJ9%(RxaI3dt$0;n z5q7q@eBB$y9E5U~G`({E`nQh{ru!ZbKPW;BC=i2^52@t&RjV*Rre+x`en(^V(Mavp z+En4`|A^-0=zYrPI2ecQTVr$eZyxS3-F_{ltpz60MrHR~3syP&_34*eM(CDXjoY@u z;efUFZNB?#)++y2AdBsrL`FaV?`>_{do&bFe}x1bS{7vISoBWP5&#tRf%0+uTFjmS zZ*VhNNMnbk5Q8UZoaw8hh|!q-@+R)k33R;uz-p^r-`d!i{tQL6M}}2U@8HKSs-Uvb zn3xUr@%C%=T&ql&=S-W!u&{N0y~PKhNfxbTWh|%6X5(#JoSe1x#kkSx%W-#ZY+^=1 zH*-h+v-E`AJF6t+kyBR-+b=)+Um*|i-k*^lfx`Y-YNKD|ZqBGEJfU5c<`n5sV32-m z;ArR0BaM-mXxvr8Erbz!Rc7?v@PAWL3QL1kPdup!!JfhnKnJ7t?9n%9E92<8$VuF;y+Oi_5!>$`Huvdz$S#Sn2g9I1&UeqO?GGJ>XNikR&$~qSMA~ zDQ~{oD}Y!s#KO;k5+Yg)`RlXiuya8#nmtd0KspuUz1+T0V)o7Z{DOx1-TAx$-C|Gn zQovRcIWRi`c4GZ}M>5^q*pviY9)3)()P;osx#b``oJsJYNYj91C6q=WH`FxKCh~o? z(tAso4+o9i{Hb+-G2t9TCNk5htnu}7adawE;>K4Mp4^n3rA?D|7Orx=?&C# zOze>i6N-unQY;L8R=dzGFrdoymnEH=i?%MPmvp+RZi8aDA8ShO;3H+2I=5cR(j0;( zrq(XZTn}C$>iFAZATle>!{+u-JSaNBeWjYIrYBbZFgCTDU(iQB_dY!wrW z=RY0rN*NmDgo0%3vvz~Gyc3Lo3fzM%F6Xm_7mH?>-0Ahkm5tgYFv{YxH?AXKu6uW7 zg}o+3ml)u_X9rL}%q~km?6PY$}-UDET^92yfG#inH8VSy|)xcr=4hk6_+`x`lZs$+2Dh^uxiGWH@~dn;4JXx z_a=*vCPBD9>u`RnPIyf04GPS%Pj4`)w{wMDP$46x46RD?sca z*o8VrPdriAv2#^I#&G0*>xxi<3B^}5IA3?A(zzx9%Tm*|;bFK%LB0#Ara8%{n|RTT ztI+anW1?P+&0S+@k}Dl7Dg}itTs@O)9$_k5s@9>;AmG7fzf&F3zQA^~r^Y6welqjr zMWQm;^aOO4J46@^cV$aTXw@EGrWK?rX0WN=X{sFucB3Nol9NH?bH51AT8H4-p3|;~lJ>I=I@7ZyTS}YJMOx-raL&8gOq9^2+aFb^Y%5P{lTx_7AY%e_PqV!WAmjh-Mh! zVY$?>4hM8e-KR&lIT~lfH`>2z9rdxL;+`%P0jf3t8uecCRS$r!0EX4EevKT3e*=u`G{ zror~AWeEmD?2e1E=k@E(BO$EyPmGP^vz|2PSW5YPQfe_DNDG>J5&mN}x z2!cFg(pof{*ivR66s^TfM47i0B!kdBsww&&49eE4hqWY<1=OM&YX_T5@Qr7j6|$W< zE(VsQl!=~d`)y4k1bwt78%t{>`tDoDWaa}Xa)4^F5v}mK@MsQBRL;Wd^Abp~yn(@T zR`cj*eYzSYvB_Zveefop*p1%ME))Qi|sEXK{4NWXFcohg{{bTddtmkJ9+ z-_U}%SrN6!ZD4Dm$n$uOW{u>D!~@B5aq*zpp+*fpbz6_rC7Hfyx}sRXn;V*_Q4^@y zA$V+4Bq`E1%vB&dUjniNATHsAW&^giqGpuGgJvz_x$v(`(8TUYTiewDJ3#V=GWYD! zN7;fJM&+NY&ardKDceW8yT=P#{j=6$b`+;=RX-}iy9fNmvDCo&^3<@%zd=eACQLT1 ze@Ey@XjRAY-`%wN9d5WRK1e-w5vhYcWtV22TY1_T{dGNzC+e?bVjP*( zv1Vk?h!n%Y+|0Oo&6d$FsNf`wf@L&#)K8#ph_Gx#V9(W$ybfyQOhs?>a{#6!Sa(f zDi>a0hVuj8iS|^5hEL>DF({v17+u6ig?oWJj^R`v4yz8hVtL+*=>~LFuSOa z6C`YaH!$v&^2IcrMP2WLTKIOjGaRoJm!-I%qzAo#OEfyISyO{jx4?0U`Rbk6rkA&N zo;Ebaxt>DgahSYTQ8hG;M9WvsCf0s}sGyw}PgMitp^tRK+bSA27bx3ZnWCw`Ce9{q zbx<1-S-n2#Tz%^HrxoB}!qVBBZfG target: + raise ValueError('base prompt exceeds context target') + while low < high: + mid = (low + high + 1) // 2 + if len(render(mid)[1]) <= target: + low = mid + else: + high = mid - 1 + prompt, ids = render(low) + digest = hashlib.sha256(json.dumps(ids, separators=(',', ':')).encode()).hexdigest() + result.append(dict(case=f'{task}-{variant}-{target}', task=task, target_tokens=target, + expected=expected, prompt=prompt, ids=ids, input_sha256=digest)) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + export = sub.add_parser('fixtures') + export.add_argument('--model', required=True) + export.add_argument('--output', type=Path, required=True) + run = sub.add_parser('run') + run.add_argument('--fixtures', type=Path, required=True) + run.add_argument('--endpoint', required=True) + run.add_argument('--route', required=True) + run.add_argument('--output', type=Path, required=True) + run.add_argument('--model', help='request model name (default: the name frozen in the fixture); ' + 'input IDs and hashes are unchanged') + args = parser.parse_args() + if args.command == 'fixtures': + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) + data = dict(model=args.model, seed=26320261024, max_tokens=128, + thinking=False, cases=fixtures(tokenizer)) + args.output.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') + return + opener = gzip.open if args.fixtures.suffix == '.gz' else open + with opener(args.fixtures, 'rt', encoding='utf-8') as source: + data = json.load(source) + model = args.model or data['model'] + # Exclusive creation prevents silently replacing a prior run after seeing answers. + with args.output.open('x') as output: + for case in data['cases']: + body = dict(model=model, prompt=case['ids'], temperature=0, + max_tokens=data['max_tokens'], ignore_eos=False, stream=False) + request = urllib.request.Request(args.endpoint.rstrip('/') + '/v1/completions', + data=json.dumps(body).encode(), headers={'Content-Type': 'application/json', + 'x-request-id': 'quality263-' + case['case']}) + start = time.perf_counter() + row = dict(case=case['case'], route=args.route, input_sha256=case['input_sha256'], + input_tokens=len(case['ids']), expected=case['expected']) + try: + with urllib.request.urlopen(request, timeout=600) as response: + payload = json.load(response) + choice = payload['choices'][0] + row.update(text=choice['text'], tokens=choice['token_ids'], + finish_reason=choice['finish_reason'], response=payload, + **score(choice['text'], case['expected'])) + except Exception as error: + row.update(error=repr(error), answer_correct=False, format_correct=False) + output.write(json.dumps(row, ensure_ascii=False) + '\n') + output.flush() + raise + row['elapsed_seconds'] = time.perf_counter() - start + output.write(json.dumps(row, ensure_ascii=False) + '\n') + output.flush() + print(case['case'], row['answer_correct'], row['format_correct'], flush=True) + + +if __name__ == '__main__': + main() diff --git a/opt_prime/examples/pd/http_matrix/validate.py b/opt_prime/examples/pd/http_matrix/validate.py new file mode 100644 index 00000000..ab19bcc4 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/validate.py @@ -0,0 +1,310 @@ +"""Actual HTTP validation through serve_batched.build; one model/configuration per process.""" + +import argparse +import importlib.util +import itertools +import json +import sys +import threading +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace + +import torch +from transformers import AutoTokenizer + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--model", required=True) + p.add_argument("--output", required=True) + p.add_argument("--http-module", help="trusted local HTTP module candidate for isolated validation") + p.add_argument("--device", default="cuda:0") + p.add_argument("--pd-prefill-device", default=None) + p.add_argument("--pd-prefill-no-prefix-cache", action="store_true") + p.add_argument("--pd-prefill-chunk", type=int, default=2048) + p.add_argument("--prefix-cache", action="store_true") + p.add_argument("--backend", choices=["torch", "atom"], default="torch") + p.add_argument("--atom-kv-blocks", type=int, default=3) + p.add_argument("--host-kv-gib", type=float, default=0) + p.add_argument("--dtype", choices=["bfloat16", "float32"], default="bfloat16") + a = p.parse_args() + if a.http_module: + spec = importlib.util.spec_from_file_location("opt_prime.accel.serve.llmd", a.http_module) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + # Load consumers after the candidate so StageError identity is shared across the server and owner. + from opt_prime.accel.serve.llmd import serve_stage + from serve_batched import build + if a.backend == "atom" and a.dtype != "bfloat16": + p.error("ATOM precision is determined by the compiled artifact") + if a.host_kv_gib < 0 or (a.host_kv_gib and not a.prefix_cache): + p.error("host KV requires --prefix-cache and a nonnegative capacity") + devices = [] + if a.backend == "atom": + nodes = list(Path("/dev").glob("rbln[0-9]*")) + assert len(nodes) == 1, "requires one isolated allocated ATOM" + devices = [int(nodes[0].name[4:])] + torch.set_num_threads(4) + args = SimpleNamespace( + model=a.model, + backend=a.backend, + device=a.device, + dtype=a.dtype, + slots=4 if a.backend == "torch" else 1, + devices=devices, + atom_kv_blocks=a.atom_kv_blocks, + pd_remote_only=False, + max_context=9216, + max_output=256, + kv="paged", + page_size=64, + chunk=256, + prefix_cache=a.prefix_cache, + kv_events=None, + kv_host_memory_gb=a.host_kv_gib, + reload_bandwidth=None, + prefill_rate=None, + remote_kv=None, + pd_prefill_device=a.pd_prefill_device, + pd_prefill_no_prefix_cache=a.pd_prefill_no_prefix_cache, + pd_prefill_chunk=a.pd_prefill_chunk, + pd_prefill_python=None, + ) + result = { + "model_name": Path(a.model).name, + "prefix_cache": a.prefix_cache, + "host_kv_gib": a.host_kv_gib, + "passed": False, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": torch.__version__, + "dtype": a.dtype if a.backend == "torch" else "compiled artifact", + "backend": a.backend, + "decode_device": a.device, + "pd_prefill_device": a.pd_prefill_device, + "producer_prefix_cache": None, + "slots": args.slots, + "atom_kv_blocks": a.atom_kv_blocks if a.backend == "atom" else None, + "device": torch.cuda.get_device_name(torch.device(a.device)) + if a.backend == "torch" + else "allocated ATOM", + "cases": [], + } + output = Path(a.output) + output.parent.mkdir(parents=True, exist_ok=True) + try: + owner = build(args) + if a.pd_prefill_device: + producer = owner.service.batcher.worker.engine + result["producer_prefix_cache"] = producer.kv.prefix_cache + result["producer_chunk"] = producer.chunk + except Exception as exc: + result["startup_error"] = repr(exc) + output.write_text(json.dumps(result, indent=2) + "\n") + raise + tok = AutoTokenizer.from_pretrained(a.model, local_files_only=True) + head = [tok.bos_token_id] if tok.bos_token_id is not None else [] + fill = tok.encode( + "This is background context. The red box contains apples. The blue box contains books.\n", + add_special_tokens=False, + ) + tail = tok.encode( + "\nQuestion: What is the capital of France?\nAnswer:", add_special_tokens=False + ) + prompts = {} + for n in (128, 2048, 8192): + left = n - len(head) - len(tail) + assert left >= 0 and fill + prompts[n] = head + (fill * ((left + len(fill) - 1) // len(fill)))[:left] + tail + assert len(prompts[n]) == n + server, server_thread = serve_stage(owner, "decode", 0) + url = f"http://127.0.0.1:{server.server_port}" + serial = itertools.count() + + def request(path, body=None, rid=None): + req = urllib.request.Request( + url + path, + data=None if body is None else json.dumps(body).encode(), + headers={ + "Content-Type": "application/json", + "x-request-id": rid or f"http211-{next(serial)}", + }, + ) + return urllib.request.urlopen(req, timeout=180) + + def body(n, count=32, stream=False): + return { + "model": a.model, + "prompt": prompts[n], + "max_tokens": count, + "temperature": 0, + "ignore_eos": True, + "stream": stream, + } + + def complete(n): + start = time.perf_counter() + with request("/v1/completions", body(n)) as response: + assert response.status == 200 + data = json.load(response) + tokens = data["choices"][0]["token_ids"] + assert len(tokens) == 32 and data["usage"]["completion_tokens"] == 32 + return { + "input_tokens": n, + "tokens": tokens, + "e2e_s": time.perf_counter() - start, + } + + def drained(rid, timeout=30): + end = time.monotonic() + timeout + while rid in owner.service.batcher.by_id: + if time.monotonic() > end: + raise TimeoutError(f"undrained request {rid}") + time.sleep(0.01) + + def client(): + try: + end = time.monotonic() + 30 + while not owner.started: + if time.monotonic() > end: + raise TimeoutError("owner startup") + time.sleep(0.01) + with request("/ready") as response: + assert response.status == 200 + baseline = {n: complete(n) for n in prompts} + result["baseline"] = baseline + for concurrency in (4, 8): + lengths = [(128, 2048, 8192)[i % 3] for i in range(concurrency)] + start = time.perf_counter() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + rows = list(pool.map(complete, lengths)) + for row in rows: + row["baseline_equal"] = ( + row["tokens"] == baseline[row["input_tokens"]]["tokens"] + ) + result["cases"].append( + { + "concurrency": concurrency, + "wall_s": time.perf_counter() - start, + "rows": rows, + } + ) + # Explicit cancellation and socket disconnect after the first streamed token. + for mode in ("cancel", "disconnect"): + before_cancel = complete(128) + rid = f"http211-{mode}" + with request("/v1/completions", body(128, 256, True), rid) as response: + for raw in response: + if raw.startswith(b"data: ") and b"[DONE]" not in raw: + data = json.loads(raw[6:]) + if data.get("choices") and data["choices"][0].get("text"): + break + else: + raise AssertionError("stream had no text") + if mode == "cancel": + with request("/v1/cancel", {}, rid) as cancelled: + assert cancelled.status == 202 + drained(rid) + after = complete(128) + result[mode] = { + "drained": True, + "subsequent_tokens_equal": after["tokens"] + == before_cancel["tokens"], + "cold_baseline_equal": after["tokens"] == baseline[128]["tokens"], + "before_tokens": before_cancel["tokens"], + "after_tokens": after["tokens"], + } + assert result[mode]["subsequent_tokens_equal"], ( + "post-cancellation output changed" + ) + + try: + with request( + "/v1/completions", + {"model": a.model, "prompt": [], "max_tokens": 32}, + ): + raise AssertionError("invalid request accepted") + except urllib.error.HTTPError as exc: + assert exc.code == 400 + result["invalid_status"] = exc.code + with request("/ready") as response: + assert response.status == 200 + result["remaining_requests"] = len(owner.service.batcher.by_id) + assert result["remaining_requests"] == 0 + with request("/metrics") as response: + result["metrics"] = ( + response.read().decode().replace(a.model, result["model_name"]) + ) + hits = [ + float(line.rsplit(" ", 1)[1]) + for line in result["metrics"].splitlines() + if line.startswith("vllm:prefix_cache_hits_total{") + ] + assert hits, "prefix hit metric missing" + result["prefix_hit_tokens"] = sum(hits) + if a.prefix_cache: + assert result["prefix_hit_tokens"] > 0, ( + "cache enabled without observed hits" + ) + if a.host_kv_gib: + host_hits = [ + float(line.rsplit(" ", 1)[1]) + for line in result["metrics"].splitlines() + if line.startswith("airm_kv_host_hit_tokens_total{") + ] + result["host_hit_tokens"] = sum(host_hits) + assert host_hits and result["host_hit_tokens"] > 0, ( + "host tier not exercised" + ) + result["batcher_stats"] = dict(owner.service.batcher.stats) + if a.pd_prefill_device: + # Nineteen completed normal requests; cancellations can add two more prefills. + assert result["batcher_stats"]["prefills"] >= 19 + assert result["batcher_stats"]["transferred_tokens"] > 0 + result["passed"] = True + except Exception as exc: # noqa: BLE001 - preserve failed hardware evidence + result["error"] = repr(exc) + finally: + for rid in list(owner.service.batcher.by_id): + owner.service.batcher.cancel(rid) + owner.submit("stop") + + thread = threading.Thread(target=client, daemon=True) + thread.start() + try: + owner.run() + except Exception as exc: # noqa: BLE001 - preserve failure and stop this process + result["engine_error"] = repr(exc) + finally: + thread.join(190) + server.shutdown() + server.server_close() + server_thread.join(5) + worker = getattr(owner.service.batcher, "worker", None) + if worker is not None: + worker.close() + output.write_text(json.dumps(result, indent=2) + "\n") + if not result["passed"]: + raise RuntimeError( + result.get("error", result.get("engine_error", "validation failed")) + ) + print( + json.dumps( + { + k: v + for k, v in result.items() + if k not in ("cases", "baseline", "metrics") + } + ), + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/opt_prime/tests/test_benchmark.py b/opt_prime/tests/test_benchmark.py new file mode 100644 index 00000000..a40564fd --- /dev/null +++ b/opt_prime/tests/test_benchmark.py @@ -0,0 +1,52 @@ +"""Benchmark accounting uses all output tokens and whole-batch wall time.""" +import importlib.util +from pathlib import Path + +import pytest + +BENCHMARK = Path(__file__).parents[1] / 'examples/pd/http_matrix/benchmark.py' +spec = importlib.util.spec_from_file_location('http_matrix_benchmark', BENCHMARK) +benchmark = importlib.util.module_from_spec(spec) +spec.loader.exec_module(benchmark) +timing_summary = benchmark.timing_summary + + +def test_batch_accounting_and_parity(): + rows = [ + {"output_tokens": 2, "first_sse_s": .2, "tpot_est_s": .1, "e2e_s": .4, "text_equal": True}, + {"output_tokens": 4, "first_sse_s": .4, "tpot_est_s": .2, "e2e_s": .8, "text_equal": False}, + ] + out = timing_summary(rows, 2) + assert out["requests"] == 2 and out["output_tokens"] == 6 and out["output_tok_s"] == 3 + assert out["median_first_sse_s"] == pytest.approx(.3) + assert out["median_tpot_est_s"] == pytest.approx(.15) + assert out["median_e2e_s"] == pytest.approx(.6) + assert out["all_text_equal"] is False + + +def test_external_server_config_requires_port_and_metadata(tmp_path): + import json + + path = tmp_path / 'server.json' + good = {'servers': [['python', 'atom_server.py', '--port', '{port}', '--model', 'a model']], + 'metadata': {'physical_gpus': 0, 'physical_atom': 1, 'physical_rngd': 1, + 'producer_chunk': None}} + path.write_text(json.dumps(good)) + assert benchmark.load_server_config(path) == good + for bad in [[], {'servers': [['python', '{port}']], 'metadata': {'passed': True}}, {}, {'servers': []}, {'servers': [['python', 'server.py']], 'metadata': {}}, + {'servers': ['python server.py {port}'], 'metadata': {}}, + {'servers': [['python', '{port}']], 'metadata': []}]: + path.write_text(json.dumps(bad)) + with pytest.raises((ValueError, TypeError)): + benchmark.load_server_config(path) + + +def test_startup_timeout_rejects_nonpositive_before_loading_model(tmp_path): + import subprocess + import sys + + run = subprocess.run([sys.executable, str(BENCHMARK), + '--model', 'unused', '--mode', 'native', '--output', str(tmp_path / 'unused.json'), + '--startup-timeout', '0'], check=False, capture_output=True, text=True) + assert run.returncode == 2 + assert 'startup timeout must be positive' in run.stderr diff --git a/opt_prime/tests/test_heldout_quality.py b/opt_prime/tests/test_heldout_quality.py new file mode 100644 index 00000000..0d9c5872 --- /dev/null +++ b/opt_prime/tests/test_heldout_quality.py @@ -0,0 +1,134 @@ +"""Prevent answer substrings, duplicate keys, or missing pairs from inflating scores.""" +import importlib.util +from pathlib import Path +import pytest + +path = Path(__file__).parents[1] / 'examples/pd/http_matrix/heldout_quality.py' + +def module(): + assert path.exists(), 'held-out evaluator is not implemented' + spec = importlib.util.spec_from_file_location('heldout_quality', path) + value = importlib.util.module_from_spec(spec) + spec.loader.exec_module(value) + return value + +@pytest.mark.parametrize('text,correct,formatted', [ + ('{"answer":"73"}', True, True), + ('{"answer":"73","note":"ok"}', True, False), + ('{"answer":73}', False, False), + ('The answer is 73', False, False), + ('{"answer":"173"}', False, True), + ('{"answer":"wrong","answer":"73"}', False, False), + ('{"answer":"73"} trailing', False, False), +]) +def test_score(text, correct, formatted): + got = module().score(text, '73') + assert (got['answer_correct'], got['format_correct']) == (correct, formatted) + +def test_paired_regressions_are_not_hidden_by_equal_aggregate(): + m = module() + base = [{'case': 'a', 'input_sha256': 'x', 'answer_correct': True, 'tokens': [1]}, + {'case': 'b', 'input_sha256': 'y', 'answer_correct': False, 'tokens': [2]}] + other = [{'case': 'a', 'input_sha256': 'x', 'answer_correct': False, 'tokens': [3]}, + {'case': 'b', 'input_sha256': 'y', 'answer_correct': True, 'tokens': [2]}] + got = m.compare(base, other) + assert got == {'cases': 2, 'correct_to_wrong': 1, 'wrong_to_correct': 1, 'token_equal': 1} + with pytest.raises(ValueError): + m.compare(base, other[:1]) + other[0]['input_sha256'] = 'changed' + with pytest.raises(ValueError): + m.compare(base, other) + +def test_context_builder_keeps_retrieval_fact_inside_budget(): + class CharacterTokenizer: + def apply_chat_template(self, messages, **kwargs): + return list(messages[0]['content'].encode()) + m = module() + cases = m.fixtures(CharacterTokenizer()) + assert len(cases) == len({x['case'] for x in cases}) == 36 + for case in cases: + assert len(case['ids']) <= (case['target_tokens'] or 2048) + if case['task'] != 'arithmetic': + assert case['expected'] in case['prompt'] + if case['target_tokens']: + assert len(case['ids']) > case['target_tokens'] - 150 + +def test_http_runner_supplies_required_request_id_and_frozen_tokens(tmp_path, monkeypatch): + import json + import sys + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + seen = [] + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + seen.append((self.headers.get('x-request-id'), body)) + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps({'choices': [{'text': '{"answer":"73"}', + 'token_ids': [9], 'finish_reason': 'stop'}]}).encode()) + def log_message(self, *args): + pass + server = HTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + fixture = tmp_path / 'input.json' + output = tmp_path / 'output.jsonl' + fixture.write_text(json.dumps({'model':'model', 'max_tokens':128, + 'cases':[{'case':'a','ids':[4,5], 'input_sha256':'hash','expected':'73'}]})) + monkeypatch.setattr(sys, 'argv', ['heldout_quality', 'run', '--fixtures',str(fixture), + '--endpoint',f'http://127.0.0.1:{server.server_port}', '--route','test', '--output',str(output)]) + try: + module().main() + finally: + server.shutdown() + thread.join() + server.server_close() + assert seen[0][0], 'stage HTTP API requires a nonempty request ID' + assert seen[0][1]['prompt'] == [4,5] + assert seen[0][1]['ignore_eos'] is False + assert json.loads(output.read_text())['answer_correct'] is True + with pytest.raises(FileExistsError): + module().main() + +def test_frozen_fixtures_match_manifest(): + import gzip + import hashlib + import json + folder = path.parent / 'fixtures' + manifest = json.loads((folder / 'fixture-manifest.json').read_text()) + assert len(manifest) == 4 + for name, entry in manifest.items(): + raw = gzip.decompress((folder / (name + '.gz')).read_bytes()) + assert hashlib.sha256(raw).hexdigest() == entry['sha256'] + cases = json.loads(raw)['cases'] + assert len(cases) == len({x['case'] for x in cases}) == entry['cases'] + for case in cases: + ids = json.dumps(case['ids'], separators=(',', ':')).encode() + assert hashlib.sha256(ids).hexdigest() == case['input_sha256'] + +def test_runner_reads_gzip_fixture_and_overrides_model(tmp_path, monkeypatch): + import gzip + import json + import sys + m = module() + seen = [] + class Response: + def __enter__(self): + return self + def __exit__(self, *args): + return False + def read(self): + return json.dumps({'choices': [{'text': '{"answer":"73"}', 'token_ids': [9], + 'finish_reason': 'stop'}]}) + def urlopen(request, timeout): + seen.append(json.loads(request.data)) + return Response() + monkeypatch.setattr(m.urllib.request, 'urlopen', urlopen) + fixture = tmp_path / 'input.json.gz' + fixture.write_bytes(gzip.compress(json.dumps({'model': 'frozen', 'max_tokens': 128, + 'cases': [{'case': 'a', 'ids': [4, 5], 'input_sha256': 'hash', 'expected': '73'}]}).encode())) + monkeypatch.setattr(sys, 'argv', ['heldout_quality', 'run', '--fixtures', str(fixture), '--endpoint', 'http://x', + '--route', 'test', '--output', str(tmp_path / 'out.jsonl'), '--model', 'served-name']) + m.main() + assert seen[0]['model'] == 'served-name' and seen[0]['prompt'] == [4, 5]