diff --git a/README.md b/README.md index 745638c..6928f87 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,18 @@ Open . CURI scans `~/.codex/sessions` every three seconds The relay listens on `http://127.0.0.1:8080/v1`; point Codex's API base URL at that address and keep the CURI process running. Your existing API key remains in Codex and is forwarded to the configured upstream; CURI never stores it. +Enable the built-in classic tests by supplying a model. The dashboard then shows two buttons under **Classic tests**: + +```bash +CURI_TEST_API_KEY=your-key python curi.py serve \ + --upstream https://api.example.com/v1 \ + --test-model your-model +``` + +The candy button sends the fixed minimum-draw logic puzzle and displays the model's text answer. The pelican button sends `Generate an SVG of a pelican riding a bicycle` and renders a sanitized SVG result. Prompts and responses stay in memory and are never written to the usage database or relay event file; only the normal metadata-only relay event may be recorded. For a Chat Completions provider, add `--test-format chat`; for a separate compatible endpoint, use `--test-base-url`. + +The bundled candy wording follows the public candy-v2 prompt, including its “no tools and no web” constraint and first-line answer rule. The pelican wording follows Simon Willison's canonical prompt exactly; neither test adds hidden task-specific hints. + ```bash python curi.py serve \ --codex-home ~/.codex \ @@ -63,7 +75,7 @@ Do not write secrets or full payloads to that file. `reported_model` remains `un ```bash python -m unittest -v -python -m py_compile curi.py +python -m py_compile curi.py relay.py benchmarks.py ``` The project deliberately has no runtime dependencies. The dashboard and relay use Python's standard library. The scanner uses file offsets and resumes safely after a restart; a truncated or rewritten JSONL file is rescanned from the beginning. diff --git a/benchmarks.py b/benchmarks.py new file mode 100644 index 0000000..26407e4 --- /dev/null +++ b/benchmarks.py @@ -0,0 +1,141 @@ +"""Built-in CURI model probes for the classic candy and pelican tests. + +The candy wording follows the public candy-v2 prompt shown by Help AIO: +https://www.helpaio.com/transit/crowdtest/runs/qedc67cc8h +""" +from __future__ import annotations + +import json +import re +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin +from urllib.request import Request, urlopen + + +CANDY_PROMPT = """不允许调用工具和联网 + +在一个黑色的袋子里放有三种口味的糖果，每种糖果有两种不同的形状（圆形和五角星形，不同的形状靠手感可以分辨）。现已知不同口味的糖和不同形状的数量统计如下表。参赛者需要在活动前决定摸出的糖果数目，那么，最少取出多少个糖果才能保证手中同时拥有不同形状的苹果味和桃子味的糖？（同时手中有圆形苹果味匹配五角星桃子味糖果，或者有圆形桃子味匹配五角星苹果味糖果都满足要求） + +| | 苹果味 | 桃子味 | 西瓜味 | +|--------|--------|--------|--------| +| 圆形 | 7 | 9 | 8 | +| 五角星形 | 7 | 6 | 4 | + +作答与交付约定 + +1. 在最终回答的第一行只写一个纯阿拉伯整数，不带单位、标点或解释。 +2. 第二行起可以写推理过程，仅供人工查看，不参与判分。 +3. CLI 会将最终回答自动保存为 `/workspace/answer.txt`，无需你创建或修改任何文件。""" + +# Canonical Simon Willison prompt: https://simonwillison.net/2024/Oct/25/pelicans-on-a-bicycle/ +PELICAN_PROMPT = "Generate an SVG of a pelican riding a bicycle" + + +@dataclass(frozen=True) +class TestResult: + kind: str + model: str + text: str + svg: str | None + latency_ms: int + + +def _strip_svg_fences(text: str) -> str: + match = re.search(r"", text, re.IGNORECASE) + return match.group(0) if match else "" + + +def sanitize_svg(text: str) -> str: + """Keep SVG display-only: remove scripts, event handlers and external URLs.""" + source = _strip_svg_fences(text) + if not source: + return "" + try: + root = ET.fromstring(source) + except ET.ParseError: + return "" + blocked = {"script", "foreignobject", "iframe", "object", "embed"} + + def clean(parent: ET.Element) -> None: + for child in list(parent): + tag = child.tag.rsplit("}", 1)[-1].lower() if isinstance(child.tag, str) else "" + if tag in blocked: + parent.remove(child) + continue + for key, value in list(child.attrib.items()): + local = key.rsplit("}", 1)[-1].lower() + lowered = value.strip().lower() + if local.startswith("on") or local in {"href", "src"} and not lowered.startswith("data:"): + del child.attrib[key] + clean(child) + + clean(root) + if root.tag.startswith("{http://www.w3.org/2000/svg}"): + ET.register_namespace("", "http://www.w3.org/2000/svg") + return ET.tostring(root, encoding="unicode") + + +def _response_text(payload: object) -> str: + if not isinstance(payload, dict): + return "" + choices = payload.get("choices") + if isinstance(choices, list) and choices: + message = choices[0].get("message", {}) if isinstance(choices[0], dict) else {} + content = message.get("content") if isinstance(message, dict) else "" + if isinstance(content, list): + return "".join(str(item.get("text", "")) for item in content if isinstance(item, dict)) + return str(content or "") + output = payload.get("output") + if isinstance(output, list): + parts: list[str] = [] + for item in output: + if not isinstance(item, dict): + continue + for content in item.get("content", []) if isinstance(item.get("content"), list) else []: + if isinstance(content, dict) and content.get("text"): + parts.append(str(content["text"])) + return "".join(parts) + return str(payload.get("output_text") or payload.get("text") or "") + + +class TestRunner: + def __init__(self, base_url: str, model: str, api_key: str = "", api_format: str = "responses", timeout: float = 120.0): + self.base_url = base_url.rstrip("/") + self.model = model + self.api_key = api_key + self.api_format = api_format + self.timeout = timeout + + def run(self, kind: str) -> TestResult: + if kind not in {"candy", "pelican"}: + raise ValueError("unknown test kind") + prompt = CANDY_PROMPT if kind == "candy" else PELICAN_PROMPT + if self.api_format == "chat": + path = "/chat/completions" + payload = {"model": self.model, "messages": [{"role": "user", "content": prompt}], "stream": False} + else: + path = "/responses" + payload = {"model": self.model, "input": prompt, "stream": False} + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if self.api_key: + headers["Authorization"] = "Bearer " + self.api_key + request = Request(urljoin(self.base_url + "/", path.lstrip("/")), data=json.dumps(payload).encode(), headers=headers, method="POST") + started = time.monotonic() + try: + with urlopen(request, timeout=self.timeout) as response: + raw = response.read(2 * 1024 * 1024) + except HTTPError as exc: + detail = exc.read(1000).decode("utf-8", errors="replace") + raise RuntimeError(f"model request failed ({exc.code}): {detail}") from exc + except (URLError, TimeoutError, OSError) as exc: + raise RuntimeError(f"model request failed: {exc}") from exc + try: + decoded = json.loads(raw.decode("utf-8", errors="replace")) + except json.JSONDecodeError as exc: + raise RuntimeError("model returned invalid JSON") from exc + text = _response_text(decoded).strip() + svg = sanitize_svg(text) if kind == "pelican" else None + return TestResult(kind, self.model, text, svg, round((time.monotonic() - started) * 1000)) diff --git a/curi.py b/curi.py index 87bce68..530ec66 100644 --- a/curi.py +++ b/curi.py @@ -21,6 +21,7 @@ from pathlib import Path from typing import Any +from benchmarks import TestRunner from relay import RelayConfig, create_server @@ -356,7 +357,7 @@ def doctor(codex_home: str, relay_events: str, db_path: str, archive_dir: str = HTML = r'''CURI
CURI / local observability

Know the request
behind the request.

A private, loopback-only view of Codex usage, tools, quotas and relay behavior. No prompts. No responses. No telemetry.

waiting for first scan

Daily signal

Quota windows

Tool activity

Models observed

Coverage

Recent relay events

TimeRequestedReportedStatusAttemptsLatency
''' +HTML = HTML.replace('id="cards">', 'id="cards">

Classic tests

Run the fixed candy logic puzzle or the pelican-on-a-bicycle SVG probe through the configured model. Results stay local.
Configure --test-model to enable these buttons.
No test run yet.
') +HTML = HTML.replace('', '''async function runTest(kind){const buttons=[$('candyButton'),$('pelicanButton')];buttons.forEach(x=>x.disabled=true);$('testStatus').textContent=`running ${kind} test…`;$('testResult').textContent='';try{const response=await fetch('/api/tests/run',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({kind})});const result=await response.json();if(!response.ok)throw new Error(result.error||'test failed');$('testStatus').textContent=`${result.model} · ${result.latency_ms} ms`;if(kind==='pelican'&&result.svg){$('testResult').innerHTML=`pelican test result`}else{$('testResult').textContent=result.text||'The model returned no text.'}}catch(error){$('testStatus').textContent='test unavailable';$('testResult').textContent=error.message}finally{buttons.forEach(x=>x.disabled=false)}} +''') + class Handler(BaseHTTPRequestHandler): store: Store config: dict[str, str] + test_runner: TestRunner | None = None def do_GET(self) -> None: if self.path == "/healthz": self._send(200, {"status": "ok", "last_scan": self.store.last_scan}) @@ -383,6 +389,21 @@ def do_GET(self) -> None: self.send_response(200); self.send_header("Content-Type", "text/html; charset=utf-8"); self.send_header("Content-Length", str(len(data))); self.end_headers(); self.wfile.write(data) else: self.send_error(404) + def do_POST(self) -> None: + if self.path != "/api/tests/run": + self.send_error(404) + return + if self.test_runner is None: + self._send(503, {"error": "classic tests are not configured; start serve with --upstream and --test-model"}) + return + try: + length = min(int(self.headers.get("Content-Length", "0") or 0), 4096) + payload = json.loads(self.rfile.read(length)) if length else {} + result = self.test_runner.run(str(payload.get("kind", ""))) + self._send(200, {"kind": result.kind, "model": result.model, "text": result.text, + "svg": result.svg, "latency_ms": result.latency_ms}) + except (ValueError, RuntimeError) as exc: + self._send(400, {"error": str(exc)}) def _send(self, status: int, payload: dict[str, Any]) -> None: data = json.dumps(payload, ensure_ascii=False).encode() self.send_response(status); self.send_header("Content-Type", "application/json"); self.send_header("Content-Length", str(len(data))); self.end_headers(); self.wfile.write(data) @@ -393,6 +414,9 @@ def log_message(self, *_: Any) -> None: def serve(args: argparse.Namespace) -> None: store = Store(args.db) Handler.store = store + test_base_url = args.test_base_url or (f"http://{args.relay_host}:{args.relay_port}/v1" if args.upstream else "") + Handler.test_runner = TestRunner(test_base_url, args.test_model, args.test_api_key, + args.test_format, args.test_timeout) if args.test_model and test_base_url else None def scan_loop() -> None: while True: store.scan(args.codex_home, args.relay_events, args.archive_dir) @@ -437,6 +461,11 @@ def common(s: argparse.ArgumentParser) -> None: s.add_argument("--retry-backoff", type=float, default=0.5) s.add_argument("--request-timeout", type=float, default=120.0) s.add_argument("--buffer-until-success", action="store_true", help="buffer SSE until response.completed") + s.add_argument("--test-model", default=os.getenv("CURI_TEST_MODEL", ""), help="model used by the built-in tests") + s.add_argument("--test-api-key", default=os.getenv("CURI_TEST_API_KEY", ""), help="optional in-memory test key") + s.add_argument("--test-base-url", default=os.getenv("CURI_TEST_BASE_URL", ""), help="OpenAI-compatible test base URL") + s.add_argument("--test-format", choices=("responses", "chat"), default=os.getenv("CURI_TEST_FORMAT", "responses")) + s.add_argument("--test-timeout", type=float, default=120.0) s = sub.add_parser("relay", help="start the local OpenAI-compatible retry relay") s.add_argument("--upstream", default=os.getenv("UPSTREAM_BASE_URL", ""), required=False) s.add_argument("--host", default="127.0.0.1") diff --git a/test_benchmarks.py b/test_benchmarks.py new file mode 100644 index 0000000..7ec9779 --- /dev/null +++ b/test_benchmarks.py @@ -0,0 +1,59 @@ +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from benchmarks import CANDY_PROMPT, PELICAN_PROMPT, TestRunner, sanitize_svg + + +class BenchmarkUpstream(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length)) + if "pelican" in request.get("input", "") or "pelican" in request.get("messages", [{}])[0].get("content", ""): + text = "```svg\n\n```" + body = {"output": [{"content": [{"type": "output_text", "text": text}]}]} + else: + body = {"output_text": "21\n证明足够且少一颗不够。"} + encoded = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, *args): + return + + +class BenchmarkTests(unittest.TestCase): + def test_canonical_prompts(self): + self.assertIn("不允许调用工具和联网", CANDY_PROMPT) + self.assertIn("第一行只写一个纯阿拉伯整数", CANDY_PROMPT) + self.assertEqual(PELICAN_PROMPT, "Generate an SVG of a pelican riding a bicycle") + + def setUp(self): + self.server = ThreadingHTTPServer(("127.0.0.1", 0), BenchmarkUpstream) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + + def test_candy_result(self): + result = TestRunner(f"http://127.0.0.1:{self.server.server_port}/v1", "demo").run("candy") + self.assertIn("21", result.text) + self.assertIsNone(result.svg) + + def test_pelican_svg_is_sanitized(self): + result = TestRunner(f"http://127.0.0.1:{self.server.server_port}/v1", "demo").run("pelican") + self.assertIn("