From 257db87f890760b0611f8beae44d155587024954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=90=91=E9=98=B3?= Date: Fri, 25 Sep 2026 23:33:08 +0800 Subject: [PATCH 1/2] add-classic-model-tests --- README.md | 12 ++++- benchmarks.py | 132 +++++++++++++++++++++++++++++++++++++++++++++ curi.py | 31 ++++++++++- test_benchmarks.py | 54 +++++++++++++++++++ test_curi.py | 6 ++- 5 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 benchmarks.py create mode 100644 test_benchmarks.py diff --git a/README.md b/README.md index 745638c..dc04e69 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,16 @@ Open . CURI scans `~/.codex/sessions` every three seconds The relay listens on `http://127.0.0.1:8080/v1`; point Codex's API base URL at that address and keep the CURI process running. Your existing API key remains in Codex and is forwarded to the configured upstream; CURI never stores it. +Enable the built-in classic tests by supplying a model. The dashboard then shows two buttons under **Classic tests**: + +```bash +CURI_TEST_API_KEY=your-key python curi.py serve \ + --upstream https://api.example.com/v1 \ + --test-model your-model +``` + +The candy button sends the fixed minimum-draw logic puzzle and displays the model's text answer. The pelican button sends `Generate an SVG of a pelican riding a bicycle` and renders a sanitized SVG result. Prompts and responses stay in memory and are never written to the usage database or relay event file; only the normal metadata-only relay event may be recorded. For a Chat Completions provider, add `--test-format chat`; for a separate compatible endpoint, use `--test-base-url`. + ```bash python curi.py serve \ --codex-home ~/.codex \ @@ -63,7 +73,7 @@ Do not write secrets or full payloads to that file. `reported_model` remains `un ```bash python -m unittest -v -python -m py_compile curi.py +python -m py_compile curi.py relay.py benchmarks.py ``` The project deliberately has no runtime dependencies. The dashboard and relay use Python's standard library. The scanner uses file offsets and resumes safely after a restart; a truncated or rewritten JSONL file is rescanned from the beginning. diff --git a/benchmarks.py b/benchmarks.py new file mode 100644 index 0000000..d17ec8c --- /dev/null +++ b/benchmarks.py @@ -0,0 +1,132 @@ +"""Built-in CURI model probes for the classic candy and pelican tests.""" +from __future__ import annotations + +import json +import re +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin +from urllib.request import Request, urlopen + + +CANDY_PROMPT = """请解决下面的糖果测试。只使用题目给出的数量，求“最少摸出多少颗”才能保证手中同时有不同形状的苹果味和桃子味糖果。请给出最小值，并分别证明这个数量足够、少一颗不够。 + +黑色袋子中有三种口味，每种口味有圆形和五角星形两种形状： + +| | 苹果味 | 桃子味 | 西瓜味 | +|--------|--------|--------|--------| +| 圆形 | 7 | 9 | 8 | +| 五角星 | 7 | 6 | 4 | + +只要出现圆形苹果味与五角星桃子味，或圆形桃子味与五角星苹果味，就算满足条件。""" + +PELICAN_PROMPT = """Generate an SVG of a pelican riding a bicycle. Return only one complete, self-contained SVG document, with no Markdown fences or explanation. The SVG should visibly contain a recognizable pelican, two bicycle wheels, a bicycle frame, and the pelican's feet on the pedals.""" + + +@dataclass(frozen=True) +class TestResult: + kind: str + model: str + text: str + svg: str | None + latency_ms: int + + +def _strip_svg_fences(text: str) -> str: + match = re.search(r"", text, re.IGNORECASE) + return match.group(0) if match else "" + + +def sanitize_svg(text: str) -> str: + """Keep SVG display-only: remove scripts, event handlers and external URLs.""" + source = _strip_svg_fences(text) + if not source: + return "" + try: + root = ET.fromstring(source) + except ET.ParseError: + return "" + blocked = {"script", "foreignobject", "iframe", "object", "embed"} + + def clean(parent: ET.Element) -> None: + for child in list(parent): + tag = child.tag.rsplit("}", 1)[-1].lower() if isinstance(child.tag, str) else "" + if tag in blocked: + parent.remove(child) + continue + for key, value in list(child.attrib.items()): + local = key.rsplit("}", 1)[-1].lower() + lowered = value.strip().lower() + if local.startswith("on") or local in {"href", "src"} and not lowered.startswith("data:"): + del child.attrib[key] + clean(child) + + clean(root) + if root.tag.startswith("{http://www.w3.org/2000/svg}"): + ET.register_namespace("", "http://www.w3.org/2000/svg") + return ET.tostring(root, encoding="unicode") + + +def _response_text(payload: object) -> str: + if not isinstance(payload, dict): + return "" + choices = payload.get("choices") + if isinstance(choices, list) and choices: + message = choices[0].get("message", {}) if isinstance(choices[0], dict) else {} + content = message.get("content") if isinstance(message, dict) else "" + if isinstance(content, list): + return "".join(str(item.get("text", "")) for item in content if isinstance(item, dict)) + return str(content or "") + output = payload.get("output") + if isinstance(output, list): + parts: list[str] = [] + for item in output: + if not isinstance(item, dict): + continue + for content in item.get("content", []) if isinstance(item.get("content"), list) else []: + if isinstance(content, dict) and content.get("text"): + parts.append(str(content["text"])) + return "".join(parts) + return str(payload.get("output_text") or payload.get("text") or "") + + +class TestRunner: + def __init__(self, base_url: str, model: str, api_key: str = "", api_format: str = "responses", timeout: float = 120.0): + self.base_url = base_url.rstrip("/") + self.model = model + self.api_key = api_key + self.api_format = api_format + self.timeout = timeout + + def run(self, kind: str) -> TestResult: + if kind not in {"candy", "pelican"}: + raise ValueError("unknown test kind") + prompt = CANDY_PROMPT if kind == "candy" else PELICAN_PROMPT + if self.api_format == "chat": + path = "/chat/completions" + payload = {"model": self.model, "messages": [{"role": "user", "content": prompt}], "stream": False} + else: + path = "/responses" + payload = {"model": self.model, "input": prompt, "stream": False} + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if self.api_key: + headers["Authorization"] = "Bearer " + self.api_key + request = Request(urljoin(self.base_url + "/", path.lstrip("/")), data=json.dumps(payload).encode(), headers=headers, method="POST") + started = time.monotonic() + try: + with urlopen(request, timeout=self.timeout) as response: + raw = response.read(2 * 1024 * 1024) + except HTTPError as exc: + detail = exc.read(1000).decode("utf-8", errors="replace") + raise RuntimeError(f"model request failed ({exc.code}): {detail}") from exc + except (URLError, TimeoutError, OSError) as exc: + raise RuntimeError(f"model request failed: {exc}") from exc + try: + decoded = json.loads(raw.decode("utf-8", errors="replace")) + except json.JSONDecodeError as exc: + raise RuntimeError("model returned invalid JSON") from exc + text = _response_text(decoded).strip() + svg = sanitize_svg(text) if kind == "pelican" else None + return TestResult(kind, self.model, text, svg, round((time.monotonic() - started) * 1000)) diff --git a/curi.py b/curi.py index 87bce68..530ec66 100644 --- a/curi.py +++ b/curi.py @@ -21,6 +21,7 @@ from pathlib import Path from typing import Any +from benchmarks import TestRunner from relay import RelayConfig, create_server @@ -356,7 +357,7 @@ def doctor(codex_home: str, relay_events: str, db_path: str, archive_dir: str = HTML = r'''CURI
CURI / local observability

Know the request
behind the request.

A private, loopback-only view of Codex usage, tools, quotas and relay behavior. No prompts. No responses. No telemetry.

waiting for first scan

Daily signal

Quota windows

Tool activity

Models observed

Coverage

Recent relay events

TimeRequestedReportedStatusAttemptsLatency
''' +HTML = HTML.replace('id="cards">', 'id="cards">

Classic tests

Run the fixed candy logic puzzle or the pelican-on-a-bicycle SVG probe through the configured model. Results stay local.
Configure --test-model to enable these buttons.
No test run yet.
') +HTML = HTML.replace('', '''async function runTest(kind){const buttons=[$('candyButton'),$('pelicanButton')];buttons.forEach(x=>x.disabled=true);$('testStatus').textContent=`running ${kind} test…`;$('testResult').textContent='';try{const response=await fetch('/api/tests/run',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({kind})});const result=await response.json();if(!response.ok)throw new Error(result.error||'test failed');$('testStatus').textContent=`${result.model} · ${result.latency_ms} ms`;if(kind==='pelican'&&result.svg){$('testResult').innerHTML=`pelican test result`}else{$('testResult').textContent=result.text||'The model returned no text.'}}catch(error){$('testStatus').textContent='test unavailable';$('testResult').textContent=error.message}finally{buttons.forEach(x=>x.disabled=false)}} +''') + class Handler(BaseHTTPRequestHandler): store: Store config: dict[str, str] + test_runner: TestRunner | None = None def do_GET(self) -> None: if self.path == "/healthz": self._send(200, {"status": "ok", "last_scan": self.store.last_scan}) @@ -383,6 +389,21 @@ def do_GET(self) -> None: self.send_response(200); self.send_header("Content-Type", "text/html; charset=utf-8"); self.send_header("Content-Length", str(len(data))); self.end_headers(); self.wfile.write(data) else: self.send_error(404) + def do_POST(self) -> None: + if self.path != "/api/tests/run": + self.send_error(404) + return + if self.test_runner is None: + self._send(503, {"error": "classic tests are not configured; start serve with --upstream and --test-model"}) + return + try: + length = min(int(self.headers.get("Content-Length", "0") or 0), 4096) + payload = json.loads(self.rfile.read(length)) if length else {} + result = self.test_runner.run(str(payload.get("kind", ""))) + self._send(200, {"kind": result.kind, "model": result.model, "text": result.text, + "svg": result.svg, "latency_ms": result.latency_ms}) + except (ValueError, RuntimeError) as exc: + self._send(400, {"error": str(exc)}) def _send(self, status: int, payload: dict[str, Any]) -> None: data = json.dumps(payload, ensure_ascii=False).encode() self.send_response(status); self.send_header("Content-Type", "application/json"); self.send_header("Content-Length", str(len(data))); self.end_headers(); self.wfile.write(data) @@ -393,6 +414,9 @@ def log_message(self, *_: Any) -> None: def serve(args: argparse.Namespace) -> None: store = Store(args.db) Handler.store = store + test_base_url = args.test_base_url or (f"http://{args.relay_host}:{args.relay_port}/v1" if args.upstream else "") + Handler.test_runner = TestRunner(test_base_url, args.test_model, args.test_api_key, + args.test_format, args.test_timeout) if args.test_model and test_base_url else None def scan_loop() -> None: while True: store.scan(args.codex_home, args.relay_events, args.archive_dir) @@ -437,6 +461,11 @@ def common(s: argparse.ArgumentParser) -> None: s.add_argument("--retry-backoff", type=float, default=0.5) s.add_argument("--request-timeout", type=float, default=120.0) s.add_argument("--buffer-until-success", action="store_true", help="buffer SSE until response.completed") + s.add_argument("--test-model", default=os.getenv("CURI_TEST_MODEL", ""), help="model used by the built-in tests") + s.add_argument("--test-api-key", default=os.getenv("CURI_TEST_API_KEY", ""), help="optional in-memory test key") + s.add_argument("--test-base-url", default=os.getenv("CURI_TEST_BASE_URL", ""), help="OpenAI-compatible test base URL") + s.add_argument("--test-format", choices=("responses", "chat"), default=os.getenv("CURI_TEST_FORMAT", "responses")) + s.add_argument("--test-timeout", type=float, default=120.0) s = sub.add_parser("relay", help="start the local OpenAI-compatible retry relay") s.add_argument("--upstream", default=os.getenv("UPSTREAM_BASE_URL", ""), required=False) s.add_argument("--host", default="127.0.0.1") diff --git a/test_benchmarks.py b/test_benchmarks.py new file mode 100644 index 0000000..eb01841 --- /dev/null +++ b/test_benchmarks.py @@ -0,0 +1,54 @@ +import json +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from benchmarks import TestRunner, sanitize_svg + + +class BenchmarkUpstream(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length)) + if "pelican" in request.get("input", "") or "pelican" in request.get("messages", [{}])[0].get("content", ""): + text = "```svg\n\n```" + body = {"output": [{"content": [{"type": "output_text", "text": text}]}]} + else: + body = {"output_text": "21\n证明足够且少一颗不够。"} + encoded = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, *args): + return + + +class BenchmarkTests(unittest.TestCase): + def setUp(self): + self.server = ThreadingHTTPServer(("127.0.0.1", 0), BenchmarkUpstream) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + + def test_candy_result(self): + result = TestRunner(f"http://127.0.0.1:{self.server.server_port}/v1", "demo").run("candy") + self.assertIn("21", result.text) + self.assertIsNone(result.svg) + + def test_pelican_svg_is_sanitized(self): + result = TestRunner(f"http://127.0.0.1:{self.server.server_port}/v1", "demo").run("pelican") + self.assertIn(" Date: Fri, 25 Sep 2026 23:35:42 +0800 Subject: [PATCH 2/2] use-canonical-benchmark-prompts --- README.md | 2 ++ benchmarks.py | 21 +++++++++++++++------ test_benchmarks.py | 7 ++++++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dc04e69..6928f87 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ CURI_TEST_API_KEY=your-key python curi.py serve \ The candy button sends the fixed minimum-draw logic puzzle and displays the model's text answer. The pelican button sends `Generate an SVG of a pelican riding a bicycle` and renders a sanitized SVG result. Prompts and responses stay in memory and are never written to the usage database or relay event file; only the normal metadata-only relay event may be recorded. For a Chat Completions provider, add `--test-format chat`; for a separate compatible endpoint, use `--test-base-url`. +The bundled candy wording follows the public candy-v2 prompt, including its “no tools and no web” constraint and first-line answer rule. The pelican wording follows Simon Willison's canonical prompt exactly; neither test adds hidden task-specific hints. + ```bash python curi.py serve \ --codex-home ~/.codex \ diff --git a/benchmarks.py b/benchmarks.py index d17ec8c..26407e4 100644 --- a/benchmarks.py +++ b/benchmarks.py @@ -1,4 +1,8 @@ -"""Built-in CURI model probes for the classic candy and pelican tests.""" +"""Built-in CURI model probes for the classic candy and pelican tests. + +The candy wording follows the public candy-v2 prompt shown by Help AIO: +https://www.helpaio.com/transit/crowdtest/runs/qedc67cc8h +""" from __future__ import annotations import json @@ -11,18 +15,23 @@ from urllib.request import Request, urlopen -CANDY_PROMPT = """请解决下面的糖果测试。只使用题目给出的数量，求“最少摸出多少颗”才能保证手中同时有不同形状的苹果味和桃子味糖果。请给出最小值，并分别证明这个数量足够、少一颗不够。 +CANDY_PROMPT = """不允许调用工具和联网 -黑色袋子中有三种口味，每种口味有圆形和五角星形两种形状： +在一个黑色的袋子里放有三种口味的糖果，每种糖果有两种不同的形状（圆形和五角星形，不同的形状靠手感可以分辨）。现已知不同口味的糖和不同形状的数量统计如下表。参赛者需要在活动前决定摸出的糖果数目，那么，最少取出多少个糖果才能保证手中同时拥有不同形状的苹果味和桃子味的糖？（同时手中有圆形苹果味匹配五角星桃子味糖果，或者有圆形桃子味匹配五角星苹果味糖果都满足要求） | | 苹果味 | 桃子味 | 西瓜味 | |--------|--------|--------|--------| | 圆形 | 7 | 9 | 8 | -| 五角星 | 7 | 6 | 4 | +| 五角星形 | 7 | 6 | 4 | + +作答与交付约定 -只要出现圆形苹果味与五角星桃子味，或圆形桃子味与五角星苹果味，就算满足条件。""" +1. 在最终回答的第一行只写一个纯阿拉伯整数，不带单位、标点或解释。 +2. 第二行起可以写推理过程，仅供人工查看，不参与判分。 +3. CLI 会将最终回答自动保存为 `/workspace/answer.txt`，无需你创建或修改任何文件。""" -PELICAN_PROMPT = """Generate an SVG of a pelican riding a bicycle. Return only one complete, self-contained SVG document, with no Markdown fences or explanation. The SVG should visibly contain a recognizable pelican, two bicycle wheels, a bicycle frame, and the pelican's feet on the pedals.""" +# Canonical Simon Willison prompt: https://simonwillison.net/2024/Oct/25/pelicans-on-a-bicycle/ +PELICAN_PROMPT = "Generate an SVG of a pelican riding a bicycle" @dataclass(frozen=True) diff --git a/test_benchmarks.py b/test_benchmarks.py index eb01841..7ec9779 100644 --- a/test_benchmarks.py +++ b/test_benchmarks.py @@ -3,7 +3,7 @@ import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from benchmarks import TestRunner, sanitize_svg +from benchmarks import CANDY_PROMPT, PELICAN_PROMPT, TestRunner, sanitize_svg class BenchmarkUpstream(BaseHTTPRequestHandler): @@ -27,6 +27,11 @@ def log_message(self, *args): class BenchmarkTests(unittest.TestCase): + def test_canonical_prompts(self): + self.assertIn("不允许调用工具和联网", CANDY_PROMPT) + self.assertIn("第一行只写一个纯阿拉伯整数", CANDY_PROMPT) + self.assertEqual(PELICAN_PROMPT, "Generate an SVG of a pelican riding a bicycle") + def setUp(self): self.server = ThreadingHTTPServer(("127.0.0.1", 0), BenchmarkUpstream) threading.Thread(target=self.server.serve_forever, daemon=True).start()