From e29fe9565ab8a08bead7342838b0415570991f92 Mon Sep 17 00:00:00 2001 From: ianu82 Date: Sat, 26 Sep 2026 13:06:32 +0100 Subject: [PATCH 1/2] Add portable frozen-actor builds and the five-app validation harness --- .gitignore | 8 + README.md | 4 + evals/.gitignore | 5 + evals/HILLCLIMB.md | 3 + evals/PROTOCOL.md | 47 + evals/README.md | 41 + evals/RUBRIC.md | 7 + evals/actors.json | 235 ++ evals/browser/checks.mjs | 76 + evals/browser/drag.mjs | 48 + evals/browser/grade.mjs | 38 + evals/experiment.example.json | 15 + evals/fixtures/application/backend/index.ts | 16 + .../fixtures/application/frontend/src/App.tsx | 12 + evals/fixtures/common/.gitignore | 4 + evals/fixtures/common/backend/db.ts | 4 + evals/fixtures/common/backend/decisions.ts | 2 + evals/fixtures/common/backend/index.ts | 5 + evals/fixtures/common/backend/llm.ts | 3 + evals/fixtures/common/frontend/index.html | 1 + evals/fixtures/common/frontend/src/App.tsx | 1 + evals/fixtures/common/frontend/src/main.tsx | 5 + evals/fixtures/common/frontend/src/style.css | 22 + evals/fixtures/common/frontend/vite.config.ts | 8 + evals/fixtures/common/package-lock.json | 3054 +++++++++++++++++ evals/fixtures/common/package.json | 17 + evals/fixtures/common/scripts/restart.sh | 5 + evals/fixtures/common/scripts/start.sh | 15 + evals/fixtures/common/scripts/stop.sh | 7 + evals/fixtures/common/tsconfig.json | 4 + evals/fixtures/common/yolocoder.json | 1 + evals/fixtures/csv/backend/index.ts | 26 + evals/fixtures/csv/frontend/src/App.tsx | 11 + evals/fixtures/inventory/backend/index.ts | 13 + evals/fixtures/inventory/frontend/src/App.tsx | 9 + evals/fixtures/notes/backend/index.ts | 20 + evals/fixtures/notes/frontend/src/App.tsx | 8 + evals/fixtures/quiz/backend/index.ts | 25 + evals/fixtures/quiz/frontend/src/App.tsx | 8 + evals/package-lock.json | 76 + evals/package.json | 7 + evals/reproduce.py | 106 + evals/tests/test_controls.mjs | 32 + evals/tests/test_drag.mjs | 113 + evals/tests/test_paired.py | 51 + evals/tests/test_recorder.py | 116 + evals/tests/test_runtime.py | 46 + evals/tests/test_scoring.py | 58 + evals/tests/test_unseen.py | 34 + evals/validation/runtime.py | 62 + evals/validation/semantics.py | 41 + evals/yoloeval/__init__.py | 1 + evals/yoloeval/catalog.py | 33 + evals/yoloeval/cli.py | 89 + evals/yoloeval/fixtures.py | 102 + evals/yoloeval/judge.py | 66 + evals/yoloeval/paired.py | 54 + evals/yoloeval/provider.py | 156 + evals/yoloeval/report.py | 46 + evals/yoloeval/runner.py | 178 + evals/yoloeval/runtime.py | 158 + evals/yoloeval/scoring.py | 86 + evals/yoloeval/solutions.py | 34 + evals/yoloeval/unseen.py | 121 + 64 files changed, 5699 insertions(+) create mode 100644 evals/.gitignore create mode 100644 evals/HILLCLIMB.md create mode 100644 evals/PROTOCOL.md create mode 100644 evals/README.md create mode 100644 evals/RUBRIC.md create mode 100644 evals/actors.json create mode 100644 evals/browser/checks.mjs create mode 100644 evals/browser/drag.mjs create mode 100644 evals/browser/grade.mjs create mode 100644 evals/experiment.example.json create mode 100644 evals/fixtures/application/backend/index.ts create mode 100644 evals/fixtures/application/frontend/src/App.tsx create mode 100644 evals/fixtures/common/.gitignore create mode 100644 evals/fixtures/common/backend/db.ts create mode 100644 evals/fixtures/common/backend/decisions.ts create mode 100644 evals/fixtures/common/backend/index.ts create mode 100644 evals/fixtures/common/backend/llm.ts create mode 100644 evals/fixtures/common/frontend/index.html create mode 100644 evals/fixtures/common/frontend/src/App.tsx create mode 100644 evals/fixtures/common/frontend/src/main.tsx create mode 100644 evals/fixtures/common/frontend/src/style.css create mode 100644 evals/fixtures/common/frontend/vite.config.ts create mode 100644 evals/fixtures/common/package-lock.json create mode 100644 evals/fixtures/common/package.json create mode 100644 evals/fixtures/common/scripts/restart.sh create mode 100644 evals/fixtures/common/scripts/start.sh create mode 100644 evals/fixtures/common/scripts/stop.sh create mode 100644 evals/fixtures/common/tsconfig.json create mode 100644 evals/fixtures/common/yolocoder.json create mode 100644 evals/fixtures/csv/backend/index.ts create mode 100644 evals/fixtures/csv/frontend/src/App.tsx create mode 100644 evals/fixtures/inventory/backend/index.ts create mode 100644 evals/fixtures/inventory/frontend/src/App.tsx create mode 100644 evals/fixtures/notes/backend/index.ts create mode 100644 evals/fixtures/notes/frontend/src/App.tsx create mode 100644 evals/fixtures/quiz/backend/index.ts create mode 100644 evals/fixtures/quiz/frontend/src/App.tsx create mode 100644 evals/package-lock.json create mode 100644 evals/package.json create mode 100644 evals/reproduce.py create mode 100644 evals/tests/test_controls.mjs create mode 100644 evals/tests/test_drag.mjs create mode 100644 evals/tests/test_paired.py create mode 100644 evals/tests/test_recorder.py create mode 100644 evals/tests/test_runtime.py create mode 100644 evals/tests/test_scoring.py create mode 100644 evals/tests/test_unseen.py create mode 100644 evals/validation/runtime.py create mode 100644 evals/validation/semantics.py create mode 100644 evals/yoloeval/__init__.py create mode 100644 evals/yoloeval/catalog.py create mode 100644 evals/yoloeval/cli.py create mode 100644 evals/yoloeval/fixtures.py create mode 100644 evals/yoloeval/judge.py create mode 100644 evals/yoloeval/paired.py create mode 100644 evals/yoloeval/provider.py create mode 100644 evals/yoloeval/report.py create mode 100644 evals/yoloeval/runner.py create mode 100644 evals/yoloeval/runtime.py create mode 100644 evals/yoloeval/scoring.py create mode 100644 evals/yoloeval/solutions.py create mode 100644 evals/yoloeval/unseen.py diff --git a/.gitignore b/.gitignore index 2934354..8841b49 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,11 @@ go.work.sum # on disk and the compiler still reads it. `find . -name '* [0-9]*'` finds # them when a build fails for no apparent reason. * [0-9]* + +# Local evaluation dependencies and evidence +/evals/node_modules/ +/evals/.cache/ +/evals/runs/ +/dist/ +__pycache__/ +*.pyc diff --git a/README.md b/README.md index 663dd2d..d182a32 100644 --- a/README.md +++ b/README.md @@ -327,3 +327,7 @@ for and installs updates. Non-interactive output remains animation-free. Version tags matching `v*` create permanent GitHub releases. Every push to `main` refreshes the rolling `latest` release used by the self-updater. + +## Reproduce the Muse + Jev benchmark + +See [the evaluation guide](evals/README.md) for source-verified builds of original vanilla and the frozen winner, offline calibration, and the five-app comparison. The paid benchmark is a separate explicit step. diff --git a/evals/.gitignore b/evals/.gitignore new file mode 100644 index 0000000..d411719 --- /dev/null +++ b/evals/.gitignore @@ -0,0 +1,5 @@ +.cache/ +node_modules/ +__pycache__/ +*.pyc +results/ diff --git a/evals/HILLCLIMB.md b/evals/HILLCLIMB.md new file mode 100644 index 0000000..4cc8c2e --- /dev/null +++ b/evals/HILLCLIMB.md @@ -0,0 +1,3 @@ +# Validation only + +Do not hillclimb on these tasks during the frozen comparison. See ../VALIDATION.md. Once failures are used for product tuning, this suite is no longer an unseen validation set. diff --git a/evals/PROTOCOL.md b/evals/PROTOCOL.md new file mode 100644 index 0000000..457cf3e --- /dev/null +++ b/evals/PROTOCOL.md @@ -0,0 +1,47 @@ +# YoloCoder unseen-app validation 002 + +This suite was authored after the Round 032 winner was frozen. It is validation, not a new hillclimb. The winner and original vanilla binaries are copied verbatim and identified by SHA-256; neither actor is edited or rebuilt. + +| App | Distinctive behaviour | Seeded bug | Feature | +|---|---|---|---| +| CSV explorer | Quoted CSV, validation, persistent datasets, row search | Case-sensitive search | Numeric/text sorting composed with search | +| Quiz builder | Question editor, answer validation, practice and scoring | Choice zero never scores | Persisted explanations and schema migration | +| Inventory manager | Case-insensitive SKU uniqueness and atomic stock adjustments | Negative stock is clipped instead of refused | Live low-stock filtering | +| Notes editor | Literal text, partial edits, search and preservation | Title-only updates erase body | Archive/restore with migration and filtering | +| Application form | Drafts, three-step navigation, validation, immutable submissions | Back navigation clears the statement | Duplicate a submission into an independent draft | + +Every app also has a scaffold-to-app creation task and a small heading/button-colour edit. All 20 tasks are independent; three repetitions produce 60 attempts per actor, 120 planned in total. Noncreation tasks start from handwritten reference implementations, with pre-existing records. No generated app becomes the next task's starting point. + +## Fixed comparison + +Both actors use MindsHub Muse `muse-spark-1-3` for coding. The frozen winner additionally uses its existing `jev-1.13.0` router. The original vanilla binary is the exact one used for the first baseline, not the later context/checklist control. + +One actor runs at a time. Within matching task/repetition pairs, A/B and B/A order is balanced; creation tasks run first, followed by features, bugs, then edits. Order inside each block is seeded. Both have a 180-second actor budget and 30 forwarded model calls. Setup and independent grading are excluded from agent time. Timeout cleanup can make observed wall time exceed the nominal budget; raw timings are retained. + +An attempt passes only if the actor reports no error and all required typecheck, functional browser/API, protected-file, persistence and mobile-fit checks pass. Graders and reference solutions are outside the project given to the actor. This is process and HOME isolation, not an OS filesystem sandbox. + +The winner validates only if it has at least as many passes as vanilla, no paired failures of the two storage-preservation checks when vanilla passes those checks, and strictly lower mean and nearest-rank P50 across all attempts, including failures. The candidate's 60/60 result is reported separately. Five app families cannot establish universal reliability, and repetitions are correlated. + +The authoritative executable rules are `evals/yoloeval/unseen.py`. A JSON preregistration, manifests, evaluator/dependency hashes, binary hashes and full schedule are written before the first paid call. Do not use the legacy general-purpose comparator's promotion rule for this validation. + +Early stopping occurs only after an attempt and all its upstream calls are drained: an unrecoverable pass-count deficit, a paired storage-check regression, or a mathematically impossible latency result. Unknown future vanilla times have no invented upper bound; latency stopping therefore waits until vanilla's full timing set is known. Early-stopped campaigns are incomplete and are never presented as 60-sample results. Infrastructure gaps stop the campaign without silently retrying or replacing an attempt. + +## Offline validation + +Each task has a correct positive control and a deliberately incomplete or buggy negative control: 40 controls. Noncreation negatives must fail exactly the intended requirement while other checks pass. Creation negatives must fail the app contract but retain valid TypeScript and protected files. Exploratory calibration exposed exact-label matching problems, migration requirements incorrectly grouped with preservation, an overbroad reference colour change and header rows included in CSV sort assertions. The final `calibration-frozen` directory reruns every control against fixed source fingerprints. Earlier calibration directories are preserved as development evidence and excluded from the final calibration gate. + +Unit tests cover recorder behaviour, scoring, pairing, deadline cleanup and new stopping rules. The runtime integration check uses a local fake provider and no paid model calls. No source/scoring changes are allowed once the paid comparison begins. Any future product tuning against these tasks makes them development cases, requiring another fresh holdout. + +Visual polish, comprehensive accessibility/security, internet integrations and deployment are outside this functional benchmark. Provider token accounting is preserved as reported; unknown prices and inconsistent token usage are not converted into invented cost or TPS estimates. + +## Campaign 002: native search-input correction + +Campaign 001 was stopped after 24 completed attempts and one interrupted attempt because the grader incorrectly excluded the browser's `searchbox` role. The generated `notes.create` trial 1 app had a correctly associated label and a native `type="search"` input. Its failed search check was a harness false negative. The original artifacts and interruption were retained in the private working archive; that campaign is invalidated and cannot establish acceptance. + +Campaign 002 adds searchbox support and a direct associated-label fallback. Browser unit tests cover text, search, number, email, URL, telephone and password inputs, both wrapping and external labels, populated textareas, selects and checkboxes. Additional offline controls test native search inputs in both CSV and notes, and the exact generated notes source that exposed the false negative. + +The original vanilla and winning binaries, public tasks, repetitions, budgets, order seed, pass requirements and acceptance rules are unchanged. No YoloCoder source, prompts or model configuration were tuned against these cases. The entire 120-attempt comparison restarts under a newly frozen evaluator; no previous paid row is substituted into it. This is an infrastructure correction, not a candidate revision. + +## Portable replication + +This published harness preserves the tasks, fixtures, graders, ordering, budgets and decision function from campaign 002. Its launcher accepts locally rebuilt actor hashes rather than requiring the original macOS binaries or the original account-specific credential prefix. `reproduce.py` verifies every actor source file against the frozen inventory, runs offline controls, and records new binary, evaluator and dependency hashes before a new campaign. The published suite digest therefore differs from the historical receipt. See [README.md](README.md). diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..b14b5b2 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,41 @@ +# Replicate the Muse + Jev comparison + +This reproduces the **frozen Round 032 winner versus original vanilla**, using the five-app unseen-validation suite. It does not substitute current main or the new integration branch for either actor. Source files are checked against [actors.json](actors.json) before building. Build hashes vary across platforms/toolchains and are recorded as a new campaign. + +## Requirements + +- A full clone of this repository, including `review/frozen-r032` (or fetch that branch). +- Go 1.24+, Python 3.10+, Node.js 22 and npm; macOS or Linux with `git`, `tar`, and `/bin/sh`. SQLite's native dependency may need local build tools. +- For the paid step only, your own YoloCoder MindsHub configuration in `~/.config/yolocoder/config.json` and `credentials.json`. Set it up through YoloCoder's `config` command. The harness requires `https://api.mindshub.ai`, pins coding to `muse-spark-1-3`, and allows `jev-1.13.0` only for the candidate. Never commit credentials. + +## Workflow + +From the repository root: + +```sh +python3 evals/reproduce.py setup +python3 evals/reproduce.py prepare +python3 evals/reproduce.py check +# Paid step: up to 120 attempts; it is not started by setup/prepare/check. +python3 evals/reproduce.py run +``` + +Use `--output /absolute/path/to/new-campaign` on prepare/check/run to choose a different output directory. `GO=/path/to/go` selects a Go executable. Local output and dependencies are ignored by Git. Preparation will not overwrite an existing build; use a fresh directory for another campaign. Setup installs pinned npm dependencies and Playwright Chromium. + +The offline check runs unit/browser tests, all **40 positive/negative controls**, and fake-provider timeout/cleanup checks. The paid runner rejects changed actors, graders or dependencies; preregisters the whole schedule; preserves all attempts; and cannot retry an interrupted attempt in place. Resuming a fully recorded boundary reuses its original rows. Do not edit source/dependencies or run multiple campaigns against the same cache while a campaign is active. Eval apps receive a proxy credential, not your provider key. Isolation is per process/project/HOME, not an OS security sandbox. + +Results appear in `evals/runs/replication/comparison/`: `decision.json` is the authoritative acceptance result; each actor has a report and every attempt's result, model-call metadata, generated source changes, browser screenshots and logs. Auto-update is disabled for actor processes. Provider errors and timeouts count in the primary outcome; no retries or deletions improve the score. + +## Tasks and scoring + +Five apps: CSV explorer, quiz builder, inventory manager, notes editor, and a three-step application form. Each has app creation, a small UI edit, a feature, and a seeded bug, repeated three times per actor. Tasks start independently from a scaffold or handwritten reference with existing data. The coding agent does not receive the graders or solutions. + +Passing requires every functional browser/API, TypeScript, data-preservation/restart, protected-file and mobile-fit check, plus no actor error. A new candidate is accepted only with at least vanilla's passes, no paired storage regressions, and lower mean **and** nearest-rank P50 agent time over all 60 attempts. Setup/grading time is excluded. Difficult tasks run first, with balanced serial A/B order. Mathematically impossible candidates stop early and are labelled incomplete. + +See [the full protocol](PROTOCOL.md), [task definitions](yoloeval/catalog.py), [grading code](browser/checks.mjs), and [acceptance/early-stop rules](yoloeval/unseen.py). Jev routes the candidate; deterministic checks grade the results. There is no LLM judge in this protocol. + +## Interpretation + +The recorded unseen result was **45/60 → 53/60**, mean **70.58s → 47.37s**, P50 **54.30s → 30.92s**. This is a complete-bundle result, not an isolated Jev ablation. Campaign 001 was invalidated for a native-search-input grader bug; campaign 002 restarted all 120 attempts after correcting it. A provider outage was held between attempts; all scored service errors remained in the final data. + +The earlier tuned suite reached 60/60; it is distinct from this holdout. These published tasks are now exposed and must not become evidence of generalization after further tuning. Use fresh apps for the next holdout. This checks a specific React/Express/SQLite stack, not comprehensive visual design, accessibility, security, deployment, or other languages. A replicated run measures current provider conditions, so identical scores and timings are not guaranteed. diff --git a/evals/RUBRIC.md b/evals/RUBRIC.md new file mode 100644 index 0000000..dc30c30 --- /dev/null +++ b/evals/RUBRIC.md @@ -0,0 +1,7 @@ +# Scoring + +An attempt passes only with no actor error and all mandatory checks passing. Weighted scores (acceptance 60%, regression 30%, usability 10%) are diagnostic only and cannot turn a failed attempt into a success. + +Campaign acceptance requires at least vanilla's pass count, no paired storage-check regression, and lower mean and nearest-rank median agent time across all attempts. The full executable rules are yoloeval/unseen.py and the generated preregistration.json. First-visible-correct-preview latency is not measured. + +Five new applications × four independent tasks × three repetitions = 60 attempts per actor. The repeat count does not mean 60 independent applications. No model judge determines functional pass/fail. diff --git a/evals/actors.json b/evals/actors.json new file mode 100644 index 0000000..b807b81 --- /dev/null +++ b/evals/actors.json @@ -0,0 +1,235 @@ +{ + "schema_version": 1, + "baseline": { + "revision": "6a00e382caca8af8c9be452596a348beb01a87b0", + "version": "main", + "version_commit": "6a00e38", + "flags": [], + "reported_binary_sha256": "654e3049a6b4adc2469f6576aee7b842d54565d763ef3e92af350c8b939c8478", + "source_sha256": { + "cmd/yolocoder/main.go": "5175151a82dd2a8349d0fc66bfb20fd96f3b9ee5351e16df029a1ce24eed46dd", + "go.mod": "899e81c23c1c7711ef47349ed376acc382bea479b2033291922b49861eea7b9a", + "go.sum": "fc37dfaee1833a1117fdecc9fc5cc82fb9605263303cf2d2c395f77320ad7bd1", + "internal/agent/agent.go": "ae3bb8b3981d262e8423cbffa89188aaee827ca001b650e94ec7f4f240b03210", + "internal/agent/agent_test.go": "08cdc8f99cb1b020d5bbf10da25f6c526e2d4a9fe895feb13b62407ec2c6d0c3", + "internal/agent/chat.go": "f15b7a752a6d7a02dc4951c608b1e3bb44dac493c5a15808e093bc52bc4f69d0", + "internal/agent/chat_test.go": "24cf06657dc5557261493c5a219b4dfbb10bda01aa5d10f1edf437b268ceb716", + "internal/agent/client.go": "fe7c960eea65f4b673f630cd50a51dcce48b612fd9d46aa5b90b9d0671f212fc", + "internal/agent/client_test.go": "be923aa16aac5a21bafacdfc37cfedf06a4682dde71455ac440cb5412d0bbb36", + "internal/agent/loop_test.go": "fcfda890b0e8465ac4c4e95dde11e8a1da086a06ee13e174c192d838e15dbea9", + "internal/agent/profile.go": "b4d92697751aa88e0056b6e130d83f7d9d0476fa25422a5bc874fa9db8aca230", + "internal/agent/profile_test.go": "14e48055b57d4cd0ef636b8534da1b619e3388b0a95feb670b4449fdce8113fc", + "internal/agent/recall_test.go": "c50b5f8dd62f465505a062cee3bbe7fc47cb8de0789ae845bc950d8acd3b3787", + "internal/agent/tester.go": "de28e3ec237e0cae02e67bf9224a83849bf04f36bf9e9258ee67bf85baa249dd", + "internal/agent/tester_test.go": "6914e401c949574454a9f3e9bcb9faa825f8e871e61a3b46d0d7ebedea249f75", + "internal/agent/usage.go": "bd877466ae9ae4ecd7c16acc3bb7cedd01204bf1620cca2a15700acd580d7f82", + "internal/agent/usage_test.go": "d580e5579dc1b87f525361452b405ce3d15a77b05e94571c359781fdf264cf7a", + "internal/app/app.go": "0e76977ff6d566bf6624ec3d3a93e6f1026f9c49e1069d5b90b0a62f07647594", + "internal/app/app_test.go": "98b216460d70998661e38a10633bc4034f5d81d7750df31c0e0ad777e8cd5987", + "internal/app/args.go": "c68c692bc3df21f57d0721a2b5645c667f544a10e78b52eb1ad5f69e6f83ffb5", + "internal/app/args_test.go": "c6ed8da2196123aeca659ca601b632d2b24ad80634a3f406bb7fc8ec2adb03b9", + "internal/app/coding.go": "b1c980dabcf393f187a1b6b844337055d8f2a2f60f8eb40bb30429e65bd9e0e9", + "internal/auth/auth.go": "4c15d81a0fd9fb1f3a5bbd4f76754fd7b28e910b85415ca85359d4d87e9dca36", + "internal/config/config.go": "c7fcbf876a648e2a9a73cfc2beb6947a530f06b0be82f5a626c33ded946302d7", + "internal/config/config_test.go": "065e010e726e6cc29e324028d6da74c07e5a00dbdce6c095dbabf5a27dcc7b9a", + "internal/config/providers.go": "0c019d1cbae9893a05976b7223c9313a5d094c36d424846346158fcf7fe2c339", + "internal/debug/debug.go": "6c9ca85966234c35b3e515a3404e0d382f59eeaee3a0e646856124a58a33144d", + "internal/debug/debug_test.go": "8de060512078a0bb6b36f3944f2363427c47bbd1bbd0cc28f6ccb28cd9851044", + "internal/repo/compact.go": "3a44fb4822bd23a0e941a9210cf83725cc12860110b555cee5b2cf3be13cd980", + "internal/repo/compact_test.go": "34d46ede502c4096f999f109ca0055202c334dbe38fe8ec28c552d797c6bf517", + "internal/repo/patch.go": "7a77a77f37c3c79d2e96144606185dabf4c63a8f9f70850a7ff05f6e5c5d0055", + "internal/repo/patch_test.go": "8262af45eef0c50875da06ba79a807a0d0bdc5c8f02774793f86a3a2395c6f02", + "internal/repo/repo.go": "b5e1e28874e669280dca04de42dcda7570d6e85bc2073bff3bec1a90062b538a", + "internal/repo/repo_test.go": "f3afa129ae27a8de7413bde2b31cae68437ff864b5cfc237766e566d1e3e7d7a", + "internal/session/session.go": "4e4335e282f8032db7d014ece1fee21b94bd871829805edbd5189f33c8e1a3d0", + "internal/session/session_test.go": "c6ad08eaac60a87fdd7f8d91cba655e53fd3baa9f0eb0b485b3f320ada89dbc0", + "internal/terminal/command_test.go": "398f1ff837521b9df08aafb0494e62b7a84fe24b7e9b678fb8d984d706ba9c0c", + "internal/terminal/editor.go": "b1458206089e21357d7849f62e9c52f04594e2dd3fd537c8f9a3efc5ac216b2f", + "internal/terminal/editor_test.go": "90cddfa47883b6f70430154fac434f2e80ef5c29781cffe747a32cd9c27f3d91", + "internal/terminal/layout_test.go": "3b8cda23a2ca9e8bb500629ea53c7c9d4ea1949d795fcdd58b6cb272e4686e5e", + "internal/terminal/recall_test.go": "fb9bd50a18640c98087ad644da7338b53a67d38ed8d2c4b569cf83aa3598ea92", + "internal/terminal/terminal.go": "9244f676c519f6603e11a96271088fa06217aea25dac050e3c137c02dcbd2c50", + "internal/terminal/terminal_test.go": "be4e71e8bb10e04882305ce6c37a860d429737f641c8d5a8f480245afc585368", + "internal/ui/robot.go": "750ea791c69c893be5ebaf8127a38d6d985caf0c30a87af26e0068bcb2f8a9a0", + "internal/ui/robot_test.go": "4cf558d5dceb5512344c525eedc55c76605c46c49f5488c433bbb2b5fc2bd23d", + "internal/ui/session.go": "e7a516035673ca7f502c6385ddf8c90508f5c5ca45599f935e02625053cdfc30", + "internal/update/replace.go": "fd08f5c5496733c22994625f667ce7c65ad831447aa9837662afdd4331c7ac66", + "internal/update/update.go": "01d36f3dc49b0a63a4851625e521cc7791e89b82ab160cd94ebbcee6994ce7ec", + "internal/update/update_test.go": "ee00392976ebd6bda2b6d550aac9bab6e626167007ea6f305c9b5a4bd9dccd04", + "internal/version/version.go": "d56319588513a57b56d9f96d3b240823704f45815d04f05fd4eb84bbef344a2a", + "internal/web/debug.go": "9a3c484a72db88b81ac2c99fdfb1590d7dc6da4c351198908389c8be97d7cf74", + "internal/web/errors.go": "23ecd7e57e2758b3919adcd04e45d8d723f81c9dce913d74b2232414ed61d09a", + "internal/web/errors_test.go": "89a81bb701962a0f58fde8297478595951d026d934d66aa6e36dcf3b8c9a8a14", + "internal/web/hub.go": "40181ce298494783de1bcbed29b6e5d3cef4e18f183b4affb393d1e57678c5f2", + "internal/web/hub_test.go": "2f0e9e0350007ff91231ffa95e8d5cebd255ce113ea97c3d2d343be32ddc85e9", + "internal/web/node.go": "a6fb4883ecfaaa040da13fb6e1a72c1e6dc7aa9b814e6f34b081b4a221703780", + "internal/web/process.go": "e5303d79b65d00783396ff07821b5f9d790a9bb51349dd596138af2ea654efce", + "internal/web/process_test.go": "9b760d6724f1b42ba6d2754ae50cbd3182bb66fb8e91c82bff0857cc476dba9d", + "internal/web/progress.go": "cc7392fb616770ffbc3dd7d1a3a109d880b1b296b81d4a2910d9535065733705", + "internal/web/proxy.go": "c6dea0777f662c752fd80416da652f5f91ab1e9421a967b255ed8bf456fe9fa1", + "internal/web/proxy_test.go": "9f88cd4e04ced059fd7676bcd7552626368236ece100fbe0ba6dddeb94ef252a", + "internal/web/scaffold.go": "37e7ed156ce015d0c1d27d41eedff2eb570fbaf3ad0c88171547a8681ee840e5", + "internal/web/scaffold_test.go": "ed123b2172ced924ea6697f6ec477954827cd0cbe59a1d8877f63884ef80435e", + "internal/web/server.go": "1cb60d9cc262abbad3c94918677277143b7d2128c4c75283a82522b67d4bd624", + "internal/web/server_test.go": "1eaaa857236b1159407239e714dc667afc4d20fbc98b7d68d85e83b80569040a", + "internal/web/static/app.js": "0a7134b55da5ed4e5bd520fb97739e1a9529736d9eb725627a6402f52a90ca23", + "internal/web/static/index.html": "f7007655503d31cf8479b5f82c13ed3c11a8c9b9bcf5e5a4cb34ee147562c752", + "internal/web/static/style.css": "c523bcd35c161dc4674242a77134ca829c2b561c62d4d7594cc9d4e3f1557561", + "internal/web/template/scratch/.gitignore": "54ee1981b52578d0cf0363fce5bf60275ea4535803b14200494d6d7b2f4c24c4", + "internal/web/template/scratch/ARCHITECTURE.md": "2d870ee2f5e5fe61ac4ab90e3d8c17b0e52bed5a08f761ad34dc671690c253d4", + "internal/web/template/scratch/backend/db.ts": "51bc2a3eca5199e25fbf75eb98515a8bddbea352199fe7d9d3c64caa3fa9daa9", + "internal/web/template/scratch/backend/decisions.ts": "6d3324b98f3d8fd81bf402342b3f0a747b31e9b7943b873ff15f00b128e53957", + "internal/web/template/scratch/backend/drizzle.config.ts": "f8adfa1cfbc0fdc69439743aa809c544a8b612639b97369ca7cc0bd7f8855029", + "internal/web/template/scratch/backend/index.ts": "e62ae6214990e5faadf43d78f12ec2f303f606cf0c6d8dd4185e16843048737d", + "internal/web/template/scratch/backend/llm.ts": "7bd4d94098372d056481578cc5351a1f36b9d439233d52e7076a9909cfce76bd", + "internal/web/template/scratch/components.json": "1f1a890f23e0652f738fc3cb7f27e19c1247b408f3ff6952d46b0fcd3c407dc7", + "internal/web/template/scratch/frontend/index.html": "8abeacc4fd6d946d4809f67c9c028ed1166693ae848cc8205b003988f4e14a2f", + "internal/web/template/scratch/frontend/src/App.tsx": "60ae902197171cddddf2e5c9910b6ecb37eafd36327c64d3cb96d086dd3979dd", + "internal/web/template/scratch/frontend/src/index.css": "1c04cbf678851b536cd79e81af1faaa9442ae2e41428ff02fd045109d25cbf74", + "internal/web/template/scratch/frontend/src/main.tsx": "6d64ef738d4185c22b804f5ec27ec5adc76d4c90690c91b06557d82961defa57", + "internal/web/template/scratch/frontend/vite.config.ts": "7ae7814da340134acae8eeddf9d6324c64f027734caccfe251511c6ca8fbb305", + "internal/web/template/scratch/package.json": "f4d252bc1e81be581d01c5d98de1264c0b380218ee2566c2650e5adb7dbd24e2", + "internal/web/template/scratch/scripts/restart.sh": "bd70dcd6a76080a47d81c4b73d1b6ce10adc2c2190200bb3b7dacf9fab0b05d4", + "internal/web/template/scratch/scripts/start.sh": "19a337d3fb7b6c40795d31e7d58fe07f5f4a25e689120f67254b4f41c8ebdad6", + "internal/web/template/scratch/scripts/stop.sh": "42e04dd08187317af9bf348754978f47c15c50e2a972db9281994c36b1cd58c9", + "internal/web/template/scratch/tsconfig.json": "036ab3904db11fbb7c6049090b22b0aeeaa846996f8dde38ad3cc1345ece1299", + "internal/web/template/scratch/yolocoder.json": "c225c559722ce4b494ff695ddcd4eda9382b2a8d5b0117706603ca3a30021960" + } + }, + "candidate": { + "revision": "cd513df46021ffa8d70cce846fcbdbc0cb82bad3", + "original_experiment_revision": "b1d5a3f229c0c5385fe829738da2b2c6fc323f71", + "version": "parallel-032", + "version_commit": "complete_scoped", + "flags": [ + "-X", + "github.com/mindsdb/yolocoder/internal/agent.editRouterModel=jev-1.13.0", + "-X", + "github.com/mindsdb/yolocoder/internal/agent.smallEditModel=muse-spark-1-3" + ], + "reported_binary_sha256": "87e4b8c30d08f4e164ef60c95c35088bcf7bc724d5e075dde7fc8fc4603871ed", + "source_sha256": { + "cmd/yolocoder/main.go": "5175151a82dd2a8349d0fc66bfb20fd96f3b9ee5351e16df029a1ce24eed46dd", + "go.mod": "899e81c23c1c7711ef47349ed376acc382bea479b2033291922b49861eea7b9a", + "go.sum": "fc37dfaee1833a1117fdecc9fc5cc82fb9605263303cf2d2c395f77320ad7bd1", + "internal/agent/agent.go": "04b93e70004da9743ec6cba26239255bd13b6c63aec97b9fe928d95cf77aab18", + "internal/agent/agent_test.go": "08cdc8f99cb1b020d5bbf10da25f6c526e2d4a9fe895feb13b62407ec2c6d0c3", + "internal/agent/chat.go": "f15b7a752a6d7a02dc4951c608b1e3bb44dac493c5a15808e093bc52bc4f69d0", + "internal/agent/chat_test.go": "a98bf9e201230693a47add18a016884638c5699b3186ffe9c04a0d8916fe82b6", + "internal/agent/client.go": "2c91472cf5a33ab6f2ddacd9d180b927d62b48a03f22b5db367b3caeb59f4880", + "internal/agent/client_test.go": "be923aa16aac5a21bafacdfc37cfedf06a4682dde71455ac440cb5412d0bbb36", + "internal/agent/completion_test.go": "440c6ff37233accee83ef0be872a0d34db945726968c47d56442bc6e083f497a", + "internal/agent/completion_whole_context_test.go": "3b6a29f9d35fe54ead18d16b320c0d11ce9f5186be19ebd0ff08ca8826702bab", + "internal/agent/context_prefetch_test.go": "cc8efc7d79ad8eb60234611e8bafdacda17ac56e98d74ef2c27115224f787dcf", + "internal/agent/cutoff_completion_test.go": "b907a84d9e42935dc4b05004d01de77a5112c3441aec7835b07a3ecee7f25e23", + "internal/agent/diagnostic_checkpoint.go": "fed850cea10540f69ca95d4c07147227bdc5cebfa0d95b744f2dccd8b523ac00", + "internal/agent/diagnostic_checkpoint_other.go": "0f8497c986ed253043ca3d675eb81c0868906f56ed83ea593b18a93b3e6eaff5", + "internal/agent/diagnostic_checkpoint_test.go": "04b4730afd742fad1322f8328184eb4221653e21345d921f7ad6e0fde2ed411d", + "internal/agent/diagnostic_checkpoint_unix.go": "feb783874f71cd012d78373a08784b0d4acb7a7578890a67725fe77cc56aae45", + "internal/agent/edit_feedback_test.go": "18962af4d28731978fb1d95918533c7b9566c9419f6cab2f544d92bb0cace127", + "internal/agent/edit_router.go": "a88b6aab18c1535a9f687e8be702bc5137a91338eabc50bb9d4654e952d3d675", + "internal/agent/edit_router_live_test.go": "3d98cad1b473d08b91528990f34e8746264785d9bce0fa6758dff09252c83c33", + "internal/agent/edit_router_recovery_test.go": "e63c55dad675813a2e7fdb1fbe45ca1d376d1b7512306b13579b6489b90b648a", + "internal/agent/edit_router_test.go": "4ef4edba361bef247981ee5396a03eba3d404103b4f539d7d872456cdcdfbb01", + "internal/agent/elapsed_feedback.go": "01e3f57e11a809b9d6077bf63afba38b854ec60dc70cf644c00405ddb0e983da", + "internal/agent/elapsed_feedback_test.go": "74c5048a804ee7abe413d9b400e73229a3b501f40eee607e900f7513c5bb4b20", + "internal/agent/loop_test.go": "c1fdc646a6b359d5c4aab6aada974c23a74b9c15f3eeaf43d98ac2df0bdeebfd", + "internal/agent/muse_recovery_test.go": "50c6e2e514769ef1d61d1ac0fd439f282215f58212bc178eab8006ab09a93576", + "internal/agent/needed_reads_test.go": "39194594df4a534601d3b0b4228a2520575a67ac155d9845c9c694e9eb881866", + "internal/agent/needed_whole_context_test.go": "efa059559870c44bc506bfcb2c88b2a0290418f3707fabe4d9aa8a06c4ff7894", + "internal/agent/profile.go": "b4d92697751aa88e0056b6e130d83f7d9d0476fa25422a5bc874fa9db8aca230", + "internal/agent/profile_test.go": "14e48055b57d4cd0ef636b8534da1b619e3388b0a95feb670b4449fdce8113fc", + "internal/agent/recall_test.go": "c50b5f8dd62f465505a062cee3bbe7fc47cb8de0789ae845bc950d8acd3b3787", + "internal/agent/rejected_paths.go": "6ba3dedfc1040fc6dbb1d9b62bf84e139886b920143f45113fbd18f7bc0c8769", + "internal/agent/rejected_paths_test.go": "84e495e016c688707c15fcdacad121bd7b41b89c8c3a0ac0012e52dc95012c4a", + "internal/agent/replace.go": "63a50e5a8c051cc1be90727542ee3ff6ba96a19163d0384daad65266e26f623f", + "internal/agent/replace_test.go": "e3ce5387092be78d40ef758d1db0463402a1ba9f442011ff794e33368b8f97ea", + "internal/agent/small_writer.go": "870b4d36acbe92585b753fdf2cdcda6b4119cdbd06e18e2ccc5cbb7a3654da43", + "internal/agent/small_writer_test.go": "32520a04d95458e47bfb6101773134c4f3b8dfc85c9a9653c1dc8c61aa1ed9c2", + "internal/agent/tester.go": "de28e3ec237e0cae02e67bf9224a83849bf04f36bf9e9258ee67bf85baa249dd", + "internal/agent/tester_test.go": "6914e401c949574454a9f3e9bcb9faa825f8e871e61a3b46d0d7ebedea249f75", + "internal/agent/transient_retry_test.go": "c1896ffdebaf32d1b2e0c9a499243a8f2bd5d9af26999b46d6ca0e747bfe7d6e", + "internal/agent/usage.go": "bd877466ae9ae4ecd7c16acc3bb7cedd01204bf1620cca2a15700acd580d7f82", + "internal/agent/usage_test.go": "d580e5579dc1b87f525361452b405ce3d15a77b05e94571c359781fdf264cf7a", + "internal/agent/whole_context.go": "4972d31c6fb0761def1605d17abd510c294ea68159d3063807511f2ff1be2573", + "internal/agent/whole_context_test.go": "849eaf2ec51b6a1b58cdbc2331d5c49fcf36033fa0eedc6a68710aa5a78315de", + "internal/app/app.go": "0e76977ff6d566bf6624ec3d3a93e6f1026f9c49e1069d5b90b0a62f07647594", + "internal/app/app_test.go": "98b216460d70998661e38a10633bc4034f5d81d7750df31c0e0ad777e8cd5987", + "internal/app/args.go": "c68c692bc3df21f57d0721a2b5645c667f544a10e78b52eb1ad5f69e6f83ffb5", + "internal/app/args_test.go": "c6ed8da2196123aeca659ca601b632d2b24ad80634a3f406bb7fc8ec2adb03b9", + "internal/app/coding.go": "b1c980dabcf393f187a1b6b844337055d8f2a2f60f8eb40bb30429e65bd9e0e9", + "internal/auth/auth.go": "4c15d81a0fd9fb1f3a5bbd4f76754fd7b28e910b85415ca85359d4d87e9dca36", + "internal/config/config.go": "c7fcbf876a648e2a9a73cfc2beb6947a530f06b0be82f5a626c33ded946302d7", + "internal/config/config_test.go": "065e010e726e6cc29e324028d6da74c07e5a00dbdce6c095dbabf5a27dcc7b9a", + "internal/config/providers.go": "0c019d1cbae9893a05976b7223c9313a5d094c36d424846346158fcf7fe2c339", + "internal/debug/debug.go": "6c9ca85966234c35b3e515a3404e0d382f59eeaee3a0e646856124a58a33144d", + "internal/debug/debug_test.go": "8de060512078a0bb6b36f3944f2363427c47bbd1bbd0cc28f6ccb28cd9851044", + "internal/repo/anchor_contract_test.go": "e6582ce79ddf759a40bbad0cfa76e78ffe18402a56e110e5d37430a3a08f3cf4", + "internal/repo/anchor_repair_test.go": "47fcaa1373bf0039298fc4e9487b83c0a8705d2828478f560124c5faf7377b6d", + "internal/repo/compact.go": "3a44fb4822bd23a0e941a9210cf83725cc12860110b555cee5b2cf3be13cd980", + "internal/repo/compact_test.go": "34d46ede502c4096f999f109ca0055202c334dbe38fe8ec28c552d797c6bf517", + "internal/repo/patch.go": "f6524d49fe973ecb4050b547f82f9e0e392a455e948852d33fad606b2344f9d3", + "internal/repo/patch_test.go": "8262af45eef0c50875da06ba79a807a0d0bdc5c8f02774793f86a3a2395c6f02", + "internal/repo/replace.go": "135d84e4480c6c876bc3a0180aa2a12944fb3ac52cd3796665d5330b9a988290", + "internal/repo/replace_test.go": "f76a84f87a2f3ada2b7955ab9375afbcadc8f86c0ce402f28748162858fda765", + "internal/repo/repo.go": "b5e1e28874e669280dca04de42dcda7570d6e85bc2073bff3bec1a90062b538a", + "internal/repo/repo_test.go": "f3afa129ae27a8de7413bde2b31cae68437ff864b5cfc237766e566d1e3e7d7a", + "internal/session/session.go": "4e4335e282f8032db7d014ece1fee21b94bd871829805edbd5189f33c8e1a3d0", + "internal/session/session_test.go": "c6ad08eaac60a87fdd7f8d91cba655e53fd3baa9f0eb0b485b3f320ada89dbc0", + "internal/terminal/command_test.go": "398f1ff837521b9df08aafb0494e62b7a84fe24b7e9b678fb8d984d706ba9c0c", + "internal/terminal/editor.go": "b1458206089e21357d7849f62e9c52f04594e2dd3fd537c8f9a3efc5ac216b2f", + "internal/terminal/editor_test.go": "90cddfa47883b6f70430154fac434f2e80ef5c29781cffe747a32cd9c27f3d91", + "internal/terminal/layout_test.go": "3b8cda23a2ca9e8bb500629ea53c7c9d4ea1949d795fcdd58b6cb272e4686e5e", + "internal/terminal/recall_test.go": "fb9bd50a18640c98087ad644da7338b53a67d38ed8d2c4b569cf83aa3598ea92", + "internal/terminal/terminal.go": "9244f676c519f6603e11a96271088fa06217aea25dac050e3c137c02dcbd2c50", + "internal/terminal/terminal_test.go": "be4e71e8bb10e04882305ce6c37a860d429737f641c8d5a8f480245afc585368", + "internal/ui/robot.go": "750ea791c69c893be5ebaf8127a38d6d985caf0c30a87af26e0068bcb2f8a9a0", + "internal/ui/robot_test.go": "4cf558d5dceb5512344c525eedc55c76605c46c49f5488c433bbb2b5fc2bd23d", + "internal/ui/session.go": "e7a516035673ca7f502c6385ddf8c90508f5c5ca45599f935e02625053cdfc30", + "internal/update/replace.go": "fd08f5c5496733c22994625f667ce7c65ad831447aa9837662afdd4331c7ac66", + "internal/update/update.go": "01d36f3dc49b0a63a4851625e521cc7791e89b82ab160cd94ebbcee6994ce7ec", + "internal/update/update_test.go": "ee00392976ebd6bda2b6d550aac9bab6e626167007ea6f305c9b5a4bd9dccd04", + "internal/version/version.go": "d56319588513a57b56d9f96d3b240823704f45815d04f05fd4eb84bbef344a2a", + "internal/web/debug.go": "9a3c484a72db88b81ac2c99fdfb1590d7dc6da4c351198908389c8be97d7cf74", + "internal/web/errors.go": "23ecd7e57e2758b3919adcd04e45d8d723f81c9dce913d74b2232414ed61d09a", + "internal/web/errors_test.go": "89a81bb701962a0f58fde8297478595951d026d934d66aa6e36dcf3b8c9a8a14", + "internal/web/hub.go": "40181ce298494783de1bcbed29b6e5d3cef4e18f183b4affb393d1e57678c5f2", + "internal/web/hub_test.go": "2f0e9e0350007ff91231ffa95e8d5cebd255ce113ea97c3d2d343be32ddc85e9", + "internal/web/node.go": "a6fb4883ecfaaa040da13fb6e1a72c1e6dc7aa9b814e6f34b081b4a221703780", + "internal/web/process.go": "e5303d79b65d00783396ff07821b5f9d790a9bb51349dd596138af2ea654efce", + "internal/web/process_test.go": "9b760d6724f1b42ba6d2754ae50cbd3182bb66fb8e91c82bff0857cc476dba9d", + "internal/web/progress.go": "cc7392fb616770ffbc3dd7d1a3a109d880b1b296b81d4a2910d9535065733705", + "internal/web/proxy.go": "c6dea0777f662c752fd80416da652f5f91ab1e9421a967b255ed8bf456fe9fa1", + "internal/web/proxy_test.go": "9f88cd4e04ced059fd7676bcd7552626368236ece100fbe0ba6dddeb94ef252a", + "internal/web/scaffold.go": "37e7ed156ce015d0c1d27d41eedff2eb570fbaf3ad0c88171547a8681ee840e5", + "internal/web/scaffold_test.go": "ed123b2172ced924ea6697f6ec477954827cd0cbe59a1d8877f63884ef80435e", + "internal/web/server.go": "1cb60d9cc262abbad3c94918677277143b7d2128c4c75283a82522b67d4bd624", + "internal/web/server_test.go": "1eaaa857236b1159407239e714dc667afc4d20fbc98b7d68d85e83b80569040a", + "internal/web/static/app.js": "0a7134b55da5ed4e5bd520fb97739e1a9529736d9eb725627a6402f52a90ca23", + "internal/web/static/index.html": "f7007655503d31cf8479b5f82c13ed3c11a8c9b9bcf5e5a4cb34ee147562c752", + "internal/web/static/style.css": "c523bcd35c161dc4674242a77134ca829c2b561c62d4d7594cc9d4e3f1557561", + "internal/web/template/scratch/.gitignore": "54ee1981b52578d0cf0363fce5bf60275ea4535803b14200494d6d7b2f4c24c4", + "internal/web/template/scratch/ARCHITECTURE.md": "2d870ee2f5e5fe61ac4ab90e3d8c17b0e52bed5a08f761ad34dc671690c253d4", + "internal/web/template/scratch/backend/db.ts": "51bc2a3eca5199e25fbf75eb98515a8bddbea352199fe7d9d3c64caa3fa9daa9", + "internal/web/template/scratch/backend/decisions.ts": "6d3324b98f3d8fd81bf402342b3f0a747b31e9b7943b873ff15f00b128e53957", + "internal/web/template/scratch/backend/drizzle.config.ts": "f8adfa1cfbc0fdc69439743aa809c544a8b612639b97369ca7cc0bd7f8855029", + "internal/web/template/scratch/backend/index.ts": "e62ae6214990e5faadf43d78f12ec2f303f606cf0c6d8dd4185e16843048737d", + "internal/web/template/scratch/backend/llm.ts": "7bd4d94098372d056481578cc5351a1f36b9d439233d52e7076a9909cfce76bd", + "internal/web/template/scratch/components.json": "1f1a890f23e0652f738fc3cb7f27e19c1247b408f3ff6952d46b0fcd3c407dc7", + "internal/web/template/scratch/frontend/index.html": "8abeacc4fd6d946d4809f67c9c028ed1166693ae848cc8205b003988f4e14a2f", + "internal/web/template/scratch/frontend/src/App.tsx": "60ae902197171cddddf2e5c9910b6ecb37eafd36327c64d3cb96d086dd3979dd", + "internal/web/template/scratch/frontend/src/index.css": "1c04cbf678851b536cd79e81af1faaa9442ae2e41428ff02fd045109d25cbf74", + "internal/web/template/scratch/frontend/src/main.tsx": "6d64ef738d4185c22b804f5ec27ec5adc76d4c90690c91b06557d82961defa57", + "internal/web/template/scratch/frontend/vite.config.ts": "7ae7814da340134acae8eeddf9d6324c64f027734caccfe251511c6ca8fbb305", + "internal/web/template/scratch/package.json": "f4d252bc1e81be581d01c5d98de1264c0b380218ee2566c2650e5adb7dbd24e2", + "internal/web/template/scratch/scripts/restart.sh": "bd70dcd6a76080a47d81c4b73d1b6ce10adc2c2190200bb3b7dacf9fab0b05d4", + "internal/web/template/scratch/scripts/start.sh": "19a337d3fb7b6c40795d31e7d58fe07f5f4a25e689120f67254b4f41c8ebdad6", + "internal/web/template/scratch/scripts/stop.sh": "42e04dd08187317af9bf348754978f47c15c50e2a972db9281994c36b1cd58c9", + "internal/web/template/scratch/tsconfig.json": "036ab3904db11fbb7c6049090b22b0aeeaa846996f8dde38ad3cc1345ece1299", + "internal/web/template/scratch/yolocoder.json": "c225c559722ce4b494ff695ddcd4eda9382b2a8d5b0117706603ca3a30021960" + } + }, + "coding_model": "muse-spark-1-3", + "candidate_router_model": "jev-1.13.0", + "note": "The clean candidate snapshot has identical tracked cmd/, internal/, go.mod and go.sum contents to the frozen winner. Rebuilt binaries have new platform/toolchain/VCS hashes; benchmark them as a new campaign." +} diff --git a/evals/browser/checks.mjs b/evals/browser/checks.mjs new file mode 100644 index 0000000..a8ab4e6 --- /dev/null +++ b/evals/browser/checks.mjs @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict'; +import {expect as baseExpect} from '@playwright/test'; +const expect=baseExpect.configure({timeout:3000}); +export const headings={csv:'CSV explorer',quiz:'Quiz builder',inventory:'Inventory manager',notes:'Notes editor',application:'Application form'}; +export const editedHeadings={csv:'Data explorer',quiz:'Knowledge studio',inventory:'Stock room',notes:'My notebook',application:'Join our programme'}; +const primary={csv:'Import CSV',quiz:'Add question',inventory:'Add item',notes:'Save note',application:'Save draft'}; +const routes={csv:'/api/datasets',quiz:'/api/questions',inventory:'/api/items',notes:'/api/notes',application:'/api/applications'}; +export function control(page,name){ + const options={name,exact:true}; + const labelled=page.getByLabel(name,{exact:true}); + if(['Correct answer','Sort column','Sort direction','Track','Note view'].includes(name)||name.startsWith('Answer '))return page.getByRole('combobox',options).or(labelled); + if(name==='Low stock only')return page.getByRole('checkbox',options).or(labelled); + return page.getByRole('textbox',options).or(page.getByRole('searchbox',options)).or(page.getByRole('spinbutton',options)).or(labelled); +} +export function checksFor(app,kind,{page,url,api,restart,seed,check}){ + let serial=0;const name=()=>`Eval-${seed}-${++serial}`; + const go=()=>page.goto(url,{waitUntil:'networkidle'}); + const button=n=>page.getByRole('button',{name:n,exact:true}); + const label=n=>control(page,n); + const status=(r,s)=>assert.equal(r.status,s,JSON.stringify(r.body)); + const post=(p,b)=>api('POST',p,b), patch=(p,b)=>api('PATCH',p,b); + const create=async body=>{const r=await post(routes[app],body);status(r,201);return r.body;}; + const list=async()=>{const r=await api('GET',routes[app]);status(r,200);assert.ok(Array.isArray(r.body));return r.body;}; + const row=async id=>(await list()).find(r=>r.id===id); + const alert=()=>page.getByRole('alert').filter({hasText:/\S/}).first(); + check('ui.contract','regression',true,async()=>{await go();await expect(page.getByRole('heading',{level:1,name:kind==='edit'?editedHeadings[app]:headings[app],exact:true})).toBeVisible();await expect(button(primary[app])).toBeVisible();if(kind==='edit'){assert.equal(await button(primary[app]).evaluate(e=>getComputedStyle(e).backgroundColor),'rgb(124, 58, 237)');for(const other of await page.getByRole('button').all()){if(await other.getAttribute('aria-label')===primary[app]||(await other.innerText()).trim()===primary[app])continue;assert.equal(await other.evaluate(e=>getComputedStyle(e).backgroundColor),'rgb(37, 99, 235)','Unrequested secondary button colour changed');}}}); + + if(app==='csv'){ + check('csv.parser_and_validation','regression',true,async()=>{ + const data=await create({name:' '+name()+' ',csv:'Name,Note,Qty\r\n"Alpha, Inc","He said ""hi""",10\r\nBeta,"Line one\nLine two",2\r\n'}); + assert.equal(data.name,data.name.trim());assert.deepEqual(data.columns,['Name','Note','Qty']);assert.deepEqual(data.rows,[['Alpha, Inc','He said "hi"','10'],['Beta','Line one\nLine two','2']]); + const n=(await list()).length; + for(const csv of ['A,A\nx,y','A,\nx,y','A,B\nx','A,B\n"x,y','A,B','A,B\nx,y,z'])status(await post(routes[app],{name:name(),csv}),400); + status(await post(routes[app],{name:' ',csv:'A\nx'}),400);assert.equal((await list()).length,n); + }); + check('csv.ui_import','acceptance',true,async()=>{await go();const n=name();await label('Dataset name').fill(n);await label('CSV').fill('City,Count\nParis,12\nLondon,4');await button('Import CSV').click();await expect(page.getByRole('table',{name:'Data',exact:true})).toContainText('Paris');await page.reload();await button('Open '+n).click();await expect(page.getByRole('table',{name:'Data',exact:true})).toContainText('London');}); + check('bug.csv','acceptance',true,async()=>{const d=await create({name:name(),csv:'Name,Colour\nAlpha,Blue\nBeta,RED'});await go();await button('Open '+d.name).click();const table=page.getByRole('table',{name:'Data',exact:true});await label('Search rows').fill('aLpHa');await expect(table).toContainText('Alpha');await expect(table).not.toContainText('Beta');await label('Search rows').fill('red');await expect(table).toContainText('Beta');await expect(table).not.toContainText('Alpha');await label('Search rows').fill('');await expect(table).toContainText('Alpha');assert.deepEqual((await row(d.id)).rows,d.rows);}); + if(kind==='feature')check('feature.csv','acceptance',true,async()=>{const d=await create({name:name(),csv:'Name,Qty\nBeta,10\nAlpha,2\nGamma,-1'}),other=await create({name:name(),csv:'Letter\nz\na'});await go();await button('Open '+d.name).click();const cells=async()=>{const values=[];for(const r of (await page.getByRole('table',{name:'Data'}).getByRole('row').all()).slice(1)){if(await r.getByRole('cell').count())values.push(await r.getByRole('cell').first().textContent());}return values;};await label('Sort column').selectOption({label:'Qty'});assert.deepEqual(await cells(),['Gamma','Alpha','Beta']);await label('Sort direction').selectOption({label:'Descending'});assert.deepEqual(await cells(),['Beta','Alpha','Gamma']);await label('Search rows').fill('Alpha');assert.deepEqual(await cells(),['Alpha']);await label('Search rows').fill('');await label('Sort column').selectOption({label:'Name'});assert.deepEqual(await cells(),['Gamma','Beta','Alpha']);await label('Sort column').selectOption({label:'None'});assert.deepEqual(await cells(),['Beta','Alpha','Gamma']);assert.deepEqual((await row(d.id)).rows,d.rows);await button('Open '+other.name).click();await expect(label('Search rows')).toHaveValue('');await expect(label('Sort column').locator('option:checked')).toHaveText('None');}); + } + if(app==='quiz'){ + const question=(correctIndex=0)=>({prompt:name(),options:['First','Second'],correctIndex}); + check('quiz.validation','regression',true,async()=>{const n=(await list()).length;for(const b of [{prompt:' '},{options:['']},{options:['A',' ']},{options:['A','B','C']},{correctIndex:2},{correctIndex:'0'}])status(await post(routes[app],{...question(),...b}),400);assert.equal((await list()).length,n);const q=await create({...question(),prompt:' '+name()+' ',options:[' A ',' B ']});assert.equal(q.prompt,q.prompt.trim());assert.deepEqual(q.options,['A','B']);for(const answers of [null,{},[{id:q.id,choice:-1}],[{id:999999,choice:0}],[{id:q.id,choice:0},{id:q.id,choice:1}]])status(await post('/api/grade',{answers}),400);const r=await post('/api/grade',{answers:[]});status(r,200);assert.deepEqual(r.body,{correct:0,total:0});}); + check('bug.quiz','acceptance',true,async()=>{const a=await create(question(0)),b=await create(question(1));for(const [choices,expected] of [[[0,1],2],[[1,0],0],[[0,0],1]]){const r=await post('/api/grade',{answers:[{id:a.id,choice:choices[0]},{id:b.id,choice:choices[1]}]});status(r,200);assert.deepEqual(r.body,{correct:expected,total:2});}}); + check('quiz.ui_practice','acceptance',true,async()=>{await go();const n=name();await label('Question').fill(n);await label('Option A').fill('Left');await label('Option B').fill('Right');await label('Correct answer').selectOption({label:'B'});await button('Add question').click();await expect(page.getByRole('list',{name:'Questions'})).toContainText(n);await page.reload();await expect(page.getByRole('list',{name:'Questions'})).toContainText(n);const qs=await list();await button('Practice').click();for(const q of qs)await label('Answer '+q.prompt).selectOption({label:'B'});await button('Check answers').click();await expect(page.getByRole('status')).toContainText(`Score: ${qs.filter(q=>q.correctIndex===1).length}/${qs.length}`);}); + if(kind==='feature')check('feature.quiz','acceptance',true,async()=>{assert.equal((await row(1)).explanation,'');const explanation='Because '+name();const n=(await list()).length;status(await post(routes[app],{...question(),explanation:12}),400);assert.equal((await list()).length,n);const q=await create(question());assert.equal(q.explanation,'');await go();const title=name();await label('Question').fill(title);await label('Option A').fill('Yes');await label('Option B').fill('No');await label('Explanation').fill(explanation);await button('Add question').click();await expect(page.getByRole('list',{name:'Questions'})).toContainText(title);await page.reload();await button('Practice').click();await expect(page.getByText(explanation,{exact:true})).not.toBeVisible();for(const q of await list())await label('Answer '+q.prompt).selectOption({label:'A'});await button('Check answers').click();await expect(page.getByText(explanation,{exact:true})).toBeVisible();assert.equal((await list()).find(q=>q.prompt===title).explanation,explanation);}); + } + if(app==='inventory'){ + const item=(quantity=7)=>({sku:name(),name:name(),quantity}); + check('inventory.validation','regression',true,async()=>{const a=await create(item());for(const b of [{sku:' '},{name:' '},{quantity:-1},{quantity:1.2},{quantity:'2'}])status(await post(routes[app],{...item(),...b}),400);status(await post(routes[app],{...item(),sku:a.sku.toLowerCase()}),409);for(const delta of [0,1.5,'2',null])status(await post(`/api/items/${a.id}/adjust`,{delta}),400);status(await post('/api/items/999999/adjust',{delta:1}),404);const r=await post(`/api/items/${a.id}/adjust`,{delta:3});status(r,200);assert.equal(r.body.quantity,10);assert.equal((await row(a.id)).quantity,10);}); + check('bug.inventory','acceptance',true,async()=>{const a=await create(item(3));status(await post(`/api/items/${a.id}/adjust`,{delta:-4}),409);assert.equal((await row(a.id)).quantity,3);const rs=await Promise.all([1,2].map(()=>post(`/api/items/${a.id}/adjust`,{delta:-2})));assert.deepEqual(rs.map(r=>r.status).sort(),[200,409]);assert.equal((await row(a.id)).quantity,1);await go();await label('Adjustment '+a.sku).fill('-2');await button('Adjust '+a.sku).click();await expect(alert()).toBeVisible();assert.equal((await row(a.id)).quantity,1);}); + check('inventory.ui_and_concurrency','acceptance',true,async()=>{await go();const a=item(10);for(const [l,v] of [['SKU',a.sku],['Item name',a.name],['Quantity','10']])await label(l).fill(v);await button('Add item').click();const table=page.getByRole('table',{name:'Inventory'});await expect(table).toContainText(a.sku);const added=(await list()).find(x=>x.sku===a.sku);const rs=await Promise.all([1,2,3].map(()=>post(`/api/items/${added.id}/adjust`,{delta:2})));rs.forEach(r=>status(r,200));assert.equal((await row(added.id)).quantity,16);await page.reload();await label('Adjustment '+a.sku).fill('-6');await button('Adjust '+a.sku).click();await expect.poll(async()=>(await row(added.id)).quantity).toBe(10);await page.reload();await expect(table).toContainText(a.name);}); + if(kind==='feature')check('feature.inventory','acceptance',true,async()=>{const low=await create(item(5)),high=await create(item(6));await go();await expect(label('Low stock threshold')).toHaveValue('5');await label('Low stock only').check();const table=page.getByRole('table',{name:'Inventory'});await expect(table).toContainText(low.sku);await expect(table).not.toContainText(high.sku);await label('Adjustment '+low.sku).fill('1');await button('Adjust '+low.sku).click();await expect(table).not.toContainText(low.sku);await label('Low stock threshold').fill('6');await expect(table).toContainText(high.sku);await label('Low stock threshold').fill('-1');await expect(alert()).toBeVisible();await label('Low stock threshold').fill('5');await label('Low stock only').uncheck();await expect(table).toContainText(high.sku);assert.equal((await row(high.id)).quantity,6);}); + } + if(app==='notes'){ + const note=()=>({title:name(),body:'Line one\n literal\n'}); + check('notes.validation_and_text','regression',true,async()=>{for(const b of [{title:' ',body:''},{title:'Valid',body:0}])status(await post(routes[app],b),400);const a=await create(note());for(const b of [{},{title:''},{body:null},{title:'Changed',body:12}])status(await patch(`/api/notes/${a.id}`,b),400);assert.deepEqual(await row(a.id),a);status(await patch('/api/notes/999999',{body:''}),404);await go();const article=page.getByRole('article',{name:a.title,exact:true});await expect(article).toContainText('literal');assert.equal(await article.locator('b').count(),0);}); + check('bug.notes','acceptance',true,async()=>{const a=await create(note());let r=await patch(`/api/notes/${a.id}`,{title:' '+name()+' '});status(r,200);assert.equal(r.body.body,a.body);assert.equal(r.body.title,r.body.title.trim());const title=r.body.title;r=await patch(`/api/notes/${a.id}`,{body:' replacement\n'});status(r,200);assert.equal(r.body.title,title);assert.equal(r.body.body,' replacement\n');}); + check('notes.ui_edit_and_search','acceptance',true,async()=>{await go();const title=name(),body='SpecialWORD \n next';await label('Note title').fill(title);await label('Note body').fill(body);await button('Save note').click();await expect(page.getByRole('article',{name:title,exact:true})).toBeVisible();const count=(await list()).length;await button('Edit '+title).click();await expect(label('Note body')).toHaveValue(body);await label('Note title').fill(title+' revised');await button('Save note').click();await expect(page.getByRole('article',{name:title+' revised',exact:true})).toBeVisible();assert.equal((await list()).length,count);await label('Search notes').fill('specialword');await expect(page.getByRole('article',{name:title+' revised',exact:true})).toBeVisible();await label('Search notes').fill('NoSuchText987');await expect(page.getByRole('article')).toHaveCount(0);await label('Search notes').fill('');await button('New note').click();await expect(label('Note title')).toHaveValue('');await expect(label('Note body')).toHaveValue('');await page.reload();await expect(page.getByRole('article',{name:title+' revised',exact:true})).toContainText('SpecialWORD');}); + if(kind==='feature')check('feature.notes','acceptance',true,async()=>{assert.equal((await row(1)).archived,false);const a=await create(note());assert.equal(a.archived,false);status(await patch(`/api/notes/${a.id}`,{archived:'true'}),400);await go();await button('Archive '+a.title).click();await expect(page.getByRole('article',{name:a.title,exact:true})).not.toBeVisible();await label('Note view').selectOption({label:'Archived'});await expect(page.getByRole('article',{name:a.title,exact:true})).toBeVisible();await label('Search notes').fill('noSuchArchived123');await expect(page.getByRole('article')).toHaveCount(0);await label('Search notes').fill('');assert.deepEqual(await row(a.id),{...a,archived:true});await page.reload();await label('Note view').selectOption({label:'All'});await button('Restore '+a.title).click();await expect.poll(async()=>(await row(a.id)).archived).toBe(false);await label('Note view').selectOption({label:'Active'});await expect(page.getByRole('article',{name:a.title,exact:true})).toBeVisible();const archived=await patch(`/api/notes/${a.id}`,{archived:true,body:a.body});status(archived,200);assert.equal(archived.body.archived,true);}); + } + if(app==='application'){ + const data=()=>({name:name(),email:'new@example.com',track:'Engineering',statement:'A sufficiently detailed statement.'}); + const step1=async d=>{await label('Full name').fill(d.name);await label('Email').fill(d.email);await button('Next').click();}; + const step2=async d=>{await label('Track').selectOption({label:d.track});await label('Statement').fill(d.statement);await button('Next').click();}; + check('application.api_validation','regression',true,async()=>{status(await post(routes[app],{...data(),name:null}),400);const incomplete=await create({name:'',email:'bad',track:'',statement:''});status(await post(`/api/applications/${incomplete.id}/submit`),400);status(await api('PUT',`/api/applications/${incomplete.id}`,{...data(),statement:4}),400);assert.deepEqual(await row(incomplete.id),incomplete);status(await post('/api/applications/999999/submit'),404);status(await api('PUT','/api/applications/999999',data()),404);for(const fields of [{name:' '},{email:'bad'},{track:'Invalid'},{statement:'short'}]){const d=await create({...data(),...fields});status(await post(`/api/applications/${d.id}/submit`),400);}}); + check('application.submission_immutable','regression',true,async()=>{const a=await create(data());let r=await post(`/api/applications/${a.id}/submit`);status(r,200);assert.equal(r.body.status,'submitted');const saved=r.body;status(await api('PUT',`/api/applications/${a.id}`,data()),409);status(await post(`/api/applications/${a.id}/submit`),409);assert.deepEqual(await row(a.id),saved);}); + check('bug.application','acceptance',true,async()=>{const d=data();await go();await step1(d);await step2(d);await expect(page.getByRole('region',{name:'Review',exact:true})).toContainText(d.statement);await button('Back').click();await expect(label('Statement')).toHaveValue(d.statement);await expect(label('Track').locator('option:checked')).toHaveText(d.track);await button('Back').click();await expect(label('Full name')).toHaveValue(d.name);await expect(label('Email')).toHaveValue(d.email);}); + check('application.ui_draft_and_submit','acceptance',true,async()=>{await go();await label('Full name').fill('');await label('Email').fill('invalid');await button('Next').click();await expect(label('Full name')).toBeVisible();await expect(alert()).toBeVisible();const d=data();await step1(d);await label('Statement').fill('short');await button('Next').click();await expect(label('Statement')).toBeVisible();await expect(alert()).toBeVisible();await label('Track').selectOption({label:d.track});await label('Statement').fill(d.statement);await button('Save draft').click();await expect.poll(async()=>(await list()).some(r=>r.name===d.name&&r.statement===d.statement&&r.status==='draft')).toBe(true);await page.reload();await expect(label('Full name')).toHaveValue(d.name);await button('Next').click();await expect(label('Statement')).toHaveValue(d.statement);await button('Next').click();await expect(page.getByRole('region',{name:'Review'})).toContainText(d.name);await button('Submit application').click();await expect(page.getByRole('status')).toContainText('Application submitted');await expect(page.getByRole('list',{name:'Submitted applications'})).toContainText(d.name);await page.reload();await expect(page.getByRole('list',{name:'Submitted applications'})).toContainText(d.name);}); + if(kind==='feature')check('feature.application','acceptance',true,async()=>{const a=await create(data());status(await post(`/api/applications/${a.id}/duplicate`),409);status(await post('/api/applications/999999/duplicate'),404);status(await post(`/api/applications/${a.id}/submit`),200);const source=await row(a.id);await go();await button('Duplicate '+a.name).click();await expect(label('Full name')).toHaveValue(a.name);await expect.poll(async()=>(await list()).filter(x=>x.id!==a.id&&x.name===a.name&&x.status==='draft').length).toBe(1);const copy=(await list()).find(x=>x.id!==a.id&&x.name===a.name&&x.status==='draft');for(const key of ['name','email','track','statement'])assert.equal(copy[key],source[key]);await label('Full name').fill(a.name+' copy');await button('Save draft').click();await expect.poll(async()=>(await row(copy.id)).name).toBe(a.name+' copy');assert.deepEqual(await row(a.id),source);await page.reload();await expect(label('Full name')).toHaveValue(a.name+' copy');}); + } + check('storage.backend_restart','regression',true,async()=>{const before=await list();assert.ok(before.length>0);await restart();assert.deepEqual((await list()).sort((a,b)=>a.id-b.id),before.sort((a,b)=>a.id-b.id));}); + if(kind!=='create')check('storage.existing_records','regression',true,async()=>{ + const expected={csv:{id:1,name:'Existing dataset',columns:['Name','Value'],rows:[['Original','07']]},quiz:{id:1,prompt:'Existing question',options:['Old A','Old B'],correctIndex:0},inventory:{id:1,sku:'OLD-001',name:'Existing item',quantity:7},notes:{id:1,title:'Existing note',body:'Original body literal'},application:{id:1,name:'Existing applicant',email:'old@example.com',track:'Design',statement:'Original submitted statement',status:'submitted'}}[app];const existing=await row(1);assert.ok(existing,'Existing record lost');for(const [k,v]of Object.entries(expected))assert.deepEqual(existing[k],v,'Existing field changed: '+k); + }); + check('ui.mobile_fit','usability',true,async()=>{await page.setViewportSize({width:390,height:844});await go();await expect(button(primary[app])).toBeVisible();assert.ok(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth+1),'Page overflows a 390px viewport');}); +} diff --git a/evals/browser/drag.mjs b/evals/browser/drag.mjs new file mode 100644 index 0000000..97a1459 --- /dev/null +++ b/evals/browser/drag.mjs @@ -0,0 +1,48 @@ +// Select a grab point without interacting, then perform one real pointer drag. +// The target and subsequent acceptance assertions remain the caller's concern. +export async function dragFromGrabArea(article, target, title) { + const deadline = Date.now() + 3000; + const remaining = () => { + const ms = deadline - Date.now(); + if (ms <= 0) throw new Error('No usable card grab area within the 3000ms drag budget'); + return ms; + }; + await article.scrollIntoViewIfNeeded({ timeout: remaining() }); + const sourcePosition = await article.evaluate((source, preferredText) => { + const box = source.getBoundingClientRect(); + const style = getComputedStyle(source); + const origin = { x: box.x + parseFloat(style.borderLeftWidth), y: box.y + parseFloat(style.borderTopWidth) }; + const interactive = 'input,select,textarea,button,a,label,summary,iframe,object,embed,audio,video,[contenteditable]:not([contenteditable="false"]),[role="button"],[role="link"],[role="textbox"],[role="combobox"],[role="listbox"],[role="option"],[role="checkbox"],[role="radio"],[role="switch"],[role="slider"],[role="spinbutton"],[role="menuitem"],[role="tab"]'; + const usable = (x, y) => { + x = Math.floor(x); y = Math.floor(y); + if (x <= origin.x || y <= origin.y || x >= box.right || y >= box.bottom || x < 0 || y < 0 || x >= innerWidth || y >= innerHeight) return null; + const hit = document.elementFromPoint(x, y); + if (!hit || !source.contains(hit) || hit.closest(interactive) || hit.isContentEditable) return null; + for (let node = hit; node; node = node.parentElement) { + const css = getComputedStyle(node); + if (css.visibility !== 'visible' || css.display === 'none' || Number(css.opacity) === 0) return null; + } + return { x: x - origin.x, y: y - origin.y }; + }; + // A visible title is a natural grab area even when the card center is a select. + // Bound traversal; unusual markup can still use the generic interior samples. + const normalize = text => text.trim().replace(/\s+/g, ' '); + const walker = document.createTreeWalker(source, NodeFilter.SHOW_TEXT); + for (let n = 0, node; n < 256 && (node = walker.nextNode()); n++) { + if (!preferredText || normalize(node.textContent) !== normalize(preferredText)) continue; + const range = document.createRange(); range.selectNodeContents(node); + for (const rect of Array.from(range.getClientRects()).slice(0, 16)) { + if (rect.width <= 0 || rect.height <= 0) continue; + const point = usable(rect.x + rect.width / 2, rect.y + rect.height / 2); + if (point) return point; + } + } + for (const fy of [0.5, 0.2, 0.8]) for (const fx of [0.5, 0.2, 0.8]) { + const point = usable(box.x + box.width * fx, box.y + box.height * fy); + if (point) return point; + } + return null; + }, title, { timeout: remaining() }); + if (!sourcePosition) throw new Error('No unobstructed noninteractive grab area in card'); + await article.dragTo(target, { sourcePosition, timeout: remaining() }); +} diff --git a/evals/browser/grade.mjs b/evals/browser/grade.mjs new file mode 100644 index 0000000..bbf7db8 --- /dev/null +++ b/evals/browser/grade.mjs @@ -0,0 +1,38 @@ +// Runs in an evaluator-owned process. This source is never in the agent cwd. +import { chromium } from '@playwright/test'; +import { writeFile, mkdir } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { checksFor } from './checks.mjs'; + +const [app,kind,url,controlUrl,output,seed='1'] = process.argv.slice(2); +if(!output) throw new Error('app kind app-url control-url output-dir seed required'); +await mkdir(output,{recursive:true}); +const browser=await chromium.launch({headless:true}); +const context=await browser.newContext({viewport:{width:1280,height:900},locale:'en-GB',timezoneId:'UTC'}); +const page=await context.newPage();page.setDefaultTimeout(3000);page.setDefaultNavigationTimeout(12000); +const errors=[];page.on('pageerror',e=>errors.push(e.message)); +const tests=[];const results=[]; +async function api(method,path,body) { + const r=await fetch(url+path,{method,headers:body?{'Content-Type':'application/json'}:{},body:body?JSON.stringify(body):undefined,signal:AbortSignal.timeout(10000)}); + const text=await r.text();let decoded;try{decoded=JSON.parse(text);}catch{decoded=text;} + return {status:r.status,body:decoded}; +} +async function restart() { + const r=await fetch(controlUrl+'/process/restart',{method:'POST',signal:AbortSignal.timeout(30000)}); + if(!r.ok) throw new Error(`Restart failed: ${r.status}`); + for(let i=0;i<100;i++) {try {if((await api('GET','/api/health')).status===200) return;}catch{} await new Promise(r=>setTimeout(r,100));} + throw new Error('Server did not recover after restart'); +} +checksFor(app,kind,{page,url,api,restart,seed,check:(id,group,critical,fn)=>tests.push({id,group,critical,fn})}); +try { + for(const test of tests) { + const start=performance.now();const initialErrors=errors.length; + try {await test.fn();if(errors.length>initialErrors) throw new Error(errors.slice(initialErrors).join('\n'));results.push({id:test.id,group:test.group,critical:test.critical,passed:true,duration_s:(performance.now()-start)/1000});} + catch(error) {results.push({id:test.id,group:test.group,critical:test.critical,passed:false,duration_s:(performance.now()-start)/1000,error:String(error).slice(0,3000)});} + } + for(const [label,width,height] of [['mobile',390,844],['desktop',1280,900]]) { + await page.setViewportSize({width,height});try{await page.goto(url,{waitUntil:'networkidle'});await page.screenshot({path:resolve(output,`${label}.png`),fullPage:true});}catch{} + } + await writeFile(resolve(output,'browser.json'),JSON.stringify({schema_version:1,checks:results,page_errors:errors,passed:results.length>0&&results.every(r=>r.passed)},null,2)); +} finally {await browser.close();} +process.exitCode=results.every(r=>r.passed)?0:1; diff --git a/evals/experiment.example.json b/evals/experiment.example.json new file mode 100644 index 0000000..563e3bb --- /dev/null +++ b/evals/experiment.example.json @@ -0,0 +1,15 @@ +{ + "id": "001-short-edit-route", + "status": "proposed", + "hypothesis": "A constrained path for title/colour edits removes one model round without correctness regressions.", + "single_change": "Describe the exact product change before implementation", + "baseline_run": null, + "candidate_run": null, + "screening_tasks": ["tictactoe.edit", "todos.edit", "expenses.edit", "booking.edit"], + "repeats_for_confirmation": 3, + "acceptance": {"paired_correctness_regressions": 0, "minimum_practical_speedup": 1.15, "unexplained_cost_increase_allowed": false}, + "paired_comparison": null, + "independent_visual_review": null, + "reserved_validation": null, + "decision_and_evidence": null +} diff --git a/evals/fixtures/application/backend/index.ts b/evals/fixtures/application/backend/index.ts new file mode 100644 index 0000000..5773bc7 --- /dev/null +++ b/evals/fixtures/application/backend/index.ts @@ -0,0 +1,16 @@ +import express from 'express'; +import { db } from './db'; +const app=express();app.use(express.json()); +app.get('/api/health',(_req,res)=>{res.json({ok:true});}); +const bad=(v:unknown)=>typeof v!=='string'||!v.trim(); + +db.exec("CREATE TABLE IF NOT EXISTS applications (id INTEGER PRIMARY KEY,name TEXT NOT NULL,email TEXT NOT NULL,track TEXT NOT NULL,statement TEXT NOT NULL,status TEXT NOT NULL DEFAULT 'draft')"); +const fields=['name','email','track','statement'];const strings=(b:any)=>fields.every(k=>typeof b[k]==='string'); +const get=(id:any)=>db.prepare('SELECT * FROM applications WHERE id=?').get(id) as any; +app.get('/api/applications',(_req,res)=>{res.json(db.prepare('SELECT * FROM applications ORDER BY id').all());}); +app.post('/api/applications',(req,res)=>{if(!strings(req.body)){res.status(400).json({error:'Invalid fields'});return;}const id=db.prepare('INSERT INTO applications (name,email,track,statement) VALUES (?,?,?,?)').run(...fields.map(k=>req.body[k])).lastInsertRowid;res.status(201).json(get(id));}); +app.put('/api/applications/:id',(req,res)=>{const row=get(req.params.id);if(!row){res.status(404).json({error:'Missing application'});return;}if(row.status==='submitted'){res.status(409).json({error:'Already submitted'});return;}if(!strings(req.body)){res.status(400).json({error:'Invalid fields'});return;}db.prepare('UPDATE applications SET name=?,email=?,track=?,statement=? WHERE id=?').run(...fields.map(k=>req.body[k]),req.params.id);res.json(get(req.params.id));}); +app.post('/api/applications/:id/submit',(req,res)=>{const row=get(req.params.id);if(!row){res.status(404).json({error:'Missing application'});return;}if(row.status==='submitted'){res.status(409).json({error:'Already submitted'});return;}if(bad(row.name)||!/^\S+@\S+\.\S+$/.test(row.email.trim())||!['Design','Engineering'].includes(row.track)||row.statement.trim().length<10){res.status(400).json({error:'Incomplete application'});return;}db.prepare("UPDATE applications SET status='submitted' WHERE id=?").run(req.params.id);res.json(get(req.params.id));}); +/*DUPLICATE_ROUTE*/ + +app.listen(Number(process.env.BACKEND_PORT),'127.0.0.1'); diff --git a/evals/fixtures/application/frontend/src/App.tsx b/evals/fixtures/application/frontend/src/App.tsx new file mode 100644 index 0000000..050dd4a --- /dev/null +++ b/evals/fixtures/application/frontend/src/App.tsx @@ -0,0 +1,12 @@ +import {useState,useEffect} from 'react'; +type Data={name:string;email:string;track:string;statement:string};type Row=Data&{id:number;status:string}; +const empty:Data={name:'',email:'',track:'Design',statement:''}; +export default function App(){const [data,D]=useState(empty),[id,I]=useState(null),[step,S]=useState(1),[rows,R]=useState([]),[error,E]=useState(''),[status,T]=useState(''); + const load=async()=>{const r:Row[]=await fetch('/api/applications').then(r=>r.json());R(r);return r;};useEffect(()=>{void load().then(r=>{const draft=r.filter(a=>a.status==='draft').at(-1);if(draft){D(draft);I(draft.id);}});},[]); + function field(k:keyof Data,v:string){D({...data,[k]:v});} + function next(){if(step===1&&(!data.name.trim()||!/^\S+@\S+\.\S+$/.test(data.email.trim()))||step===2&&(!['Design','Engineering'].includes(data.track)||data.statement.trim().length<10)){E('Complete required fields');return;}E('');S(step+1);} + function back(){S(step-1);} + async function save(){const r=await fetch(id===null?'/api/applications':`/api/applications/${id}`,{method:id===null?'POST':'PUT',headers:{'Content-Type':'application/json'},body:JSON.stringify(data)});if(!r.ok){E('Cannot save draft');return null;}const row:Row=await r.json();I(row.id);E('');await load();return row;} + async function submit(){const row=await save();if(!row)return;const r=await fetch(`/api/applications/${row.id}/submit`,{method:'POST'});if(!r.ok){E('Cannot submit');return;}T('Application submitted');I(null);D(empty);S(1);await load();} + return

Application form

{step===1&&<>}{step===2&&<>'); + await control(page,'Note body').fill('Changed\nbody');assert.equal(await control(page,'Note body').inputValue(),'Changed\nbody'); + await page.setContent(''); + await control(page,'Search rows').fill('Alpha');assert.equal(await control(page,'Search rows').inputValue(),'Alpha'); + }finally{await browser.close();} +}); + +test('selects and checkboxes remain unambiguous with wrapping labels',async()=>{ + const browser=await chromium.launch();const page=await browser.newPage(); + try{ + await page.setContent(''); + assert.equal(await control(page,'Correct answer').count(),1);await control(page,'Correct answer').selectOption({label:'B'});assert.equal(await control(page,'Correct answer').inputValue(),'1'); + await control(page,'Low stock only').check();assert.equal(await control(page,'Low stock only').isChecked(),true); + assert.equal(await control(page,'Missing label').count(),0); + }finally{await browser.close();} +}); diff --git a/evals/tests/test_drag.mjs b/evals/tests/test_drag.mjs new file mode 100644 index 0000000..95cd1a9 --- /dev/null +++ b/evals/tests/test_drag.mjs @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict'; +import { after, before, test } from 'node:test'; +import { createServer } from 'node:http'; +import { chromium, expect as baseExpect } from '@playwright/test'; +import { dragFromGrabArea } from '../browser/drag.mjs'; + +const expect = baseExpect.configure({ timeout: 3000 }); +let browser, server, origin; +const fixtures = new Map(); +function html(mode, state) { + const title = 'A useful card'; + const interior = mode.startsWith('all-') ? { + 'all-select': '', + 'all-button': '', + 'all-label': '', + 'all-link': 'A useful card', + 'all-editable': '
A useful card
', + 'all-role': '
A useful card
', + }[mode] : mode === 'blank' ? '' : '

A useful card

'; + return `
+

Left

${state === 'Left' ? `
${interior}
` : ''}
+

Right

${state === 'Right' ? `
${interior}
` : ''}
+
`; +} +before(async () => { + server = createServer(async (req, res) => { + const key = req.url.split('?')[0]; + const fixture = fixtures.get(key); + if (!fixture) { res.writeHead(404).end(); return; } + if (req.method === 'PATCH') { + let body = ''; for await (const chunk of req) body += chunk; + fixture.patches.push(body); fixture.state = body; + res.writeHead(200, { 'Content-Type': 'application/json' }).end('{}'); return; + } + res.writeHead(200, { 'Content-Type': 'text/html' }).end(html(fixture.mode, fixture.state)); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + origin = `http://127.0.0.1:${server.address().port}`; + browser = await chromium.launch({ headless: true }); +}); +after(async () => { await browser?.close(); if (server) await new Promise(resolve => server.close(resolve)); }); +async function runFixture(mode, fn) { + const path = '/' + mode; + const fixture = { mode, state: 'Left', patches: [] }; fixtures.set(path, fixture); + const context = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + try { + await context.route('**/*', route => new URL(route.request().url()).origin === origin ? route.continue() : route.abort()); + const page = await context.newPage(); page.setDefaultTimeout(3000); + await page.goto(origin + path); + const article = page.getByRole('article', { name: 'A useful card', exact: true }); + await fn({ page, article, target: page.getByRole('region', { name: 'Right', exact: true }), fixture }); + } finally { await context.close(); } +} +test('native-select center uses visible title; one real drag reaches unchanged target and persists', async () => { + await runFixture('valid', async ({ page, article, target, fixture }) => { + assert.equal(await article.evaluate(el => { const r = el.getBoundingClientRect(); return document.elementFromPoint(r.x+r.width/2,r.y+r.height/2).tagName; }), 'SELECT'); + await dragFromGrabArea(article, target, 'A useful card'); + await expect(target.getByRole('article', { name: 'A useful card', exact: true })).toBeVisible(); + const events = await page.evaluate(() => observed); + assert.deepEqual(events, [{ type: 'mousedown', tag: 'P', region: 'Left' }, { type: 'dragstart', tag: 'ARTICLE', region: 'Left' }, { type: 'drop', tag: 'SECTION', region: 'Right' }]); + assert.deepEqual(fixture.patches, ['Right']); + await page.reload(); + await expect(target.getByRole('article', { name: 'A useful card', exact: true })).toBeVisible(); + }); +}); +test('broken application drop behavior still fails the unchanged visibility obligation', async () => { + await runFixture('broken', async ({ page, article, target, fixture }) => { + await dragFromGrabArea(article, target, 'A useful card'); + await assert.rejects(expect(target.getByRole('article', { name: 'A useful card', exact: true })).toBeVisible(), /3000ms/); + assert.equal((await page.evaluate(() => observed.filter(e => e.type === 'dragstart'))).length, 1); + assert.deepEqual(fixture.patches, []); assert.equal(fixture.state, 'Left'); + }); +}); +test('fully interactive cards decline before any pointer gesture', async t => { + for (const mode of ['all-select', 'all-button', 'all-label', 'all-link', 'all-editable', 'all-role']) await t.test(mode, async () => { + await runFixture(mode, async ({ page, article, target, fixture }) => { + await assert.rejects(dragFromGrabArea(article, target, 'A useful card'), /No unobstructed noninteractive grab area/); + assert.deepEqual(await page.evaluate(() => observed), []); assert.deepEqual(fixture.patches, []); + }); + }); +}); +test('an external overlay declines rather than pressing it or dispatching synthetic events', async () => { + await runFixture('overlay', async ({ page, article, target, fixture }) => { + await assert.rejects(dragFromGrabArea(article, target, 'A useful card'), /No unobstructed noninteractive grab area/); + assert.deepEqual(await page.evaluate(() => observed), []); assert.deepEqual(fixture.patches, []); + }); +}); +test('a plain card without title text keeps working using an unobstructed interior sample', async () => { + await runFixture('blank', async ({ page, article, target, fixture }) => { + await dragFromGrabArea(article, target, 'A useful card'); + await expect(target.getByRole('article', { name: 'A useful card', exact: true })).toBeVisible(); + assert.equal((await page.evaluate(() => observed.filter(e => e.type === 'dragstart'))).length, 1); + assert.deepEqual(fixture.patches, ['Right']); + }); +}); diff --git a/evals/tests/test_paired.py b/evals/tests/test_paired.py new file mode 100644 index 0000000..436f6c9 --- /dev/null +++ b/evals/tests/test_paired.py @@ -0,0 +1,51 @@ +import unittest +import contextlib +import hashlib +import io +import json +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +from yoloeval.catalog import tasks +from yoloeval.paired import schedule,run_paired + + +class PairingTests(unittest.TestCase): + def test_every_pair_once_and_order_balanced(self): + plan=schedule(tasks('holdout'),3,42) + self.assertEqual(len(plan),60) + self.assertEqual(len({(task.id,trial) for task,trial,_ in plan}),60) + firsts=[order[0] for _,_,order in plan] + self.assertLessEqual(abs(firsts.count('baseline')-firsts.count('candidate')),1) + self.assertEqual(plan,schedule(tasks('holdout'),3,42)) + self.assertNotEqual(plan,schedule(tasks('holdout'),3,43)) + for _,_,order in plan:self.assertEqual(set(order),{'baseline','candidate'}) + + def test_complete_run_resumes_without_repeating_attempts(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);binary=root/'binary';binary.write_bytes(b'test binary') + fingerprint=hashlib.sha256(binary.read_bytes()).hexdigest() + args=SimpleNamespace(split='holdout',tasks='notes.edit,inventory.edit',repeats=3,seed=42,timeout=180,prices=None,output=root/'run', + baseline_binary=binary,candidate_binary=binary,baseline_model=None,candidate_model=None,baseline_allow_model=None,candidate_allow_model=['jev-test']) + calls=[] + def execute(task,trial,folder,*args,**kwargs): + folder.mkdir(parents=True);role=folder.parent.name;calls.append((task.id,trial,role)) + row={'task_id':task.id,'trial':trial,'kind':task.kind,'status':'completed','passed':True, + 'timing':{'agent_wall_s':10 if role=='baseline' else 5,'verified_wall_s':12}, + 'usage':{'usage_complete':True,'tokens_reported':{'total':100}},'cost_usd':None} + (folder/'result.json').write_text(json.dumps(row));return row + config={'suite_digest':'suite','dependency_digest':'deps','binary_sha256':fingerprint} + with patch('yoloeval.paired.load_provider',return_value={'model':'test','base_url':'https://example.invalid','api_key':'test'}), \ + patch('yoloeval.paired.manifest',side_effect=lambda *a:dict(config)), \ + patch('yoloeval.paired.suite_digest',return_value='suite'), \ + patch('yoloeval.paired.dependency_digest',return_value='deps'), \ + patch('yoloeval.paired.execute',side_effect=execute),contextlib.redirect_stdout(io.StringIO()): + first=run_paired(args);second=run_paired(args) + self.assertEqual(len(calls),12);self.assertEqual(first,second) + self.assertEqual(first['joint_success_pairs'],6);self.assertEqual(first['speedup_geomean'],2) + self.assertFalse(first['promotion_candidate']) # only two task families + self.assertEqual(len(json.loads((args.output/'schedule.json').read_text())),6) + + +if __name__=='__main__':unittest.main() diff --git a/evals/tests/test_recorder.py b/evals/tests/test_recorder.py new file mode 100644 index 0000000..d87d6b8 --- /dev/null +++ b/evals/tests/test_recorder.py @@ -0,0 +1,116 @@ +import json +import http.client +import io +import tempfile +import threading +import unittest +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from unittest.mock import Mock, patch + +from yoloeval.provider import Recorder + + +class RecorderTests(unittest.TestCase): + def test_roundtrip_usage_and_secret_boundary(self): + seen=[] + class Upstream(BaseHTTPRequestHandler): + def log_message(self,*args):pass + def do_POST(self): + seen.append((self.path,self.headers.get('Authorization'),self.headers.get('User-Agent'))) + self.rfile.read(int(self.headers['Content-Length'])) + data=json.dumps({'model':'test-model','usage':{'input_tokens':10,'output_tokens':3},'echo':'private-test-key'}).encode() + self.send_response(200);self.send_header('Content-Length',str(len(data)));self.end_headers();self.wfile.write(data) + upstream=ThreadingHTTPServer(('127.0.0.1',0),Upstream) + thread=threading.Thread(target=upstream.serve_forever,daemon=True);thread.start() + try: + with tempfile.TemporaryDirectory() as directory: + output=Path(directory)/'calls.json' + provider={'base_url':f'http://127.0.0.1:{upstream.server_port}','api_key':'private-test-key','model':'test-model','allow_models':['jev-test']} + with Recorder(provider,output,max_calls=2) as recorder: + def send(model='test-model',key='eval-proxy-token',endpoint='/responses'): + req=urllib.request.Request(recorder.url+endpoint,data=json.dumps({'model':model,'input':'private prompt'}).encode(),headers={'Authorization':'Bearer '+key}) + with urllib.request.urlopen(req) as response:return json.load(response) + with self.assertRaises(urllib.error.HTTPError) as rejected:send(key='wrong') + self.assertEqual(rejected.exception.code,401);rejected.exception.close() + with self.assertRaises(urllib.error.HTTPError) as rejected:send(model='wrong') + self.assertEqual(rejected.exception.code,400);rejected.exception.close() + response=send();self.assertEqual(response['echo'],'[REDACTED]') + send(model='jev-test',endpoint='/decisions') + with self.assertRaises(urllib.error.HTTPError) as rejected:send() + self.assertEqual(rejected.exception.code,429);rejected.exception.close() + self.assertEqual(seen,[('/v1/responses','Bearer private-test-key','YoloCoder-evals/1.0'),('/v1/decisions','Bearer private-test-key','YoloCoder-evals/1.0')]) + self.assertEqual(recorder.summary()['tokens_reported']['total'],26) + self.assertEqual(recorder.summary()['model_calls'],2) + self.assertTrue(all(c['response_origin']=='upstream' and c['upstream_status']==200 + and c['transport_error_type'] is None for c in recorder.calls)) + saved=output.read_text() + self.assertNotIn('private-test-key',saved);self.assertNotIn('private prompt',saved) + finally: + upstream.shutdown();upstream.server_close() + + def test_transport_failures_preserve_origin_and_consume_one_call(self): + def response(status, body=None, error=None): + value = Mock(status=status) + value.__enter__ = Mock(return_value=value) + value.__exit__ = Mock(return_value=False) + value.read = Mock(return_value=body, side_effect=error) + return value + + secret = 'private-test-key' + read_error = ConnectionResetError('body reset ' + secret) + http_read_error = urllib.error.HTTPError('https://example.invalid', 503, 'unavailable', {}, io.BytesIO()) + http_read_error.read = Mock(side_effect=ConnectionResetError('error body reset ' + secret)) + upstream_error_body = json.dumps({'error': {'message': 'upstream refusal ' + secret}}).encode() + upstream_error = urllib.error.HTTPError('https://example.invalid', 524, 'timeout', {}, io.BytesIO(upstream_error_body)) + cases = [ + ('before_headers', None, urllib.error.URLError('connect failed ' + secret), 502, None, 'recorder_transport_error', 'URLError'), + ('after_200_headers', response(200, error=read_error), None, 502, 200, 'recorder_transport_error', 'ConnectionResetError'), + ('error_body_read', None, http_read_error, 502, 503, 'recorder_transport_error', 'ConnectionResetError'), + ('upstream_refusal', None, upstream_error, 524, 524, 'upstream', None), + ] + for name, value, error, expected_status, upstream_status, origin, error_type in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + output = Path(directory)/'calls.json' + provider = {'base_url':'https://example.invalid', 'api_key':secret, 'model':'test-model'} + with patch('yoloeval.provider.urllib.request.urlopen', return_value=value, side_effect=error) as forward: + with Recorder(provider, output, max_calls=1) as recorder: + # http.client avoids mocking the local client together with the upstream. + def send(): + connection = http.client.HTTPConnection('127.0.0.1', recorder.server.server_port, timeout=5) + try: + connection.request('POST', '/responses', json.dumps({'model':'test-model'}), + {'Authorization':'Bearer eval-proxy-token'}) + result = connection.getresponse() + return result.status, result.read() + finally: + connection.close() + status, body = send() + self.assertEqual(status, expected_status) + self.assertNotIn(secret.encode(), body) + self.assertIn(b'[REDACTED]', body) + self.assertEqual(send()[0], 429) + forward.assert_called_once() + if value is not None: + value.__exit__.assert_called_once() + if isinstance(error, urllib.error.HTTPError): + self.assertTrue(error.closed) + saved = json.loads(output.read_text()) + self.assertEqual(len(saved), 1) + call = saved[0] + self.assertTrue(call['complete']) + self.assertEqual(call['status'], expected_status) + self.assertEqual(call['upstream_status'], upstream_status) + self.assertEqual(call['response_origin'], origin) + self.assertEqual(call['transport_error_type'], error_type) + self.assertEqual(call['requested_model'], 'test-model') + self.assertIsNone(call['returned_model']) + self.assertEqual(call['response_bytes'], len(body)) + self.assertGreaterEqual(call['duration_s'], 0) + self.assertEqual(recorder.summary()['failed_calls'], 1) + self.assertNotIn(secret, output.read_text()) + + +if __name__=='__main__':unittest.main() diff --git a/evals/tests/test_runtime.py b/evals/tests/test_runtime.py new file mode 100644 index 0000000..4aec490 --- /dev/null +++ b/evals/tests/test_runtime.py @@ -0,0 +1,46 @@ +import signal +import socket +import subprocess +import unittest +from unittest.mock import Mock,patch,call +from yoloeval.runtime import Runtime + + +class CleanupTests(unittest.TestCase): + def runtime(self, alive): + runtime=Runtime.__new__(Runtime) + runtime.closed=False;runtime.stop_stream=False;runtime.stream=None;runtime.event_thread=None + runtime.proc=Mock(pid=100);runtime.proc.poll.return_value=None if alive else 0 + runtime.log=Mock();runtime.temporary=Mock() + return runtime + + def test_cleanup_is_idempotent_and_does_not_signal_a_reaped_group(self): + r=self.runtime(True) + with patch('yoloeval.runtime.os.killpg') as kill: + def wait(**kwargs):r.proc.poll.return_value=0 + r.proc.wait.side_effect=wait + r.close();r.close() + self.assertEqual(kill.call_args_list,[call(100,signal.SIGTERM)]) + r.log.close.assert_called_once();r.temporary.cleanup.assert_called_once() + + def test_already_exited_process_never_receives_a_signal(self): + r=self.runtime(False) + with patch('yoloeval.runtime.os.killpg',side_effect=PermissionError(1,'Operation not permitted')) as kill: + r.close();kill.assert_not_called() + + def test_stuck_live_process_is_killed(self): + r=self.runtime(True);r.proc.wait.side_effect=[subprocess.TimeoutExpired('test',4),0] + with patch('yoloeval.runtime.os.killpg') as kill: + r.close();self.assertEqual(kill.call_args_list,[call(100,signal.SIGTERM),call(100,signal.SIGKILL)]) + + def test_stream_is_released_before_shutdown(self): + r=self.runtime(False) + left,right=socket.socketpair() + try: + r.stream=Mock();r.stream.fileno.return_value=left.fileno() + r.close();right.settimeout(.2);self.assertEqual(right.recv(1),b'') + r.stream.close.assert_called_once() + finally:left.close();right.close() + + +if __name__=='__main__':unittest.main() diff --git a/evals/tests/test_scoring.py b/evals/tests/test_scoring.py new file mode 100644 index 0000000..c441b8c --- /dev/null +++ b/evals/tests/test_scoring.py @@ -0,0 +1,58 @@ +import unittest +from yoloeval.provider import priced_cost, usage_of +from yoloeval.scoring import compare, score, summarize +from yoloeval.catalog import tasks + + +def row(task,trial,passed,seconds,tokens=100): + return {'task_id':task,'trial':trial,'status':'completed','passed':passed, + 'timing':{'agent_wall_s':seconds,'verified_wall_s':seconds+2}, + 'usage':{'usage_complete':True,'tokens_reported':{'total':tokens}},'cost_usd':None} + + +class ScoringTests(unittest.TestCase): + def test_failure_cannot_be_bought_with_partial_credit(self): + checks=[{'id':'a','group':'acceptance','critical':True,'passed':False}, + {'id':'b','group':'regression','critical':True,'passed':True}] + self.assertFalse(score(checks)['passed']) + self.assertFalse(score([])['passed']) + self.assertFalse(score([{'id':'a','group':'acceptance','passed':True}],agent_error='timeout')['passed']) + + def test_failures_count_toward_efficiency(self): + s=summarize([row('a',1,True,10,100),row('b',1,False,90,900)]) + self.assertEqual(s['agent_seconds_per_success_including_failures'],100) + self.assertEqual(s['reported_tokens_per_success_including_failures'],1000) + self.assertIsNone(s['cost_usd']) + + def test_regression_blocks_promotion_despite_speed(self): + a=[row(f'a{i}',j,True,20) for i in range(5) for j in range(1,4)] + b=[row(f'a{i}',j,True,5) for i in range(5) for j in range(1,4)] + self.assertTrue(compare(a,b)['promotion_candidate']) + b[0]['passed']=False + self.assertFalse(compare(a,b)['promotion_candidate']) + + def test_pairing_and_infra_errors_fail_closed(self): + a=[row('a',1,True,20)] + with self.assertRaises(ValueError):compare(a,[]) + with self.assertRaises(ValueError):compare(a,a+a) + b=[{**a[0],'status':'infra_error'}] + with self.assertRaises(ValueError):compare(a,b) + + def test_unknown_usage_is_not_zero_cost(self): + self.assertIsNone(usage_of({})) + self.assertEqual(usage_of({'usage':{'prompt_tokens':100,'completion_tokens':4,'prompt_tokens_details':{'cached_tokens':20}}}),{'input':100,'output':4,'cached':20,'total':104,'consistent':True}) + odd=usage_of({'usage':{'input_tokens':10,'output_tokens':4,'input_tokens_details':{'cached_tokens':100}}}) + self.assertFalse(odd['consistent']) + self.assertIsNone(priced_cost([{'usage':odd,'requested_model':'m'}],{'m':{'input_per_million':1,'output_per_million':1,'cached_input_per_million':1}})) + self.assertIsNone(priced_cost([{'usage':None,'requested_model':'m'}],{})) + prices={'m':{'input_per_million':1,'output_per_million':4,'cached_input_per_million':.5}} + self.assertAlmostEqual(priced_cost([{'usage':{'input':100,'output':10,'cached':20},'requested_model':'m'}],prices),.00013) + + def test_catalog_balance(self): + all_tasks=tasks();self.assertEqual(len(all_tasks),20) + self.assertEqual(len(tasks('dev')),0);self.assertEqual(len(tasks('holdout')),20) + for app in {t.app for t in all_tasks}: + self.assertEqual({t.kind for t in all_tasks if t.app==app},{'create','edit','feature','bug'}) + + +if __name__=='__main__':unittest.main() diff --git a/evals/tests/test_unseen.py b/evals/tests/test_unseen.py new file mode 100644 index 0000000..1bdffe4 --- /dev/null +++ b/evals/tests/test_unseen.py @@ -0,0 +1,34 @@ +import unittest +from yoloeval.unseen import decision,plan + +def row(i,passed=True,time=10,critical=None,status='completed'): + return {'task_id':str(i),'trial':1,'passed':passed,'status':status,'timing':{'agent_wall_s':time},'checks':[] if critical is None else [{'id':'storage.existing_records','passed':critical}]} + +class Gates(unittest.TestCase): + def test_recoverable_failure_does_not_stop(self): + self.assertEqual(decision({'baseline':[row(1)],'candidate':[row(1,False)]},3)['status'],'running') + def test_irrecoverable_quality_stops(self): + self.assertEqual(decision({'baseline':[row(1),row(2)],'candidate':[row(1,False),row(2,False)]},3)['status'],'stopped_early') + def test_no_invented_bound_on_future_vanilla(self): + self.assertEqual(decision({'baseline':[row(1)],'candidate':[row(1,time=1000)]},3)['status'],'running') + def test_mean_impossible_with_one_remaining(self): + d=decision({'baseline':[row(i) for i in range(3)],'candidate':[row(0,time=20),row(1,time=20)]},3) + self.assertEqual(d['status'],'stopped_early') + def test_median_impossible_even_if_mean_could_win(self): + d=decision({'baseline':[row(0,time=10),row(1,time=10),row(2,time=100)],'candidate':[row(0,time=11),row(1,time=11)]},3) + self.assertEqual(d['status'],'stopped_early') + def test_storage_requires_paired_control_check(self): + rows={'baseline':[row(1,critical=None)],'candidate':[row(1,False,critical=False)]} + self.assertEqual(decision(rows,3)['status'],'running') + rows['baseline'][0]['checks']=[{'id':'storage.existing_records','passed':True}] + self.assertEqual(decision(rows,3)['status'],'stopped_early') + def test_equal_quality_lower_mean_and_median_accepted(self): + d=decision({'baseline':[row(i,time=20,passed=i!=0) for i in range(3)],'candidate':[row(i,time=10,passed=i!=0) for i in range(3)]},3) + self.assertTrue(d['accepted']);self.assertFalse(d['perfect_candidate']) + def test_infra_is_never_acceptance(self): + self.assertEqual(decision({'baseline':[row(1,status='infra_error')],'candidate':[]},3)['status'],'blocked_infrastructure') + def test_balanced_complete_schedule(self): + p=plan();self.assertEqual(len(p),60);self.assertEqual(len({(t.id,i) for t,i,_ in p}),60) + self.assertEqual(sum(o[0]=='baseline' for _,_,o in p),30);self.assertTrue(all(t.kind=='create' for t,_,_ in p[:15])) + +if __name__=='__main__':unittest.main() diff --git a/evals/validation/runtime.py b/evals/validation/runtime.py new file mode 100644 index 0000000..c2af810 --- /dev/null +++ b/evals/validation/runtime.py @@ -0,0 +1,62 @@ +"""Real YoloCoder/process/browser integration checks; no paid model calls.""" +import argparse +import json +import socket +import threading +import time +from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from yoloeval.catalog import get_task +from yoloeval.paired import run_paired +from yoloeval.runner import execute,write_json +from yoloeval.runtime import Runtime + + +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--output',type=Path,required=True) + parser.add_argument('--binary',type=Path,default=Path.home()/'.local/bin/yolocoder') + args=parser.parse_args();out=args.output.resolve();out.mkdir(parents=True,exist_ok=True) + instances=[] + class ObservedRuntime(Runtime): + def __init__(self,*args,**kwargs):super().__init__(*args,**kwargs);instances.append(self) + class SlowProvider(BaseHTTPRequestHandler): + def log_message(self,*args):pass + def do_POST(self): + self.rfile.read(int(self.headers['Content-Length']));time.sleep(2) + body=json.dumps({'model':'offline-control','output':[],'usage':{'input_tokens':1,'output_tokens':1}}).encode() + try: + self.send_response(200);self.send_header('Content-Length',str(len(body)));self.end_headers();self.wfile.write(body) + except (BrokenPipeError,ConnectionResetError):pass + server=ThreadingHTTPServer(('127.0.0.1',0),SlowProvider) + threading.Thread(target=server.serve_forever,daemon=True).start() + provider={'base_url':f'http://127.0.0.1:{server.server_port}','api_key':'local-test-only','model':'offline-control','api':'responses'} + try: + with patch('yoloeval.runner.Runtime',ObservedRuntime): + timeout=execute(get_task('notes.bug'),0,out/'timeout',args.binary,provider,timeout=.5,control='solution') + deadline=json.loads((out/'timeout/deadline.json').read_text()) + timeout_ok=timeout['status']=='completed' and not timeout['passed'] and timeout.get('agent_error')=='Task exceeded 0.5s' and deadline['agent_wall_s_at_deadline']>=.5 + config=SimpleNamespace(output=out/'paired',split='holdout',tasks='notes.edit',repeats=1,seed=42,timeout=180,prices=None, + baseline_binary=args.binary,candidate_binary=args.binary,baseline_model=None,candidate_model=None,baseline_allow_model=None,candidate_allow_model=None) + def positive(task,trial,folder,binary,*args,**kwargs):return execute(task,trial,folder,binary,provider=None,control='solution') + with patch('yoloeval.paired.load_provider',return_value=provider),patch('yoloeval.paired.execute',side_effect=positive): + comparison=run_paired(config) + paired_ok=comparison['joint_success_pairs']==1 and not comparison['promotion_candidate'] + finally:server.shutdown();server.server_close() + # All sessions are owned by this check; verify their listening ports closed. + ports_closed=True + for runtime in instances: + runtime.close() # repeated cleanup must be harmless + for port in (runtime.frontend,runtime.backend,runtime.web): + with socket.socket() as sock: + sock.settimeout(.2);ports_closed &= sock.connect_ex(('127.0.0.1',port))!=0 + result={'passed':timeout_ok and paired_ok and ports_closed,'paid_model_calls':0,'timeout_recorded':timeout_ok, + 'deadline':deadline,'cleanup_ports_closed':ports_closed,'paired_controls_passed':paired_ok, + 'note':'The timeout uses a local delayed fake provider; paired attempts use reference solutions and no generation.'} + write_json(out/'validation.json',result);print(json.dumps(result,indent=2)) + return 0 if result['passed'] else 1 + + +if __name__=='__main__':raise SystemExit(main()) diff --git a/evals/validation/semantics.py b/evals/validation/semantics.py new file mode 100644 index 0000000..ced43ff --- /dev/null +++ b/evals/validation/semantics.py @@ -0,0 +1,41 @@ +"""Additional model-free semantic controls, including the actual false negative.""" +import argparse +import shutil +import sqlite3 +from pathlib import Path +from unittest.mock import patch +from yoloeval.catalog import get_task +from yoloeval.fixtures import materialize +from yoloeval.runner import execute,write_json + +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--output',type=Path,required=True);parser.add_argument('--binary',type=Path,required=True);parser.add_argument('--original-app',type=Path,required=True);args=parser.parse_args() + out=args.output.resolve();out.mkdir(parents=True) + cases=[('search-notes-reference','notes.create',None),('search-csv-reference','csv.feature',None),('original-generated-notes','notes.create',None),('global-accent','notes.edit','ui.contract'),('existing-data-erasure','notes.feature','storage.existing_records')] + rows=[] + for name,task,expected_failure in cases: + def altered(task,destination,control): + protected=materialize(task,destination,control) + if name.startswith('search-'): + p=destination/'frontend/src/App.tsx';text=p.read_text();needle='