From 01a84f950baf0bab050225e35af0c39b27cdff9f Mon Sep 17 00:00:00 2001 From: ahd-weekly-eval Date: Mon, 31 Aug 2026 13:52:24 +0000 Subject: [PATCH] docs(evals): weekly run 2026-08-31 Automated weekly eval against Cloudflare Workers AI OSS roster. See the report for per-cell numbers. --- docs/evals/weekly/2026-08-31.md | 95 ++++++ docs/evals/weekly/2026-08-31.replay.json | 372 +++++++++++++++++++++++ 2 files changed, 467 insertions(+) create mode 100644 docs/evals/weekly/2026-08-31.md create mode 100644 docs/evals/weekly/2026-08-31.replay.json diff --git a/docs/evals/weekly/2026-08-31.md b/docs/evals/weekly/2026-08-31.md new file mode 100644 index 0000000..6555f6d --- /dev/null +++ b/docs/evals/weekly/2026-08-31.md @@ -0,0 +1,95 @@ +# ahd eval · swiss-editorial · 2026-08-31T13:52:24.053Z + +```yaml ahd-replay +schema_version: 1 +kind: eval-live +ahd_version: 0.11.0 +ahd_commit: f241329a0fbbeebfb7c33280907d12bb6964bb7f +git_dirty: true +node_version: v20.20.2 +platform: linux-x64 +invoked_at: 2026-08-31T13:20:07.489Z +token: + path: /home/runner/work/ahd/ahd/tokens/swiss-editorial.yml + hash: sha256:380a3d833d94 +brief: + path: briefs/landing.yml + hash: sha256:8b7d42759643 +sampling: + n: 30 + temperature: null + seed: null +models: + - id: @cf/google/gemma-4-26b-a4b-it + provider: cloudflare-workers-ai + provider_request_ids: 49 captured + - id: @cf/meta/llama-4-scout-17b-16e-instruct + provider: cloudflare-workers-ai + provider_request_ids: 60 captured + - id: @cf/mistralai/mistral-small-3.1-24b-instruct + provider: cloudflare-workers-ai + provider_request_ids: 60 captured + - id: @cf/openai/gpt-oss-120b + provider: cloudflare-workers-ai + provider_request_ids: 60 captured + - id: @cf/qwen/qwen3-30b-a3b-fp8 + provider: cloudflare-workers-ai + provider_request_ids: 60 captured +conditions: + requested: [raw, compiled] + effective: [raw, compiled] +``` + +Replay this run: + +```sh +git checkout f241329a0fbb +npm ci && npm run build +/opt/hostedtoolcache/node/20.20.2/x64/bin/node /home/runner/work/ahd/ahd/bin/ahd.js eval-live swiss-editorial --brief briefs/landing.yml --models cf:@cf/google/gemma-4-26b-a4b-it,cf:@cf/meta/llama-4-scout-17b-16e-instruct,cf:@cf/mistralai/mistral-small-3.1-24b-instruct,cf:@cf/openai/gpt-oss-120b,cf:@cf/qwen/qwen3-30b-a3b-fp8 --n 30 --sample-concurrency 6 --out /home/runner/work/_temp/weekly-samples.d2dTnd --report docs/evals/weekly/2026-08-31.md +``` + +## Run + +- Brief: `briefs/landing.yml` +- Samples per cell: **30** +- Max tokens: 12000 +- Models: + - `@cf/google/gemma-4-26b-a4b-it` (cloudflare-workers-ai) · spec `cf:@cf/google/gemma-4-26b-a4b-it` + - `@cf/meta/llama-4-scout-17b-16e-instruct` (cloudflare-workers-ai) · spec `cf:@cf/meta/llama-4-scout-17b-16e-instruct` + - `@cf/mistralai/mistral-small-3.1-24b-instruct` (cloudflare-workers-ai) · spec `cf:@cf/mistralai/mistral-small-3.1-24b-instruct` + - `@cf/openai/gpt-oss-120b` (cloudflare-workers-ai) · spec `cf:@cf/openai/gpt-oss-120b` + - `@cf/qwen/qwen3-30b-a3b-fp8` (cloudflare-workers-ai) · spec `cf:@cf/qwen/qwen3-30b-a3b-fp8` + +## Per-model slop reduction + +| model | raw attempted → scored | compiled attempted → scored | raw mean tells | compiled mean tells | Δ | reduction | +|---|---:|---:|---:|---:|---:|---:| +| `@cf/google/gemma-4-26b-a4b-it` | 30 → 21 | 30 → 28 | 2.76 | 1.43 | 1.33 | 48.3% | +| `@cf/meta/llama-4-scout-17b-16e-instruct` | 30 → 30 | 30 → 30 | 2.00 | 1.97 | 0.03 | 1.7% | +| `@cf/mistralai/mistral-small-3.1-24b-instruct` | 30 → 30 | 30 → 30 | 3.27 | 1.20 | 2.07 | 63.3% | +| `@cf/openai/gpt-oss-120b` | 30 → 30 | 30 → 30 | 3.10 | 0.87 | 2.23 | 72.0% | +| `@cf/qwen/qwen3-30b-a3b-fp8` | 30 → 30 | 30 → 30 | 1.43 | 1.60 | -0.17 | -11.6% | + +## Per-tell frequency (scored samples only) + +| tell | @cf/google/gemma-4-26b-a4b-it/raw | @cf/google/gemma-4-26b-a4b-it/compiled | @cf/meta/llama-4-scout-17b-16e-instruct/raw | @cf/meta/llama-4-scout-17b-16e-instruct/compiled | @cf/mistralai/mistral-small-3.1-24b-instruct/raw | @cf/mistralai/mistral-small-3.1-24b-instruct/compiled | @cf/openai/gpt-oss-120b/raw | @cf/openai/gpt-oss-120b/compiled | @cf/qwen/qwen3-30b-a3b-fp8/raw | @cf/qwen/qwen3-30b-a3b-fp8/compiled | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| ahd/a11y/heading-skip | 0% | 4% | 0% | 0% | 0% | 0% | 0% | 0% | 0% | 0% | +| ahd/line-height-per-size | 95% | 0% | 0% | 67% | 47% | 23% | 100% | 3% | 43% | 33% | +| ahd/no-default-grotesque | 0% | 0% | 0% | 0% | 0% | 0% | 0% | 3% | 0% | 0% | +| ahd/no-em-dashes-in-prose | 5% | 0% | 0% | 0% | 0% | 0% | 0% | 3% | 0% | 0% | +| ahd/no-flat-dark-mode | 0% | 0% | 0% | 0% | 0% | 0% | 3% | 0% | 0% | 0% | +| ahd/radius-hierarchy | 57% | 4% | 0% | 97% | 100% | 7% | 70% | 0% | 7% | 37% | +| ahd/require-named-grid | 0% | 0% | 100% | 0% | 100% | 63% | 23% | 0% | 3% | 0% | +| ahd/require-type-pairing | 19% | 0% | 100% | 33% | 80% | 0% | 53% | 0% | 80% | 0% | +| ahd/respect-reduced-motion | 0% | 4% | 0% | 0% | 0% | 0% | 0% | 0% | 0% | 0% | +| ahd/tracking-per-size | 0% | 39% | 0% | 0% | 0% | 20% | 0% | 7% | 0% | 0% | +| ahd/weight-variety | 100% | 93% | 0% | 0% | 0% | 7% | 60% | 70% | 10% | 90% | + +## Caveats +- Scoring runs the deterministic AHD linter (38 source-level rules) over every sample that passes a basic HTML sanity check. +- Counts reported per cell: attempted (runs initiated) / errored (API / runtime errors) / extractionFailed (response contained no usable HTML) / scored (linted). A large gap between attempted and scored is a signal that the model is struggling with the instruction, not that it passed the taxonomy. +- Raw condition: the brief is expanded as plain prose (intent + audience + surfaces + mustInclude + mustAvoid) with no AHD system prompt, no style token, no forbidden list. Compiled condition: same brief plus the AHD-compiled system prompt. The only thing that differs between conditions is the AHD intervention. +- Vision-only tells (14 rules in the critic) are not scored in this pipeline; run the critic on rendered screenshots for full taxonomy coverage. +- Tells-per-page is a proxy metric: a thin page has little surface for rules to fire against. Read the Δ alongside the actual rendered HTML, not in isolation. +- Model versions change. See the run manifest for exact canonical model ids. \ No newline at end of file diff --git a/docs/evals/weekly/2026-08-31.replay.json b/docs/evals/weekly/2026-08-31.replay.json new file mode 100644 index 0000000..dc041d8 --- /dev/null +++ b/docs/evals/weekly/2026-08-31.replay.json @@ -0,0 +1,372 @@ +{ + "schema_version": 1, + "kind": "eval-live", + "ahd_version": "0.11.0", + "ahd_commit": "f241329a0fbbeebfb7c33280907d12bb6964bb7f", + "git_dirty": true, + "node_version": "v20.20.2", + "platform": "linux-x64", + "invoked_at": "2026-08-31T13:20:07.489Z", + "argv": [ + "/opt/hostedtoolcache/node/20.20.2/x64/bin/node", + "/home/runner/work/ahd/ahd/bin/ahd.js", + "eval-live", + "swiss-editorial", + "--brief", + "briefs/landing.yml", + "--models", + "cf:@cf/google/gemma-4-26b-a4b-it,cf:@cf/meta/llama-4-scout-17b-16e-instruct,cf:@cf/mistralai/mistral-small-3.1-24b-instruct,cf:@cf/openai/gpt-oss-120b,cf:@cf/qwen/qwen3-30b-a3b-fp8", + "--n", + "30", + "--sample-concurrency", + "6", + "--out", + "/home/runner/work/_temp/weekly-samples.d2dTnd", + "--report", + "docs/evals/weekly/2026-08-31.md" + ], + "token": { + "path": "/home/runner/work/ahd/ahd/tokens/swiss-editorial.yml", + "hash": "sha256:380a3d833d9463dbc681df7465993ab6413d8e77188f55f97359f60dc4b746b1" + }, + "brief": { + "path": "briefs/landing.yml", + "hash": "sha256:8b7d4275964399a91e6ddec525151ba672ed9c4721279a37ec16ab3450493a4c" + }, + "sampling": { + "n": 30, + "temperature": null, + "seed": null + }, + "models": [ + { + "id": "@cf/google/gemma-4-26b-a4b-it", + "provider": "cloudflare-workers-ai", + "provider_request_ids": [ + "a33c54af5a94a153-IAD", + "a33c54af7ed3c76b-IAD", + "a33c54af6ac91727-IAD", + "a33c54af6a2e9422-IAD", + "a33c54af5bc7eb4a-IAD", + "a33c54af6e743000-IAD", + "a33c55b91ff41727-IAD", + "a33c55d55bc19422-IAD", + "a33c55b7fac8c76b-IAD", + "a33c55fc09f2eb4a-IAD", + "a33c56c99d351727-IAD", + "a33c5692ffb43000-IAD", + "a33c55a6bea8a153-IAD", + "a33c57051b839422-IAD", + "a33c57270d79c76b-IAD", + "a33c574badc6eb4a-IAD", + "a33c57dc0b821727-IAD", + "a33c5810b961a153-IAD", + "a33c5814cd299422-IAD", + "a33c58de1d221727-IAD", + "a33c5cf7edcdc76b-IAD", + "a33c5fae88815414-IAD", + "a33c5fae8913e144-IAD", + "a33c5fae8810eb4a-IAD", + "a33c5fae8ffe052a-IAD", + "a33c5fae7c79c76b-IAD", + "a33c6110d9f3e144-IAD", + "a33c610759735414-IAD", + "a33c61b90baac76b-IAD", + "a33c6153ac72eb4a-IAD", + "a33c62805c8ce144-IAD", + "a33c629c8cded6e1-IAD", + "a33c639d0d6b5414-IAD", + "a33c63b8880c6d4d-IAD", + "a33c63a4cd87c76b-IAD", + "a33c6477ccbb4b8a-IAD", + "a33c6406ad47e144-IAD", + "a33c64cd6b64ef65-IAD", + "a33c65883cda5414-IAD", + "a33c65a2b8b96d4d-IAD", + "a33c65b8e9dcc76b-IAD", + "a33c661c1b1a4b8a-IAD", + "a33c665daaa1e144-IAD", + "a33c671ecfa76d4d-IAD", + "a33c66f9e8375414-IAD", + "a33c6688e8d7ef65-IAD", + "a33c677b5a30c76b-IAD", + "a33c68105e1a4b8a-IAD", + "a33c681cab51e144-IAD" + ] + }, + { + "id": "@cf/meta/llama-4-scout-17b-16e-instruct", + "provider": "cloudflare-workers-ai", + "provider_request_ids": [ + "a33c69733e92eb4a-IAD", + "a33c69733f835648-IAD", + "a33c69733fe63000-IAD", + "a33c697328514b8a-IAD", + "a33c69732d0fe144-IAD", + "a33c69733cd61727-IAD", + "a33c69c51adbeb4a-IAD", + "a33c69c53b2e3000-IAD", + "a33c69c53b025648-IAD", + "a33c69c8de16e144-IAD", + "a33c69c8be7a4b8a-IAD", + "a33c69c99c121727-IAD", + "a33c6a17db98eb4a-IAD", + "a33c6a1aeb7a5648-IAD", + "a33c6a1d29cfe144-IAD", + "a33c6a1d3e274b8a-IAD", + "a33c6a1abd033000-IAD", + "a33c6a1e3f3a1727-IAD", + "a33c6a669b9a5648-IAD", + "a33c6a63ebcceb4a-IAD", + "a33c6a6dcfad1727-IAD", + "a33c6a6b7c254b8a-IAD", + "a33c6a6b7a2ce144-IAD", + "a33c6a6bfede3000-IAD", + "a33c6ac699c65648-IAD", + "a33c6ac7ef95eb4a-IAD", + "a33c6ace2dade144-IAD", + "a33c6acb5cf01727-IAD", + "a33c6ace282b4b8a-IAD", + "a33c6ace5d823000-IAD", + "a33c6b26893beb4a-IAD", + "a33c6b268ab74b8a-IAD", + "a33c6b268b7e1727-IAD", + "a33c6b268b283000-IAD", + "a33c6b268f9e5648-IAD", + "a33c6b268b62e144-IAD", + "a33c6b96b910eb4a-IAD", + "a33c6bb4eda34b8a-IAD", + "a33c6bb73c6f3000-IAD", + "a33c6bb52ff11727-IAD", + "a33c6bb7ef095648-IAD", + "a33c6bc97bf1e144-IAD", + "a33c6c161f6beb4a-IAD", + "a33c6c48f94e1727-IAD", + "a33c6c592914e144-IAD", + "a33c6c3e1ad34b8a-IAD", + "a33c6c4e39275648-IAD", + "a33c6c48ac0f3000-IAD", + "a33c6cb0cc641727-IAD", + "a33c6c9f2d80eb4a-IAD", + "a33c6cba5ee5e144-IAD", + "a33c6ccd1e2c4b8a-IAD", + "a33c6ce46afa3000-IAD", + "a33c6ce1aace5648-IAD", + "a33c6d24c92ee144-IAD", + "a33c6d1b98241727-IAD", + "a33c6d373de04b8a-IAD", + "a33c6d20c919eb4a-IAD", + "a33c6d5ebbd93000-IAD", + "a33c6d7499e65648-IAD" + ] + }, + { + "id": "@cf/mistralai/mistral-small-3.1-24b-instruct", + "provider": "cloudflare-workers-ai", + "provider_request_ids": [ + "a33c6df26c915648-IAD", + "a33c6df28c40e637-IAD", + "a33c6df27a18cbf8-IAD", + "a33c6df26e833000-IAD", + "a33c6df27d70a153-IAD", + "a33c6df27e2e50e2-IAD", + "a33c6e5b39435648-IAD", + "a33c6e6c3e07cbf8-IAD", + "a33c6e6e6cf23000-IAD", + "a33c6e6aaba0e637-IAD", + "a33c6e7fddbda153-IAD", + "a33c6e887fa350e2-IAD", + "a33c6ee63a67cbf8-IAD", + "a33c6ec838785648-IAD", + "a33c6f12ad8150e2-IAD", + "a33c6ef478d53000-IAD", + "a33c6f0c1c8ea153-IAD", + "a33c6efc6c4fe637-IAD", + "a33c6f427aa7cbf8-IAD", + "a33c6f82adb9a153-IAD", + "a33c6f7c281b3000-IAD", + "a33c6f5a6ca35648-IAD", + "a33c6f747ad950e2-IAD", + "a33c6f90ed8fe637-IAD", + "a33c6fc519d2cbf8-IAD", + "a33c6fee7eb6a153-IAD", + "a33c70008be85648-IAD", + "a33c70149ed450e2-IAD", + "a33c6ff76d4b3000-IAD", + "a33c701e8916e637-IAD", + "a33c70912b25e637-IAD", + "a33c70912f085648-IAD", + "a33c70912d0950e2-IAD", + "a33c70912e343000-IAD", + "a33c70914d5a1727-IAD", + "a33c70914b181fd1-IAD", + "a33c71346a4de637-IAD", + "a33c71713cb95648-IAD", + "a33c7174c97b50e2-IAD", + "a33c71b24b5d1727-IAD", + "a33c71a03a683000-IAD", + "a33c71db28661fd1-IAD", + "a33c71f0dfe9e637-IAD", + "a33c72775c6f5648-IAD", + "a33c72b41e8e50e2-IAD", + "a33c72c1ba241727-IAD", + "a33c72c6be2e3000-IAD", + "a33c7321bfcae637-IAD", + "a33c72f8d9151fd1-IAD", + "a33c73b44dcd50e2-IAD", + "a33c73607f1d5648-IAD", + "a33c740348d03000-IAD", + "a33c73c2cace1727-IAD", + "a33c74332cbce637-IAD", + "a33c744218511fd1-IAD", + "a33c74a3ae5e5648-IAD", + "a33c749af9be50e2-IAD", + "a33c74fc4c173000-IAD", + "a33c753bfb34e637-IAD", + "a33c75091eb21727-IAD" + ] + }, + { + "id": "@cf/openai/gpt-oss-120b", + "provider": "cloudflare-workers-ai", + "provider_request_ids": [ + "a33c76310df3a153-IAD", + "a33c7630eb441727-IAD", + "a33c7630eaeae637-IAD", + "a33c76887a861727-IAD", + "a33c76310f3bd692-IAD", + "a33c76310d87052a-IAD", + "a33c76310f123e80-IAD", + "a33c765ddbaca153-IAD", + "a33c76a16dcae637-IAD", + "a33c76dced83d692-IAD", + "a33c770e5a53052a-IAD", + "a33c7768db58d692-IAD", + "a33c77156cec3e80-IAD", + "a33c76ca58661727-IAD", + "a33c7766cab5e637-IAD", + "a33c7743cc41a153-IAD", + "a33c77fd381ba153-IAD", + "a33c779d8922052a-IAD", + "a33c77ccf8201727-IAD", + "a33c77c4dd64d692-IAD", + "a33c78354df7052a-IAD", + "a33c77c80dba3e80-IAD", + "a33c77f2a8d8e637-IAD", + "a33c7828bb55a153-IAD", + "a33c784008891727-IAD", + "a33c7884683b052a-IAD", + "a33c7897dfa23e80-IAD", + "a33c78793a09d692-IAD", + "a33c78abee59e637-IAD", + "a33c78d67a82a153-IAD", + "a33c796e2e58eb4a-IAD", + "a33c796e2b756d4d-IAD", + "a33c79ca8bbaeb4a-IAD", + "a33c796e1afce637-IAD", + "a33c7a630f03eb4a-IAD", + "a33c796e292e1773-IAD", + "a33c796e1c0ea153-IAD", + "a33c7a2e5f6a6d4d-IAD", + "a33c7ab61a19e637-IAD", + "a33c796e2aa31727-IAD", + "a33c7ac1fb91eb4a-IAD", + "a33c7ac438461773-IAD", + "a33c7adf5d7ba153-IAD", + "a33c7bf57af51773-IAD", + "a33c7afe8fc26d4d-IAD", + "a33c7b1e5b76e637-IAD", + "a33c7b220ea41727-IAD", + "a33c7bc83df3eb4a-IAD", + "a33c7c8acb386d4d-IAD", + "a33c7c8aab3d1773-IAD", + "a33c7cb3c97d1727-IAD", + "a33c7c833d59a153-IAD", + "a33c7c98c8f5e637-IAD", + "a33c7dcce88de637-IAD", + "a33c7cfadc61eb4a-IAD", + "a33c7da79d5a1773-IAD", + "a33c7d56abca6d4d-IAD", + "a33c7dc988d1a153-IAD", + "a33c7dc5d8731727-IAD", + "a33c7e233961e637-IAD" + ] + }, + { + "id": "@cf/qwen/qwen3-30b-a3b-fp8", + "provider": "cloudflare-workers-ai", + "provider_request_ids": [ + "a33c7fc77aca50e2-IAD", + "a33c7fc7795e1727-IAD", + "a33c7fc77d1b6d4d-IAD", + "a33c7fc78d884b8a-IAD", + "a33c7fc7786ee637-IAD", + "a33c7fc77cf6eb4a-IAD", + "a33c801e7d811727-IAD", + "a33c802979134b8a-IAD", + "a33c8032ad44eb4a-IAD", + "a33c802a1c3fe637-IAD", + "a33c8027edb26d4d-IAD", + "a33c806f5af44b8a-IAD", + "a33c80827f82eb4a-IAD", + "a33c808c7a756d4d-IAD", + "a33c80a74ab94b8a-IAD", + "a33c805ccbad1727-IAD", + "a33c80162cae50e2-IAD", + "a33c80d94d4e4b8a-IAD", + "a33c8084da02e637-IAD", + "a33c80c61c6e6d4d-IAD", + "a33c80dc5e101727-IAD", + "a33c80e74f3b50e2-IAD", + "a33c80c018fbeb4a-IAD", + "a33c8108aee6e637-IAD", + "a33c81076cb34b8a-IAD", + "a33c81133e8e1727-IAD", + "a33c811128e36d4d-IAD", + "a33c811d4e02eb4a-IAD", + "a33c81184b3d50e2-IAD", + "a33c81346f08e637-IAD", + "a33c81933b59cbf8-IAD", + "a33c81934bd6ef65-IAD", + "a33c81932ec8e637-IAD", + "a33c81933c824b8a-IAD", + "a33c81933fb550e2-IAD", + "a33c81933eb06d4d-IAD", + "a33c82016eb2ef65-IAD", + "a33c82168ba84b8a-IAD", + "a33c8242bb116d4d-IAD", + "a33c820a9f3ee637-IAD", + "a33c81fa2f6ccbf8-IAD", + "a33c821e49ae50e2-IAD", + "a33c8296fc42e637-IAD", + "a33c825a9971ef65-IAD", + "a33c82959a996d4d-IAD", + "a33c827cea064b8a-IAD", + "a33c82a7795fcbf8-IAD", + "a33c82bb4e8b50e2-IAD", + "a33c82ddcdece637-IAD", + "a33c82df0be9ef65-IAD", + "a33c82e58a656d4d-IAD", + "a33c83239f60cbf8-IAD", + "a33c83258f9b50e2-IAD", + "a33c83405f0ee637-IAD", + "a33c83191f654b8a-IAD", + "a33c837dc903cbf8-IAD", + "a33c83880efa50e2-IAD", + "a33c835e6f02ef65-IAD", + "a33c83733ff86d4d-IAD", + "a33c838a4dbee637-IAD" + ] + } + ], + "conditions": { + "requested": [ + "raw", + "compiled" + ], + "effective": [ + "raw", + "compiled" + ] + } +}