From c2d8d1a717024fbf7544774c388448e1ba04bc9d Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Wed, 30 Sep 2026 09:45:17 +0800 Subject: [PATCH 1/3] Handle seeds that chose different 8-way aggregations; add AC1b attempt ledger AC1b attempt 2 stopped in make figures: ModernBERT k=100 seeds chose argmax, summed, summed on validation and risk_coverage assumed one aggregation for all three seeds. A k with mixed choices now gets one risk-coverage panel per chosen aggregation, titled with the seeds whose final router uses it, and the README caption says so. Nothing is chosen on test. Unanimous results render byte-identically. docs/ac1b/attempts.json records every AC1b attempt with its result, reason and Haiku spend. Comparison and README report the budget in three parts: the fixed original AC6 cost, each attempt, and the reproduction-validation total. PLAN 5.1 gains the per-attempt US$5 cap. --- DEVLOG.md | 28 +++ README.md | 18 +- docs/PLAN.md | 1 + docs/ac1b/README.md | 26 +++ docs/ac1b/attempt-1/INCIDENT.md | 13 ++ docs/ac1b/attempt-1/comparison.md | 53 +++++ docs/ac1b/attempt-2/INCIDENT.md | 11 + docs/ac1b/attempt-2/comparison.md | 344 ++++++++++++++++++++++++++++++ docs/ac1b/attempts.json | 34 +++ src/tinyrouter/ac1b_ledger.py | 186 ++++++++++++++++ src/tinyrouter/comparison.py | 54 +++-- src/tinyrouter/figures.py | 70 +++++- src/tinyrouter/report.py | 21 +- tests/test_ac1b_ledger.py | 107 ++++++++++ tests/test_comparison.py | 60 +++++- tests/test_figures.py | 53 ++++- tests/test_report.py | 19 ++ 17 files changed, 1052 insertions(+), 46 deletions(-) create mode 100644 docs/ac1b/README.md create mode 100644 docs/ac1b/attempt-1/INCIDENT.md create mode 100644 docs/ac1b/attempt-1/comparison.md create mode 100644 docs/ac1b/attempt-2/INCIDENT.md create mode 100644 docs/ac1b/attempt-2/comparison.md create mode 100644 docs/ac1b/attempts.json create mode 100644 src/tinyrouter/ac1b_ledger.py create mode 100644 tests/test_ac1b_ledger.py diff --git a/DEVLOG.md b/DEVLOG.md index 003f723..6b1cb1d 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -4,6 +4,34 @@ --- +## 2026-09-30：AC1b 兩次嘗試失敗；修正各 seed 選不同聚合時的畫圖缺陷，新增嘗試帳本 + +### 本次工作 / 執行摘要 +- 狀態不變：RQ1 到 RQ5 完成；**Tier 1 驗收未完成，AC1b 尚未通過**。 +- **AC1b 第 1 次**（`3992840ddb3f`，2026-09-29 07:40 到 14:24 UTC）：訓練、AC2、pilot 全過；Haiku 前的 `count_tokens` 在 Anthropic 認證事故期間連 5 次 HTTP 503，花費 US$0。依凍結標準判 FAIL；Drew 判定為基礎設施事故，同一 id 與身分續跑。 +- **AC1b 第 2 次**（續跑，2026-09-30 01:13 到 01:31 UTC）：Haiku 8,600/8,600、US$3.178751、0 parse failed、20 筆預測與原 run 不同；analysis、latency、cost 完成；`make figures` 失敗：ModernBERT k=100 三個 seed 在 validation 選了 argmax、summed、summed，`figures.selected_aggregation` 直接丟 ValueError。原始 run 三個 seed 都選 argmax，所以這條路徑從沒跑過。判 FAIL（程式缺陷）。 +- **修正**：找過所有讀聚合方式的地方。`analysis` 本來就讓每個 seed 用自己 validation 選的聚合（`final`、`diagnostics` 逐 seed 算好再合併）；`report` 的聚合說明已會列出逐 seed 選擇；`cost`、`comparison`、學習曲線、router 圖、門檻圖都讀 `final` 或 `diagnostics`，不受影響。只有 risk-coverage 圖假設三個 seed 同一種。`curves.json` 對每種聚合只存三 seed 的 mean 與 std，沒有逐 seed 曲線，而補存會改動原始 `curves.json` 的位元組，所以改成：seed 一致的 k 維持一個面板；不一致的 k 每種被選到的聚合各一個面板（固定 argmax、summed 順序），標題寫出哪些 seed 的 final router 用它；README 圖說在這種情況自動加一句說明。沒有任何選擇用到 test，也沒有固定成某一種。 +- **帳本**：`docs/ac1b/attempts.json` 記錄每次嘗試（commit、id、UTC 起訖、結果與原因、實際 Haiku 支出），`docs/ac1b/attempt-/` 放 comparison 與事故說明。comparison 與 README 的預算改成三部分：原始實驗 US$3.19（固定）、每次嘗試支出與原因、reproduction-validation 累計。PLAN §5.1 預算規則補一行：每次嘗試各自獨立 US$5 上限（Drew，2026-09-30）。 +- 下一步依 Drew 決定：合併後在新 commit 完整重跑 AC1b（新的一次嘗試，新 id、新 journal、獨立 US$5）；不拿第 2 次的輸出配新 commit 的圖。 + +### 核心發現 / 數據 +- 用第 2 次重跑輸出的副本跑 `figures` 與 `report`（`--results-root`、`--readme-out`）：4 張圖與 report 都完成；risk-coverage 為 (a) k=10 summed、(b) k=100 argmax（seed 42）、(c) k=100 summed（seeds 43、44）。 +- 第 2 次重跑中其他聚合不一致的組：BERT k=10、25、50、100，ModernBERT k=25，OOS 0 消融 k=100，TF-IDF k=5、10、25。 +- 原始結果上 `make figures`、`make report` 兩次，`results/` 與 README 生成區塊逐位元組不變。 + +### Blockers / 遇到的問題 +- (無) + +### Next +- [ ] 合併後在新 commit 完整重跑 AC1b（第 3 次嘗試），結果寫進 `docs/ac1b/attempts.json` + +### Files / Budget +- 新增：`src/tinyrouter/ac1b_ledger.py`；`tests/test_ac1b_ledger.py`；`docs/ac1b/`（`attempts.json`、`README.md`、`attempt-1/`、`attempt-2/`） +- 修改：`src/tinyrouter/figures.py`、`report.py`、`comparison.py`；`tests/test_figures.py`、`test_report.py`、`test_comparison.py`；`README.md`、`docs/PLAN.md`、`DEVLOG.md` +- API 花費：本次 US$0。AC1b 累計（reproduction-validation）：US$3.178751；原始實驗成本維持 US$3.19 + +--- + ## 2026-09-29（深夜，七）：PR #19 審查修正（R1 到 R6），AC1b 實跑前固定比較範圍 ### 本次工作 / 執行摘要 diff --git a/README.md b/README.md index 4a9b26f..cde1611 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,23 @@ Training on Apple MPS is not bit-for-bit deterministic, so a rerun is not expect A clean clone is not a clean cache: the base models and the CLINC150 files come from the shared Hugging Face cache (`~/.cache/huggingface` by default). That does not affect correctness, since every model is pinned to a revision and every data file is checked against its SHA-256, but to download everything afresh point the cache at an empty directory, for example `HF_HOME=/tmp/tinyrouter-hf make reproduce`. -**Status: AC1b has not been run yet.** Until it has passed, the status is RQ1 to RQ5 complete, Tier 1 acceptance not complete. +**Status: AC1b has not passed yet.** It has been attempted twice at `3992840ddb3f` and both attempts failed (an external authentication incident, then a defect in the figure code when seeds chose different 8-way aggregations); every attempt is recorded in [`docs/ac1b/`](docs/ac1b/README.md). Until it has passed, the status is RQ1 to RQ5 complete, Tier 1 acceptance not complete. + +AC1b budget. Every attempt has its own US$5 Haiku cap, enforced per reproduction id: + + +- Original experiment (AC6, fixed): US$3.19 +- AC1b attempts (reproduction-validation, each capped at US$5): + +| attempt | reproduction id | result | reason | Haiku spend | +|---|---|---|---|---| +| 1 | `3992840ddb3f` | FAIL | infrastructure: Haiku count_tokens returned HTTP 503 credential validation failed 5 times during an Anthropic authentication incident; no classification request sent | US$0.000000 | +| 2 | `3992840ddb3f` | FAIL | program defect: make figures stopped because seeds chose different 8-way aggregations for modernbert/k100 (argmax, summed, summed); Haiku 8600/8600, 0 parse failures, 20 predictions differ | US$3.178751 | + +- Reproduction-validation total (all AC1b attempts): US$3.178751 + +Reproduction-validation spend is never reported as original experiment cost and does not change the AC6 conclusion (Haiku spend at most US$5). + ## Reproducibility notes diff --git a/docs/PLAN.md b/docs/PLAN.md index b7f9709..8443460 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -130,6 +130,7 @@ TinyRouter 問三件事：**小模型要多少標註資料才夠？它知不知 - 續跑只限同一身分：每一步記錄 HEAD、`uv.lock` 與 `configs/` 的 SHA-256、Python、torch 與 transformers 版本；同一個 reproduction 目錄若由不同身分開始，拒絕續跑，必須換新 id，所以一份 comparison 不會混到兩個 commit 的輸出（第 1 條的落實方式）。 **預算規則**：原始 AC6 實驗的 Haiku 花費固定是 US$3.19（完整 run US$3.18 加 smoke），這個數字不再變動。AC1b 的 Haiku 是另一次、明確標記為 reproduction-validation 的執行，有獨立的 US$5 上限，花費寫在 reproduction 目錄自己的紀錄裡；兩者不得混稱為原始實驗成本，也不相加。 +- 每次 AC1b 嘗試各自獨立 US$5 上限（Drew，2026-09-30）。嘗試帳本：`docs/ac1b/attempts.json`（人讀版 `docs/ac1b/README.md`，各次證據在 `docs/ac1b/attempt-/`）。 ## 6. 風險 diff --git a/docs/ac1b/README.md b/docs/ac1b/README.md new file mode 100644 index 0000000..d5042f0 --- /dev/null +++ b/docs/ac1b/README.md @@ -0,0 +1,26 @@ +# AC1b attempts + +AC1b (docs/PLAN.md section 5.1) is the whole study rerun from a clean clone with `make setup && make reproduce`. This directory records every attempt, whether it passed or not, so that a failed attempt and its cost are never hidden by a later one. + +- `attempts.json`: the ledger, one entry per attempt: commit, reproduction id, start and end (UTC), result and reason, actual Haiku spend. `python -m tinyrouter.ac1b_ledger` checks it and rewrites the budget block below and in the top-level README; `tests/test_ac1b_ledger.py` fails when either is stale. +- `attempt-/`: the attempt's `comparison.md` as written by `make reproduce`, and `INCIDENT.md`, a note on why it stopped. + +Budget rules (Drew, 2026-09-29 and 2026-09-30): + +- The original AC6 experiment cost is fixed at US$3.19. No AC1b attempt changes it, and reproduction-validation spend is never reported as original experiment cost or used to revisit the AC6 conclusion (Haiku spend at most US$5). +- Every AC1b attempt has its own US$5 cap. `make reproduce` writes the Haiku journal under `reproduction//results` and runs it with `MAX_USD=5`; the cap counts only that journal, so an attempt at a new commit starts at US$0. Attempt 2 resumed attempt 1 under the same id and identity and so shared its journal and cap (attempt 1 spent US$0). +- A comparison never mixes the outputs of two commits. After a code fix, AC1b is run again in full at the new commit. + + +- Original experiment (AC6, fixed): US$3.19 +- AC1b attempts (reproduction-validation, each capped at US$5): + +| attempt | reproduction id | result | reason | Haiku spend | +|---|---|---|---|---| +| 1 | `3992840ddb3f` | FAIL | infrastructure: Haiku count_tokens returned HTTP 503 credential validation failed 5 times during an Anthropic authentication incident; no classification request sent | US$0.000000 | +| 2 | `3992840ddb3f` | FAIL | program defect: make figures stopped because seeds chose different 8-way aggregations for modernbert/k100 (argmax, summed, summed); Haiku 8600/8600, 0 parse failures, 20 predictions differ | US$3.178751 | + +- Reproduction-validation total (all AC1b attempts): US$3.178751 + +Reproduction-validation spend is never reported as original experiment cost and does not change the AC6 conclusion (Haiku spend at most US$5). + diff --git a/docs/ac1b/attempt-1/INCIDENT.md b/docs/ac1b/attempt-1/INCIDENT.md new file mode 100644 index 0000000..461bc4e --- /dev/null +++ b/docs/ac1b/attempt-1/INCIDENT.md @@ -0,0 +1,13 @@ +# AC1b attempt 1: interrupted by an external authentication incident + +- Reproduction id: `3992840ddb3f` (clean clone of `main` at `3992840ddb3f`, `uv.lock` SHA-256 prefix `d6b5940a86b5c28c`). +- Started: 2026-09-29 about 07:45 UTC. Failed: 2026-09-29 about 14:24 UTC. +- Completed and passed before the failure (all at `3992840ddb3f`): original-haiku, ac2, pilot-lr, pilot-steps, baselines, curve-bert, curve-modernbert, oos-ablation, verify-logits. + - AC2 rerun: 96.60 / 96.36 / 96.49 percent (seeds 42 / 43 / 44), all at or above 95.7. + - Pilots selected the same values as the original run (lr 5e-5 for both encoders, S_min 400). +- Failed step: `llm`. The pre-run token count (`messages.count_tokens`) returned HTTP 503 `InternalServerError` with the message "credential validation failed" on all 5 attempts. No classification request was sent. +- Cost of attempt 1: US$0.00 (token counting is not billed; the reproduction journal holds no records). +- Cause: an Anthropic incident, "Elevated errors on claude.ai, Claude Code and Claude Cowork", Investigating since 2026-09-29 14:21 UTC, described as failed requests and being asked to sign in again (status.claude.com). The API component was listed as operational, but requests from this machine received 503 during the incident window. +- Verdict of attempt 1 under the frozen criteria: FAIL (flow did not complete). Drew's decision (2026-09-29): this is an infrastructure availability failure, not a failure of training or of the protocol; resume after the incident with the same reproduction id and identity. If the same authentication failure recurs, pause and mark AC1b infrastructure-blocked. +- Before resuming (2026-09-30 01:13 UTC): a free token count succeeded; HEAD, working tree, and `uv.lock` unchanged; the reproduction LLM journal was empty, so no prediction is paid for twice. No hyperparameter, threshold, or result file was changed. +- Evidence kept: `comparison.md`, `comparison.json`, `steps.json`, and `run.log` from attempt 1 (no API key in any of them). diff --git a/docs/ac1b/attempt-1/comparison.md b/docs/ac1b/attempt-1/comparison.md new file mode 100644 index 0000000..c77a299 --- /dev/null +++ b/docs/ac1b/attempt-1/comparison.md @@ -0,0 +1,53 @@ +# AC1b comparison + +**Verdict: FAIL** (0 item(s) REVIEW REQUIRED). Criteria: docs/PLAN.md section 5.1 (frozen 2026-09-29). FAIL only when the flow or AC2 fails. + +## Flow + +| step | status | commit | completion line(s) | +|---|---|---|---| +| original-haiku | PASS | 3992840ddb3f | `verified 1/1 release files` | +| ac2 | PASS | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/ac2.json: PASS` | +| pilot-lr | PASS | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/pilots/lr.json` | +| pilot-steps | PASS | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/pilots/steps.json` | +| baselines | PASS | 3992840ddb3f | `completed 36/36 baseline points` | +| curve-bert | PASS | 3992840ddb3f | `completed 18/18 encoder points (bert)` | +| curve-modernbert | PASS | 3992840ddb3f | `completed 18/18 encoder points (modernbert)` | +| oos-ablation | PASS | 3992840ddb3f | `completed 3/3 ablation points` | +| verify-logits | PASS | 3992840ddb3f | `OK 75 archive(s) match reproduction/3992840ddb3f/results/logits-manifest.json` | +| llm | FAIL | 3992840ddb3f | none | + +## AC2 (threshold 95.7%, every seed): PASS + +| seed | rerun (%) | original (%) | passed | +|---|---|---|---| +| 42 | 96.60 | 96.51 | True | +| 43 | 96.36 | 96.40 | True | +| 44 | 96.49 | 96.22 | True | + +## Haiku (reproduction-validation run) + +```json +{ + "reached": false, + "checks_passed": false, + "problems": [ + "no reproduction run" + ] +} +``` + +## Budget (separate, never summed) + +- original AC6 experiment: US$3.19 (fixed) +- reproduction-validation: US$None (cap US$5) + +## Pilots + +| pilot | original | rerun | status | +|---|---|---|---| +| lr | {'bert': 5e-05, 'modernbert': 5e-05} | {'bert': 5e-05, 'modernbert': 5e-05} | OK | +| steps | 400 | 400 | OK | + +Numbers: not reached (the analysis step did not finish). + diff --git a/docs/ac1b/attempt-2/INCIDENT.md b/docs/ac1b/attempt-2/INCIDENT.md new file mode 100644 index 0000000..dc77ab1 --- /dev/null +++ b/docs/ac1b/attempt-2/INCIDENT.md @@ -0,0 +1,11 @@ +# AC1b attempt 2: stopped by a figure-code defect + +Written 2026-09-30 from the attempt's `run.log`, `steps.json` and `comparison.md` (the evidence directory had no incident note for this attempt). + +- Reproduction id: `3992840ddb3f`, the same id and identity as attempt 1 (commit `3992840ddb3f`, same `uv.lock`, `configs/`, Python, torch and transformers). It resumed attempt 1: the nine steps that had passed were skipped because their outputs still had the recorded SHA-256. +- Started (resumed `llm` step): 2026-09-30 01:13:40 UTC. Failed: 2026-09-30 01:31:53 UTC. +- Passed in this attempt: llm (Haiku 8,600/8,600, 0 parse failures, 20 predictions different from the original run), verify-llm, analysis, bench-cpu, llm-latency, cost. +- Haiku spend: US$3.178751, under the US$5 cap of this reproduction id (attempt 1 spent US$0 on the same journal). +- Failed step: `figures`. `figures.selected_aggregation` raised `ValueError: modernbert/k100: seeds chose different aggregations ['argmax', 'summed', 'summed']; pick one first`. In the original run all three ModernBERT k=100 seeds chose argmax on validation, so this path had never run. +- Verdict under the frozen criteria (docs/PLAN.md 5.1): FAIL (the flow did not complete). This is a defect in the program, not an infrastructure failure. +- Drew's decision (2026-09-30): fix the defect, then run AC1b again in full at the new commit as a new attempt with its own US$5 cap. The outputs of this attempt are not combined with figures or a report from another commit. diff --git a/docs/ac1b/attempt-2/comparison.md b/docs/ac1b/attempt-2/comparison.md new file mode 100644 index 0000000..d1fe52c --- /dev/null +++ b/docs/ac1b/attempt-2/comparison.md @@ -0,0 +1,344 @@ +# AC1b comparison + +**Verdict: FAIL** (192 item(s) REVIEW REQUIRED). Criteria: docs/PLAN.md section 5.1 (frozen 2026-09-29). FAIL only when the flow or AC2 fails. + +## Flow + +| step | status | commit | completion line(s) | +|---|---|---|---| +| original-haiku | PASS (resumed) | 3992840ddb3f | `verified 1/1 release files` | +| ac2 | PASS (resumed) | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/ac2.json: PASS` | +| pilot-lr | PASS (resumed) | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/pilots/lr.json` | +| pilot-steps | PASS (resumed) | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/pilots/steps.json` | +| baselines | PASS (resumed) | 3992840ddb3f | `completed 36/36 baseline points` | +| curve-bert | PASS (resumed) | 3992840ddb3f | `completed 18/18 encoder points (bert)` | +| curve-modernbert | PASS (resumed) | 3992840ddb3f | `completed 18/18 encoder points (modernbert)` | +| oos-ablation | PASS (resumed) | 3992840ddb3f | `completed 3/3 ablation points` | +| verify-logits | PASS (resumed) | 3992840ddb3f | `OK 75 archive(s) match reproduction/3992840ddb3f/results/logits-manifest.json` | +| llm | PASS | 3992840ddb3f | `completed 8600/8600 llm predictions` | +| verify-llm | PASS | 3992840ddb3f | `completed 8600/8600 llm predictions` | +| analysis | PASS | 3992840ddb3f | `completed analysis (75/75 archives, 8600/8600 llm rows, 25 groups)` | +| bench-cpu | PASS | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/efficiency/cpu_latency.json` | +| llm-latency | PASS | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/efficiency/haiku_latency.json` | +| cost | PASS | 3992840ddb3f | `wrote reproduction/3992840ddb3f/results/cost/cost.json` | +| figures | FAIL | 3992840ddb3f | none | + +## AC2 (threshold 95.7%, every seed): PASS + +| seed | rerun (%) | original (%) | passed | +|---|---|---|---| +| 42 | 96.60 | 96.51 | True | +| 43 | 96.36 | 96.40 | True | +| 44 | 96.49 | 96.22 | True | + +## Haiku (reproduction-validation run) + +```json +{ + "reached": true, + "rows": "8600/8600", + "parse_failed": 0, + "original_parse_failed": 0, + "reproduction_cost_usd": 3.178751, + "checks_passed": true, + "problems": [], + "predictions": { + "compared_rows": 8600, + "different_predictions": 20, + "only_in_original": 0, + "only_in_reproduction": 0 + } +} +``` + +## Budget (separate, never summed) + +- original AC6 experiment: US$3.19 (fixed) +- reproduction-validation: US$3.178751 (cap US$5) + +## Pilots + +| pilot | original | rerun | status | +|---|---|---|---| +| lr | {'bert': 5e-05, 'modernbert': 5e-05} | {'bert': 5e-05, 'modernbert': 5e-05} | OK | +| steps | 400 | 400 | OK | + +## README first screen (test) + +| metric | original mean (%) | original std (%) | rerun (%) | difference (pp) | status | +|---|---|---|---|---|---| +| ModernBERT k=100 small-only 8-way accuracy | 91.87 | 0.13 | 91.42 | -0.45 | REVIEW REQUIRED | +| Haiku 4.5 8-way accuracy | 82.07 | n/a | 81.96 | -0.11 | REVIEW REQUIRED | +| ModernBERT k=10 small-only 8-way accuracy | 81.50 | 0.69 | 80.56 | -0.94 | REVIEW REQUIRED | +| ModernBERT k=10 hybrid 8-way accuracy | 87.95 | 0.39 | 87.60 | -0.35 | OK | +| ModernBERT k=10 hybrid Haiku call rate | 23.88 | 3.76 | 27.24 | 3.36 | OK | +| k=100 threshold, test selective risk | 7.34 | 0.86 | 7.75 | 0.40 | OK | +| k=100 threshold, validation selective risk | 1.50 | 0.13 | 1.57 | 0.07 | OK | +| validation OOS share | 3.23 | 0.00 | 3.23 | 0.00 | OK | +| test OOS share | 18.18 | 0.00 | 18.18 | 0.00 | OK | +| share of the gap explained by OOS share | 86.66 | 1.39 | 87.73 | 1.07 | OK | +| k=100 kept-OOS error rate, test | 36.48 | 3.04 | 39.25 | 2.77 | OK | +| k=100 kept-OOS error rate, validation | 19.91 | 2.93 | 22.75 | 2.84 | OK | +| ModernBERT k=100 hybrid Haiku call rate | 1.27 | 1.59 | 1.27 | 0.00 | OK | +| ModernBERT k=100 hybrid 8-way accuracy | 92.13 | 0.14 | 91.63 | -0.50 | REVIEW REQUIRED | + +## README router table (test, 8-way) + +| metric | original mean (%) | original std (%) | rerun (%) | difference (pp) | status | +|---|---|---|---|---|---| +| LLM-only accuracy_8 | 82.07 | n/a | 81.96 | -0.11 | REVIEW REQUIRED | +| LLM-only oos_recall | 56.80 | n/a | 56.60 | -0.20 | REVIEW REQUIRED | +| LLM-only llm_call_rate | 100.00 | n/a | 100.00 | 0.00 | OK | +| modernbert/k10 small-only accuracy_8 | 81.50 | 0.69 | 80.56 | -0.94 | REVIEW REQUIRED | +| modernbert/k10 small-only oos_recall | 23.13 | 3.18 | 20.67 | -2.47 | OK | +| modernbert/k10 small-only high_conf_oos_misroute_rate | 76.87 | 3.18 | 79.33 | 2.47 | OK | +| modernbert/k10 small-only llm_call_rate | 0.00 | 0.00 | 0.00 | 0.00 | OK | +| modernbert/k10 hybrid 0.02 accuracy_8 | 87.95 | 0.39 | 87.60 | -0.35 | OK | +| modernbert/k10 hybrid 0.02 oos_recall | 53.47 | 1.46 | 53.13 | -0.33 | OK | +| modernbert/k10 hybrid 0.02 high_conf_oos_misroute_rate | 15.37 | 4.72 | 13.23 | -2.13 | OK | +| modernbert/k10 hybrid 0.02 llm_call_rate | 23.88 | 3.76 | 27.24 | 3.36 | OK | +| modernbert/k10 hybrid 0.05 accuracy_8 | 86.05 | 0.59 | 85.75 | -0.30 | OK | +| modernbert/k10 hybrid 0.05 oos_recall | 43.53 | 3.54 | 43.00 | -0.53 | OK | +| modernbert/k10 hybrid 0.05 high_conf_oos_misroute_rate | 40.07 | 7.53 | 37.90 | -2.17 | OK | +| modernbert/k10 hybrid 0.05 llm_call_rate | 11.75 | 3.28 | 13.32 | 1.56 | OK | +| modernbert/k10 oracle accuracy_8 | 91.45 | 0.22 | 91.10 | -0.36 | REVIEW REQUIRED | +| modernbert/k10 oracle oos_recall | 61.23 | 0.68 | 60.57 | -0.67 | OK | +| modernbert/k10 oracle high_conf_oos_misroute_rate | 0.00 | 0.00 | 0.00 | 0.00 | OK | +| modernbert/k10 oracle llm_call_rate | 18.50 | 0.69 | 19.44 | 0.94 | REVIEW REQUIRED | +| modernbert/k100 small-only accuracy_8 | 91.87 | 0.13 | 91.42 | -0.45 | REVIEW REQUIRED | +| modernbert/k100 small-only oos_recall | 61.07 | 0.38 | 58.47 | -2.60 | REVIEW REQUIRED | +| modernbert/k100 small-only high_conf_oos_misroute_rate | 38.93 | 0.38 | 41.53 | 2.60 | REVIEW REQUIRED | +| modernbert/k100 small-only llm_call_rate | 0.00 | 0.00 | 0.00 | 0.00 | OK | +| modernbert/k100 hybrid 0.02 accuracy_8 | 92.13 | 0.14 | 91.63 | -0.50 | REVIEW REQUIRED | +| modernbert/k100 hybrid 0.02 oos_recall | 62.43 | 1.33 | 59.47 | -2.97 | REVIEW REQUIRED | +| modernbert/k100 hybrid 0.02 high_conf_oos_misroute_rate | 34.53 | 5.53 | 36.83 | 2.30 | OK | +| modernbert/k100 hybrid 0.02 llm_call_rate | 1.27 | 1.59 | 1.27 | 0.00 | OK | +| modernbert/k100 hybrid 0.05 accuracy_8 | 91.93 | 0.13 | 91.57 | -0.36 | REVIEW REQUIRED | +| modernbert/k100 hybrid 0.05 oos_recall | 61.37 | 0.21 | 59.23 | -2.13 | REVIEW REQUIRED | +| modernbert/k100 hybrid 0.05 high_conf_oos_misroute_rate | 38.30 | 0.17 | 39.90 | 1.60 | REVIEW REQUIRED | +| modernbert/k100 hybrid 0.05 llm_call_rate | 0.16 | 0.06 | 0.35 | 0.19 | REVIEW REQUIRED | +| modernbert/k100 oracle accuracy_8 | 95.15 | 0.05 | 94.83 | -0.32 | REVIEW REQUIRED | +| modernbert/k100 oracle oos_recall | 75.60 | 0.60 | 74.00 | -1.60 | REVIEW REQUIRED | +| modernbert/k100 oracle high_conf_oos_misroute_rate | 0.00 | 0.00 | 0.00 | 0.00 | OK | +| modernbert/k100 oracle llm_call_rate | 8.13 | 0.13 | 8.58 | 0.45 | REVIEW REQUIRED | + +## OOS ablation (OOS 250 against OOS 0) + +| metric | original mean (%) | original std (%) | rerun (%) | difference (pp) | status | +|---|---|---|---|---|---| +| oos_0/hybrid_test/0.02/accuracy_8 | 88.32 | 0.18 | 87.95 | -0.37 | REVIEW REQUIRED | +| oos_0/hybrid_test/0.02/high_conf_oos_misroute_rate | 37.67 | 2.90 | 38.57 | 0.90 | OK | +| oos_0/hybrid_test/0.02/llm_call_rate | 12.07 | 0.73 | 11.82 | -0.25 | OK | +| oos_0/hybrid_test/0.02/oos_recall | 40.60 | 1.04 | 38.90 | -1.70 | REVIEW REQUIRED | +| oos_0/hybrid_test/0.05/accuracy_8 | 81.03 | 0.07 | 81.00 | -0.03 | OK | +| oos_0/hybrid_test/0.05/high_conf_oos_misroute_rate | 99.33 | 0.23 | 99.13 | -0.20 | OK | +| oos_0/hybrid_test/0.05/llm_call_rate | 0.12 | 0.04 | 0.16 | 0.04 | REVIEW REQUIRED | +| oos_0/hybrid_test/0.05/oos_recall | 0.53 | 0.21 | 0.73 | 0.20 | OK | +| oos_0/oos_detection_test/auprc | 91.52 | 0.26 | 91.53 | 0.01 | OK | +| oos_0/oos_detection_test/auroc | 97.80 | 0.04 | 97.84 | 0.04 | REVIEW REQUIRED | +| oos_0/small_only_oos_recall | 0.00 | 0.00 | 0.00 | 0.00 | OK | +| oos_250/hybrid_test/0.02/accuracy_8 | 92.13 | 0.14 | 91.63 | -0.50 | REVIEW REQUIRED | +| oos_250/hybrid_test/0.02/high_conf_oos_misroute_rate | 34.53 | 5.53 | 36.83 | 2.30 | OK | +| oos_250/hybrid_test/0.02/llm_call_rate | 1.27 | 1.59 | 1.27 | 0.00 | OK | +| oos_250/hybrid_test/0.02/oos_recall | 62.43 | 1.33 | 59.47 | -2.97 | REVIEW REQUIRED | +| oos_250/hybrid_test/0.05/accuracy_8 | 91.93 | 0.13 | 91.57 | -0.36 | REVIEW REQUIRED | +| oos_250/hybrid_test/0.05/high_conf_oos_misroute_rate | 38.30 | 0.17 | 39.90 | 1.60 | REVIEW REQUIRED | +| oos_250/hybrid_test/0.05/llm_call_rate | 0.16 | 0.06 | 0.35 | 0.19 | REVIEW REQUIRED | +| oos_250/hybrid_test/0.05/oos_recall | 61.37 | 0.21 | 59.23 | -2.13 | REVIEW REQUIRED | +| oos_250/oos_detection_test/auprc | 94.34 | 0.25 | 94.13 | -0.21 | OK | +| oos_250/oos_detection_test/auroc | 98.33 | 0.07 | 98.26 | -0.06 | OK | +| oos_250/small_only_oos_recall | 61.07 | 0.38 | 58.47 | -2.60 | REVIEW REQUIRED | + +## Machine-dependent numbers: 583 LISTED, NOT JUDGED (all rows in the JSON) + +Latency, k=100 training time and peak memory, and the cost model vary with the machine; they are listed with their differences and never marked REVIEW REQUIRED. + +## Learning curves (test, small model alone) + +| metric | original mean (%) | original std (%) | rerun (%) | difference (pp) | status | +|---|---|---|---|---|---| +| modernbert k=1 small-only accuracy_8 | 41.71 | 2.47 | 43.30 | 1.59 | OK | +| modernbert k=1 small-only oos_recall | 1.73 | 0.71 | 2.43 | 0.70 | OK | +| modernbert k=5 small-only accuracy_8 | 70.99 | 3.40 | 71.71 | 0.72 | OK | +| modernbert k=5 small-only oos_recall | 11.00 | 4.29 | 12.30 | 1.30 | OK | +| modernbert k=10 small-only accuracy_8 | 81.50 | 0.69 | 80.56 | -0.94 | REVIEW REQUIRED | +| modernbert k=10 small-only oos_recall | 23.13 | 3.18 | 20.67 | -2.47 | OK | +| modernbert k=25 small-only accuracy_8 | 86.27 | 0.52 | 87.21 | 0.94 | REVIEW REQUIRED | +| modernbert k=25 small-only oos_recall | 36.43 | 3.00 | 40.93 | 4.50 | REVIEW REQUIRED | +| modernbert k=50 small-only accuracy_8 | 90.39 | 0.38 | 90.25 | -0.14 | OK | +| modernbert k=50 small-only oos_recall | 53.87 | 2.32 | 53.83 | -0.03 | OK | +| modernbert k=100 small-only accuracy_8 | 91.87 | 0.13 | 91.42 | -0.45 | REVIEW REQUIRED | +| modernbert k=100 small-only oos_recall | 61.07 | 0.38 | 58.47 | -2.60 | REVIEW REQUIRED | +| bert k=1 small-only accuracy_8 | 54.93 | 1.32 | 55.28 | 0.35 | OK | +| bert k=1 small-only oos_recall | 9.90 | 6.17 | 9.47 | -0.43 | OK | +| bert k=5 small-only accuracy_8 | 78.99 | 1.61 | 78.79 | -0.20 | OK | +| bert k=5 small-only oos_recall | 19.10 | 6.95 | 18.00 | -1.10 | OK | +| bert k=10 small-only accuracy_8 | 81.90 | 1.18 | 82.16 | 0.27 | OK | +| bert k=10 small-only oos_recall | 24.07 | 4.20 | 25.47 | 1.40 | OK | +| bert k=25 small-only accuracy_8 | 86.52 | 0.65 | 86.16 | -0.36 | OK | +| bert k=25 small-only oos_recall | 38.00 | 3.18 | 35.53 | -2.47 | OK | +| bert k=50 small-only accuracy_8 | 88.85 | 0.30 | 88.86 | 0.01 | OK | +| bert k=50 small-only oos_recall | 44.83 | 1.55 | 45.17 | 0.33 | OK | +| bert k=100 small-only accuracy_8 | 91.05 | 0.53 | 90.76 | -0.29 | OK | +| bert k=100 small-only oos_recall | 56.07 | 2.61 | 54.50 | -1.57 | OK | + +## Threshold diagnostics: 155 of 758 flagged (all rows in the JSON) + +| metric | original mean (%) | original std (%) | rerun (%) | difference (pp) | status | +|---|---|---|---|---|---| +| bert/k1 0.02/sensitivity_reweighted_validation/test/kept_oos_error_rate | 60.00 | n/a | 66.67 | 6.67 | REVIEW REQUIRED | +| bert/k1 0.02/sensitivity_reweighted_validation/test/kept_oos_share | 3.88 | n/a | 4.11 | 0.23 | REVIEW REQUIRED | +| bert/k1 0.02/sensitivity_reweighted_validation/test/selective_risk | 2.33 | n/a | 2.74 | 0.41 | REVIEW REQUIRED | +| bert/k1 0.02/share_of_gap_explained_by_oos_share | 84.49 | n/a | 84.46 | -0.03 | REVIEW REQUIRED | +| bert/k1 0.02/test/kept_oos_error_rate | 60.00 | n/a | 66.67 | 6.67 | REVIEW REQUIRED | +| bert/k1 0.02/test/kept_oos_share | 3.88 | n/a | 4.11 | 0.23 | REVIEW REQUIRED | +| bert/k1 0.02/test/selective_risk | 2.33 | n/a | 2.74 | 0.41 | REVIEW REQUIRED | +| bert/k1 0.02/test_reweighted_to_validation_oos_share/kept_oos_error_rate | 60.00 | n/a | 66.67 | 6.67 | REVIEW REQUIRED | +| bert/k1 0.02/test_reweighted_to_validation_oos_share/kept_oos_share | 0.60 | n/a | 0.64 | 0.04 | REVIEW REQUIRED | +| bert/k1 0.02/test_reweighted_to_validation_oos_share/selective_risk | 0.36 | n/a | 0.43 | 0.07 | REVIEW REQUIRED | +| bert/k1 0.05/sensitivity_reweighted_validation/test/high_conf_oos_misroute_rate | 0.40 | 0.35 | 0.77 | 0.37 | REVIEW REQUIRED | +| bert/k1 0.05/sensitivity_reweighted_validation/validation_weighted_coverage | 3.02 | 2.90 | 5.98 | 2.97 | REVIEW REQUIRED | +| bert/k1 0.05/validation/kept_oos_error_rate | 100.00 | 0.00 | 66.67 | -33.33 | REVIEW REQUIRED | +| bert/k1 0.05/validation/kept_oos_share | 0.43 | 0.16 | 0.78 | 0.35 | REVIEW REQUIRED | +| bert/k1 0.05/validation/selective_risk | 2.83 | 0.18 | 2.64 | -0.19 | REVIEW REQUIRED | +| bert/k10 0.02/sensitivity_reweighted_validation/tau | 96.88 | 2.54 | n/a | n/a | REVIEW REQUIRED | +| bert/k10 0.02/sensitivity_reweighted_validation/validation_weighted_coverage | 27.00 | 5.36 | 32.65 | 5.65 | REVIEW REQUIRED | +| bert/k10 0.02/tau | 81.51 | 7.57 | n/a | n/a | REVIEW REQUIRED | +| bert/k10 0.05/sensitivity_reweighted_validation/test/kept_oos_share | 3.85 | 0.84 | 4.84 | 0.99 | REVIEW REQUIRED | +| bert/k10 0.05/validation/kept_oos_share | 1.84 | 0.02 | 1.95 | 0.11 | REVIEW REQUIRED | +| bert/k100 0.02/test/high_conf_oos_misroute_rate | 27.27 | 1.07 | 29.57 | 2.30 | REVIEW REQUIRED | +| bert/k100 0.02/test/kept_oos_error_rate | 35.48 | 2.24 | 38.06 | 2.58 | REVIEW REQUIRED | +| bert/k100 0.02/test/selective_risk | 5.98 | 0.10 | 6.43 | 0.45 | REVIEW REQUIRED | +| bert/k100 0.02/test_reweighted_to_validation_oos_share/high_conf_oos_misroute_rate | 27.27 | 1.07 | 29.57 | 2.30 | REVIEW REQUIRED | +| bert/k100 0.02/test_reweighted_to_validation_oos_share/kept_oos_error_rate | 35.48 | 2.24 | 38.06 | 2.58 | REVIEW REQUIRED | +| bert/k100 0.02/test_reweighted_to_validation_oos_share/selective_risk | 1.77 | 0.08 | 1.88 | 0.10 | REVIEW REQUIRED | +| bert/k100 0.05/sensitivity_reweighted_validation/test/coverage | 93.12 | 0.62 | 93.79 | 0.68 | REVIEW REQUIRED | +| bert/k100 0.05/sensitivity_reweighted_validation/test/high_conf_oos_misroute_rate | 21.17 | 1.62 | 24.53 | 3.37 | REVIEW REQUIRED | +| bert/k100 0.05/sensitivity_reweighted_validation/test/kept_oos_error_rate | 31.11 | 2.89 | 34.91 | 3.80 | REVIEW REQUIRED | +| bert/k100 0.05/sensitivity_reweighted_validation/test/selective_risk | 4.80 | 0.40 | 5.46 | 0.65 | REVIEW REQUIRED | +| bert/k25 0.02/sensitivity_reweighted_validation/tau | 21.61 | 8.44 | n/a | n/a | REVIEW REQUIRED | +| bert/k25 0.02/sensitivity_reweighted_validation/test/kept_oos_share | 5.54 | 0.98 | 4.17 | -1.37 | REVIEW REQUIRED | +| bert/k25 0.02/tau | 11.31 | 2.09 | n/a | n/a | REVIEW REQUIRED | +| bert/k25 0.02/validation/high_conf_oos_misroute_rate | 12.67 | 2.52 | 17.00 | 4.33 | REVIEW REQUIRED | +| bert/k25 0.02/validation/in_scope_risk | 1.13 | 0.06 | 0.97 | -0.16 | REVIEW REQUIRED | +| bert/k25 0.02/validation/kept_oos_error_rate | 27.27 | 2.40 | 32.30 | 5.03 | REVIEW REQUIRED | +| bert/k25 0.02/validation/kept_oos_share | 1.62 | 0.23 | 1.89 | 0.27 | REVIEW REQUIRED | +| bert/k25 0.05/sensitivity_reweighted_validation/tau | 10.62 | 1.65 | n/a | n/a | REVIEW REQUIRED | +| bert/k25 0.05/sensitivity_reweighted_validation/test/coverage | 84.67 | 3.15 | 81.42 | -3.25 | REVIEW REQUIRED | +| bert/k25 0.05/sensitivity_reweighted_validation/test/in_scope_risk | 1.33 | 0.26 | 0.94 | -0.38 | REVIEW REQUIRED | +| bert/k25 0.05/sensitivity_reweighted_validation/validation_weighted_coverage | 86.37 | 2.25 | 83.66 | -2.70 | REVIEW REQUIRED | +| bert/k25 0.05/sensitivity_reweighted_validation/validation_weighted_risk | 4.06 | 0.05 | 4.11 | 0.05 | REVIEW REQUIRED | +| bert/k25 0.05/tau | 2.32 | 0.17 | n/a | n/a | REVIEW REQUIRED | +| bert/k25 0.05/test/coverage | 99.96 | 0.03 | 99.59 | -0.37 | REVIEW REQUIRED | +| bert/k25 0.05/test/kept_oos_share | 18.15 | 0.02 | 17.87 | -0.27 | REVIEW REQUIRED | +| bert/k25 0.05/test_reweighted_to_validation_oos_share/coverage | 99.99 | 0.00 | 99.90 | -0.10 | REVIEW REQUIRED | +| bert/k25 0.05/test_reweighted_to_validation_oos_share/kept_oos_share | 3.22 | 0.00 | 3.16 | -0.06 | REVIEW REQUIRED | +| bert/k25 0.05/validation/coverage | 100.00 | 0.00 | 99.97 | -0.03 | REVIEW REQUIRED | +| bert/k25 0.05/validation/kept_oos_share | 3.23 | 0.00 | 3.19 | -0.03 | REVIEW REQUIRED | +| bert/k5 0.02/sensitivity_reweighted_validation/tau | n/a | n/a | 98.40 | n/a | REVIEW REQUIRED | +| bert/k5 0.02/tau | n/a | n/a | 87.51 | n/a | REVIEW REQUIRED | +| bert/k5 0.02/test/coverage | 58.16 | 0.86 | 59.78 | 1.61 | REVIEW REQUIRED | +| bert/k5 0.02/test_reweighted_to_validation_oos_share/coverage | 66.07 | 0.97 | 67.95 | 1.88 | REVIEW REQUIRED | +| bert/k5 0.02/validation/coverage | 66.81 | 1.41 | 68.41 | 1.60 | REVIEW REQUIRED | +| bert/k5 0.05/sensitivity_reweighted_validation/validation_weighted_risk | 4.10 | 0.03 | 4.07 | -0.03 | REVIEW REQUIRED | +| bert/k5 0.05/tau | 60.05 | 1.53 | 57.94 | -2.11 | REVIEW REQUIRED | +| bert/k5 0.05/test/coverage | 81.46 | 1.09 | 82.74 | 1.28 | REVIEW REQUIRED | +| bert/k5 0.05/test_reweighted_to_validation_oos_share/coverage | 88.55 | 1.02 | 89.60 | 1.05 | REVIEW REQUIRED | +| bert/k5 0.05/validation/coverage | 89.11 | 0.47 | 89.76 | 0.66 | REVIEW REQUIRED | +| bert/k50 0.02/sensitivity_reweighted_validation/tau | n/a | n/a | -114.56 | n/a | REVIEW REQUIRED | +| bert/k50 0.02/sensitivity_reweighted_validation/validation_weighted_risk | 1.33 | 0.09 | 1.42 | 0.09 | REVIEW REQUIRED | +| bert/k50 0.02/tau | n/a | n/a | -246.17 | n/a | REVIEW REQUIRED | +| bert/k50 0.05/test/in_scope_risk | 1.36 | 0.03 | 1.42 | 0.05 | REVIEW REQUIRED | +| bert/k50 0.05/test_reweighted_to_validation_oos_share/in_scope_risk | 1.36 | 0.03 | 1.42 | 0.05 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.02/sensitivity_reweighted_validation/tau | -13.82 | 1.04 | n/a | n/a | REVIEW REQUIRED | +| modernbert-oos0/k100 0.02/sensitivity_reweighted_validation/test/in_scope_risk | 0.18 | 0.06 | 0.32 | 0.14 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.02/sensitivity_reweighted_validation/test/selective_risk | 2.19 | 0.57 | 2.81 | 0.62 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.02/share_of_gap_explained_by_oos_share | 93.87 | 1.34 | 92.22 | -1.65 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.02/tau | -119.38 | 44.08 | n/a | n/a | REVIEW REQUIRED | +| modernbert-oos0/k100 0.02/test/in_scope_risk | 0.75 | 0.03 | 0.86 | 0.11 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.02/test_reweighted_to_validation_oos_share/in_scope_risk | 0.75 | 0.03 | 0.86 | 0.11 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.02/test_reweighted_to_validation_oos_share/selective_risk | 1.99 | 0.12 | 2.13 | 0.14 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/sensitivity_reweighted_validation/tau | n/a | n/a | 50.79 | n/a | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/sensitivity_reweighted_validation/test/high_conf_oos_misroute_rate | 24.13 | 1.33 | 22.33 | -1.80 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/sensitivity_reweighted_validation/test/in_scope_risk | 0.50 | 0.04 | 0.56 | 0.06 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/sensitivity_reweighted_validation/test/kept_oos_share | 5.18 | 0.26 | 4.82 | -0.36 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/sensitivity_reweighted_validation/test/selective_risk | 5.65 | 0.30 | 5.35 | -0.30 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/sensitivity_reweighted_validation/validation_weighted_coverage | 83.84 | 0.25 | 83.57 | -0.27 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/share_of_gap_explained_by_oos_share | 97.97 | 0.26 | 97.33 | -0.64 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/tau | n/a | n/a | 10.34 | n/a | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/test/coverage | 99.88 | 0.04 | 99.84 | -0.04 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/test/in_scope_risk | 1.08 | 0.05 | 1.16 | 0.07 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/test_reweighted_to_validation_oos_share/coverage | 99.98 | 0.01 | 99.96 | -0.01 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/test_reweighted_to_validation_oos_share/in_scope_risk | 1.08 | 0.05 | 1.16 | 0.07 | REVIEW REQUIRED | +| modernbert-oos0/k100 0.05/test_reweighted_to_validation_oos_share/selective_risk | 4.25 | 0.05 | 4.32 | 0.07 | REVIEW REQUIRED | +| modernbert/k1 0.05/sensitivity_reweighted_validation/test/kept_oos_error_rate | 100.00 | 0.00 | 91.67 | -8.33 | REVIEW REQUIRED | +| modernbert/k1 0.05/test/kept_oos_error_rate | 100.00 | 0.00 | 91.67 | -8.33 | REVIEW REQUIRED | +| modernbert/k1 0.05/test_reweighted_to_validation_oos_share/kept_oos_error_rate | 100.00 | 0.00 | 91.67 | -8.33 | REVIEW REQUIRED | +| modernbert/k10 0.02/sensitivity_reweighted_validation/test/coverage | 61.01 | 0.49 | 62.42 | 1.41 | REVIEW REQUIRED | +| modernbert/k10 0.02/sensitivity_reweighted_validation/test/high_conf_oos_misroute_rate | 3.93 | 0.78 | 5.77 | 1.83 | REVIEW REQUIRED | +| modernbert/k10 0.02/sensitivity_reweighted_validation/test/in_scope_risk | 0.48 | 0.04 | 0.61 | 0.13 | REVIEW REQUIRED | +| modernbert/k10 0.02/sensitivity_reweighted_validation/test/kept_oos_error_rate | 54.19 | 0.72 | 61.73 | 7.54 | REVIEW REQUIRED | +| modernbert/k10 0.02/sensitivity_reweighted_validation/test/kept_oos_share | 2.16 | 0.42 | 2.75 | 0.59 | REVIEW REQUIRED | +| modernbert/k10 0.02/sensitivity_reweighted_validation/test/selective_risk | 1.64 | 0.21 | 2.26 | 0.62 | REVIEW REQUIRED | +| modernbert/k10 0.02/sensitivity_reweighted_validation/validation_weighted_coverage | 61.15 | 1.32 | 63.37 | 2.22 | REVIEW REQUIRED | +| modernbert/k10 0.02/sensitivity_reweighted_validation/validation_weighted_risk | 1.28 | 0.07 | 1.39 | 0.11 | REVIEW REQUIRED | +| modernbert/k10 0.02/validation/high_conf_oos_misroute_rate | 11.00 | 0.00 | 9.67 | -1.33 | REVIEW REQUIRED | +| modernbert/k10 0.02/validation/kept_oos_share | 1.03 | 0.12 | 0.87 | -0.16 | REVIEW REQUIRED | +| modernbert/k10 0.05/validation/selective_risk | 4.31 | 0.01 | 4.33 | 0.01 | REVIEW REQUIRED | +| modernbert/k100 0.02/sensitivity_reweighted_validation/test/in_scope_risk | 0.53 | 0.10 | 0.39 | -0.14 | REVIEW REQUIRED | +| modernbert/k100 0.05/sensitivity_reweighted_validation/test/in_scope_risk | 1.23 | 0.08 | 1.11 | -0.11 | REVIEW REQUIRED | +| modernbert/k100 0.05/sensitivity_reweighted_validation/validation_weighted_risk | 4.12 | 0.06 | 4.05 | -0.07 | REVIEW REQUIRED | +| modernbert/k100 0.05/test/coverage | 99.84 | 0.06 | 99.65 | -0.19 | REVIEW REQUIRED | +| modernbert/k100 0.05/test/high_conf_oos_misroute_rate | 38.30 | 0.17 | 39.90 | 1.60 | REVIEW REQUIRED | +| modernbert/k100 0.05/test/kept_oos_error_rate | 38.57 | 0.12 | 40.63 | 2.06 | REVIEW REQUIRED | +| modernbert/k100 0.05/test/kept_oos_share | 18.08 | 0.07 | 17.91 | -0.17 | REVIEW REQUIRED | +| modernbert/k100 0.05/test/selective_risk | 8.01 | 0.08 | 8.30 | 0.29 | REVIEW REQUIRED | +| modernbert/k100 0.05/test_reweighted_to_validation_oos_share/coverage | 99.94 | 0.01 | 99.93 | -0.02 | REVIEW REQUIRED | +| modernbert/k100 0.05/test_reweighted_to_validation_oos_share/high_conf_oos_misroute_rate | 38.30 | 0.17 | 39.90 | 1.60 | REVIEW REQUIRED | +| modernbert/k100 0.05/test_reweighted_to_validation_oos_share/kept_oos_error_rate | 38.57 | 0.12 | 40.63 | 2.06 | REVIEW REQUIRED | +| modernbert/k100 0.05/test_reweighted_to_validation_oos_share/kept_oos_share | 3.21 | 0.02 | 3.17 | -0.04 | REVIEW REQUIRED | +| modernbert/k25 0.02/sensitivity_reweighted_validation/test/high_conf_oos_misroute_rate | 6.20 | 0.26 | 6.57 | 0.37 | REVIEW REQUIRED | +| modernbert/k25 0.02/sensitivity_reweighted_validation/test/kept_oos_error_rate | 42.56 | 0.95 | 35.31 | -7.25 | REVIEW REQUIRED | +| modernbert/k25 0.02/sensitivity_reweighted_validation/test/kept_oos_share | 3.57 | 0.15 | 4.56 | 0.99 | REVIEW REQUIRED | +| modernbert/k25 0.02/test/kept_oos_error_rate | 49.52 | 4.21 | 44.33 | -5.19 | REVIEW REQUIRED | +| modernbert/k25 0.02/test/kept_oos_share | 8.62 | 0.30 | 9.54 | 0.92 | REVIEW REQUIRED | +| modernbert/k25 0.02/test_reweighted_to_validation_oos_share/kept_oos_error_rate | 49.52 | 4.21 | 44.33 | -5.19 | REVIEW REQUIRED | +| modernbert/k25 0.02/test_reweighted_to_validation_oos_share/kept_oos_share | 1.40 | 0.05 | 1.56 | 0.16 | REVIEW REQUIRED | +| modernbert/k25 0.02/validation/high_conf_oos_misroute_rate | 14.33 | 1.53 | 11.00 | -3.33 | REVIEW REQUIRED | +| modernbert/k25 0.02/validation/in_scope_risk | 1.09 | 0.05 | 1.20 | 0.11 | REVIEW REQUIRED | +| modernbert/k25 0.02/validation/kept_oos_error_rate | 32.81 | 1.66 | 24.00 | -8.81 | REVIEW REQUIRED | +| modernbert/k25 0.02/validation/selective_risk | 1.56 | 0.00 | 1.56 | -0.01 | REVIEW REQUIRED | +| modernbert/k25 0.05/sensitivity_reweighted_validation/test/coverage | 85.41 | 0.99 | 87.78 | 2.38 | REVIEW REQUIRED | +| modernbert/k25 0.05/sensitivity_reweighted_validation/test/high_conf_oos_misroute_rate | 20.10 | 0.62 | 23.17 | 3.07 | REVIEW REQUIRED | +| modernbert/k25 0.05/sensitivity_reweighted_validation/test/kept_oos_share | 8.70 | 0.54 | 10.34 | 1.65 | REVIEW REQUIRED | +| modernbert/k25 0.05/sensitivity_reweighted_validation/test/selective_risk | 5.20 | 0.57 | 5.85 | 0.65 | REVIEW REQUIRED | +| modernbert/k25 0.05/sensitivity_reweighted_validation/validation_weighted_coverage | 85.95 | 1.42 | 88.49 | 2.55 | REVIEW REQUIRED | +| modernbert/k25 0.05/sensitivity_reweighted_validation/validation_weighted_risk | 4.09 | 0.02 | 4.05 | -0.03 | REVIEW REQUIRED | +| modernbert/k25 0.05/test/kept_oos_error_rate | 63.18 | 3.12 | 59.02 | -4.16 | REVIEW REQUIRED | +| modernbert/k25 0.05/test/selective_risk | 13.51 | 0.46 | 12.74 | -0.77 | REVIEW REQUIRED | +| modernbert/k25 0.05/test_reweighted_to_validation_oos_share/kept_oos_error_rate | 63.18 | 3.12 | 59.02 | -4.16 | REVIEW REQUIRED | +| modernbert/k25 0.05/validation/high_conf_oos_misroute_rate | 51.33 | 3.79 | 45.67 | -5.67 | REVIEW REQUIRED | +| modernbert/k25 0.05/validation/kept_oos_error_rate | 52.28 | 4.96 | 45.67 | -6.61 | REVIEW REQUIRED | +| modernbert/k5 0.02/test/high_conf_oos_misroute_rate | 4.00 | 0.69 | 4.97 | 0.97 | REVIEW REQUIRED | +| modernbert/k5 0.02/test/selective_risk | 3.31 | 0.22 | 3.55 | 0.24 | REVIEW REQUIRED | +| modernbert/k5 0.02/test_reweighted_to_validation_oos_share/high_conf_oos_misroute_rate | 4.00 | 0.69 | 4.97 | 0.97 | REVIEW REQUIRED | +| modernbert/k5 0.02/validation/high_conf_oos_misroute_rate | 0.67 | 0.58 | 2.33 | 1.67 | REVIEW REQUIRED | +| modernbert/k5 0.02/validation/in_scope_risk | 1.33 | 0.02 | 1.26 | -0.07 | REVIEW REQUIRED | +| modernbert/k5 0.02/validation/kept_oos_error_rate | 8.33 | 7.22 | 15.87 | 7.54 | REVIEW REQUIRED | +| modernbert/k5 0.02/validation/kept_oos_share | 0.45 | 0.12 | 0.58 | 0.13 | REVIEW REQUIRED | +| modernbert/k5 0.05/sensitivity_reweighted_validation/tau | n/a | n/a | 202.52 | n/a | REVIEW REQUIRED | +| modernbert/k5 0.05/tau | n/a | n/a | 144.06 | n/a | REVIEW REQUIRED | +| modernbert/k5 0.05/validation/in_scope_risk | 3.68 | 0.04 | 3.60 | -0.08 | REVIEW REQUIRED | +| modernbert/k50 0.02/sensitivity_reweighted_validation/tau | 79.38 | 11.04 | n/a | n/a | REVIEW REQUIRED | +| modernbert/k50 0.02/sensitivity_reweighted_validation/test/in_scope_risk | 0.55 | 0.16 | 0.76 | 0.21 | REVIEW REQUIRED | +| modernbert/k50 0.02/sensitivity_reweighted_validation/validation_weighted_risk | 1.35 | 0.02 | 1.42 | 0.07 | REVIEW REQUIRED | +| modernbert/k50 0.02/tau | 35.06 | 7.34 | n/a | n/a | REVIEW REQUIRED | +| modernbert/k50 0.05/sensitivity_reweighted_validation/tau | n/a | n/a | -238.19 | n/a | REVIEW REQUIRED | +| modernbert/k50 0.05/sensitivity_reweighted_validation/test/in_scope_risk | 1.17 | 0.10 | 1.35 | 0.18 | REVIEW REQUIRED | +| modernbert/k50 0.05/sensitivity_reweighted_validation/validation_weighted_risk | 4.11 | 0.04 | 4.16 | 0.04 | REVIEW REQUIRED | +| modernbert/k50 0.05/share_of_gap_explained_by_oos_share | 89.41 | 1.38 | 88.02 | -1.39 | REVIEW REQUIRED | +| modernbert/k50 0.05/tau | n/a | n/a | -371.87 | n/a | REVIEW REQUIRED | +| modernbert/k50 0.05/test/in_scope_risk | 1.48 | 0.10 | 1.65 | 0.17 | REVIEW REQUIRED | +| modernbert/k50 0.05/test_reweighted_to_validation_oos_share/in_scope_risk | 1.48 | 0.10 | 1.65 | 0.17 | REVIEW REQUIRED | +| modernbert/k50 0.05/test_reweighted_to_validation_oos_share/selective_risk | 2.91 | 0.07 | 3.08 | 0.17 | REVIEW REQUIRED | + diff --git a/docs/ac1b/attempts.json b/docs/ac1b/attempts.json new file mode 100644 index 0000000..c328b43 --- /dev/null +++ b/docs/ac1b/attempts.json @@ -0,0 +1,34 @@ +{ + "format_version": 1, + "original_ac6_usd": 3.19, + "cap_usd_per_attempt": 5.0, + "cap_rule": "each AC1b attempt has its own US$5 cap (Drew, 2026-09-30); enforced per reproduction id, whose Haiku journal starts empty", + "attempts": [ + { + "attempt": 1, + "commit": "3992840ddb3ff1df8b879ba00aaee6277e3f7ce4", + "reproduction_id": "3992840ddb3f", + "started_utc": "2026-09-29T07:40:02Z", + "finished_utc": "2026-09-29T14:24:22Z", + "time_source": "steps.json: start of the first step, end of the failed llm step", + "result": "FAIL", + "failure_kind": "infrastructure", + "reason": "infrastructure: Haiku count_tokens returned HTTP 503 credential validation failed 5 times during an Anthropic authentication incident; no classification request sent", + "haiku_usd": 0.0, + "evidence": "docs/ac1b/attempt-1/" + }, + { + "attempt": 2, + "commit": "3992840ddb3ff1df8b879ba00aaee6277e3f7ce4", + "reproduction_id": "3992840ddb3f", + "started_utc": "2026-09-30T01:13:40Z", + "finished_utc": "2026-09-30T01:31:53Z", + "time_source": "steps.json: start of the resumed llm step, end of the failed figures step", + "result": "FAIL", + "failure_kind": "program defect", + "reason": "program defect: make figures stopped because seeds chose different 8-way aggregations for modernbert/k100 (argmax, summed, summed); Haiku 8600/8600, 0 parse failures, 20 predictions differ", + "haiku_usd": 3.178751, + "evidence": "docs/ac1b/attempt-2/" + } + ] +} diff --git a/src/tinyrouter/ac1b_ledger.py b/src/tinyrouter/ac1b_ledger.py new file mode 100644 index 0000000..58abdf4 --- /dev/null +++ b/src/tinyrouter/ac1b_ledger.py @@ -0,0 +1,186 @@ +"""The AC1b attempt ledger (``docs/ac1b/attempts.json``) and the three-part budget. + +docs/PLAN.md 5.1: the original AC6 experiment cost is fixed at US$3.19; +every AC1b attempt is a reproduction-validation run with its own US$5 +cap; their spend is reported per attempt and as a reproduction-validation +total, and never added to, or reported as, the original experiment cost. + +The cap is enforced per reproduction id: ``make reproduce`` writes the +Haiku journal under ``reproduction//results`` and runs it with +``MAX_USD=5``, and ``llm_run`` counts only that journal. A new commit gets +a new id, so a new attempt starts from US$0. An attempt that resumes the +same id under the same identity (attempt 2 resumed attempt 1) shares that +id's journal and therefore its cap. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +LEDGER = Path("docs/ac1b/attempts.json") +REPO = Path(__file__).resolve().parents[2] +ORIGINAL_AC6_USD = 3.19 +CAP_USD_PER_ATTEMPT = 5.0 +RESULTS = ("PASS", "FAIL", "INFRASTRUCTURE INTERRUPTED") +FIELDS = ( + "attempt", + "commit", + "reproduction_id", + "started_utc", + "finished_utc", + "result", + "reason", + "haiku_usd", +) + + +class LedgerError(ValueError): + """The ledger is malformed or breaks a budget rule.""" + + +def validate(ledger: dict) -> dict: + """Attempts numbered 1..n, a known result, a reason, spend within the per-attempt cap.""" + if ledger.get("original_ac6_usd") != ORIGINAL_AC6_USD: + raise LedgerError(f"original_ac6_usd must stay {ORIGINAL_AC6_USD}") + if ledger.get("cap_usd_per_attempt") != CAP_USD_PER_ATTEMPT: + raise LedgerError(f"cap_usd_per_attempt must be {CAP_USD_PER_ATTEMPT}") + attempts = ledger.get("attempts") + if not isinstance(attempts, list) or not attempts: + raise LedgerError("no attempts recorded") + for number, entry in enumerate(attempts, start=1): + missing = [f for f in FIELDS if f not in entry] + if missing: + raise LedgerError(f"attempt {number}: missing {missing}") + if entry["attempt"] != number: + raise LedgerError(f"attempt numbers must run 1..n; found {entry['attempt']}") + if entry["result"] not in RESULTS: + raise LedgerError(f"attempt {number}: result {entry['result']!r} not in {RESULTS}") + if not str(entry["reason"]).strip(): + raise LedgerError(f"attempt {number}: a result needs a reason") + spent = entry["haiku_usd"] + if not isinstance(spent, int | float) or not 0 <= spent <= CAP_USD_PER_ATTEMPT: + raise LedgerError(f"attempt {number}: haiku_usd {spent!r} outside 0..cap") + return ledger + + +def load(path: Path = REPO / LEDGER) -> dict: + return validate(json.loads(path.read_text(encoding="utf-8"))) + + +def total_usd(attempts: list[dict]) -> float: + return round(sum(float(a["haiku_usd"]) for a in attempts), 6) + + +def budget(ledger: dict, current: dict | None = None) -> dict[str, object]: + """Original (fixed), each attempt, and the reproduction-validation total. + + ``current`` is the run being compared (``reproduction_id``, ``usd``) + when its id has no ledger entry yet; it is listed as pending and + counted in the total. When the id is already in the ledger, the + ledger's figures are used and ``current`` adds nothing. + """ + attempts = [dict(a) for a in ledger["attempts"]] + recorded_ids = {a["reproduction_id"] for a in attempts} + if current is not None and current["reproduction_id"] not in recorded_ids: + attempts.append( + { + "attempt": len(attempts) + 1, + "commit": current.get("commit"), + "reproduction_id": current["reproduction_id"], + "result": "PENDING", + "reason": "this comparison; add it to docs/ac1b/attempts.json", + "haiku_usd": current.get("usd") or 0.0, + } + ) + return { + "original_ac6_experiment": { + "usd": ORIGINAL_AC6_USD, + "note": "fixed: full run US$3.18 plus smoke; no AC1b attempt changes it", + }, + "ac1b_attempts": [ + {k: a.get(k) for k in ("attempt", "reproduction_id", "result", "reason", "haiku_usd")} + for a in attempts + ], + "reproduction_validation_total": { + "usd": total_usd(attempts), + "note": "sum over AC1b attempts only; never added to the original experiment cost", + }, + "cap_usd_per_attempt": CAP_USD_PER_ATTEMPT, + "rule": ( + "reproduction-validation spend is never reported as original experiment cost and " + "does not change the AC6 US$5 conclusion" + ), + } + + +def markdown(section: dict) -> list[str]: + """The three-part budget as markdown lines.""" + lines = [ + f"- Original experiment (AC6, fixed): US${section['original_ac6_experiment']['usd']:.2f}", + f"- AC1b attempts (reproduction-validation, each capped at " + f"US${section['cap_usd_per_attempt']:g}):", + "", + "| attempt | reproduction id | result | reason | Haiku spend |", + "|---|---|---|---|---|", + ] + for a in section["ac1b_attempts"]: + lines.append( + f"| {a['attempt']} | `{a['reproduction_id']}` | {a['result']} | {a['reason']} | " + f"US${a['haiku_usd']:.6f} |" + ) + total = section["reproduction_validation_total"]["usd"] + return [ + *lines, + "", + f"- Reproduction-validation total (all AC1b attempts): US${total:.6f}", + "", + "Reproduction-validation spend is never reported as original experiment cost and does " + "not change the AC6 conclusion (Haiku spend at most US$5).", + ] + + +BEGIN = "" +END = "" +BUDGET_DOCS = (Path("README.md"), Path("docs/ac1b/README.md")) + + +def block(ledger: dict) -> str: + return "\n".join([BEGIN, *markdown(budget(ledger)), END]) + + +def with_block(text: str, ledger: dict) -> str: + start, end = text.find(BEGIN), text.find(END) + if start < 0 or end < start: + raise LedgerError("no AC1b budget block (BEGIN ... END markers)") + return text[:start] + block(ledger) + text[end + len(END) :] + + +def main(argv: list[str] | None = None) -> None: + """Rewrite the budget block in README.md and docs/ac1b/README.md; ``--check`` only compares.""" + import argparse + import sys + + parser = argparse.ArgumentParser(description=main.__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args(argv) + ledger = load(REPO / LEDGER) + stale = [] + for doc in BUDGET_DOCS: + path = REPO / doc + text = path.read_text(encoding="utf-8") + new = with_block(text, ledger) + if new == text: + continue + if args.check: + stale.append(str(doc)) + else: + path.write_text(new, encoding="utf-8") + print(f"wrote {doc}") + if stale: + print(f"stale, run `python -m tinyrouter.ac1b_ledger`: {', '.join(stale)}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/tinyrouter/comparison.py b/src/tinyrouter/comparison.py index 3a46e47..666ac4d 100644 --- a/src/tinyrouter/comparison.py +++ b/src/tinyrouter/comparison.py @@ -38,6 +38,7 @@ from collections.abc import Iterator from pathlib import Path +from tinyrouter import ac1b_ledger from tinyrouter.ac2 import SEEDS as AC2_SEEDS from tinyrouter.ac2 import THRESHOLD as AC2_THRESHOLD @@ -48,9 +49,9 @@ FAIL = "FAIL" EXACT_TOLERANCE = 1e-9 EXPECTED_LLM_ROWS = 8600 -# docs/PLAN.md 5.1: the original AC6 spend is fixed; the rerun has its own cap. -ORIGINAL_AC6_USD = 3.19 -REPRODUCTION_CAP_USD = 5.0 +# docs/PLAN.md 5.1: the original AC6 spend is fixed; each AC1b attempt has its own cap. +ORIGINAL_AC6_USD = ac1b_ledger.ORIGINAL_AC6_USD +REPRODUCTION_CAP_USD = ac1b_ledger.CAP_USD_PER_ATTEMPT TARGET = "0.02" CURVE_MODELS = ("modernbert", "bert") CURVE_KS = (1, 5, 10, 25, 50, 100) @@ -331,23 +332,21 @@ def haiku_section( return body -def budget_section(reproduced_summary: dict | None) -> dict[str, object]: - """Two separate budgets; the rerun's spend is never added to the original experiment's.""" +def budget_section( + reproduced_summary: dict | None, ledger: dict, reproduction_id: str | None = None +) -> dict[str, object]: + """Three parts: the fixed original AC6 cost, each AC1b attempt, and their total. + + The rerun's spend is never added to the original experiment's + (``ac1b_ledger.budget``). + """ spent = None if reproduced_summary is not None: spent = round(float(reproduced_summary.get("totals", {}).get("cost_usd", 0.0)), 6) - return { - "original_ac6_experiment": { - "usd": ORIGINAL_AC6_USD, - "note": "fixed: full run US$3.18 plus smoke; not changed by any rerun", - }, - "reproduction_validation": { - "usd": spent, - "cap_usd": REPRODUCTION_CAP_USD, - "note": "AC1b rerun only, its own journal and cap; not an experiment cost", - }, - "rule": "separate budgets; never summed or reported as one number", - } + current = None + if reproduction_id is not None: + current = {"reproduction_id": reproduction_id, "usd": spent} + return {"this_run_usd": spent, **ac1b_ledger.budget(ledger, current)} def ac2_section(reproduced_ac2: dict | None, original_ac2: dict) -> dict[str, object]: @@ -426,8 +425,14 @@ def build( steps: list[dict], expected_steps: list[str], context: dict[str, object], + ledger: dict | None = None, ) -> dict[str, object]: - """The whole comparison for one rerun; reads JSON only.""" + """The whole comparison for one rerun; reads JSON only. + + ``ledger`` defaults to this checkout's ``docs/ac1b/attempts.json``. + """ + if ledger is None: + ledger = ac1b_ledger.load() before = read_json(original_root / "analysis" / "summary.json") or {} after = read_json(reproduced_root / "analysis" / "summary.json") rep_jsonl = reproduced_root / "llm" / "haiku-8way.jsonl" @@ -467,7 +472,9 @@ def build( }, "ac2": ac2, "haiku": haiku, - "budget": budget_section(rep_summary), + "budget": budget_section( + rep_summary, ledger, str(context.get("reproduction_id") or "") or None + ), **sections, **context, } @@ -532,11 +539,12 @@ def render_markdown(body: dict) -> str: json.dumps(haiku, indent=2), "```", "", - "## Budget (separate, never summed)", + "## Budget", + "", + f"This run's Haiku spend: US${budget['this_run_usd']} " + f"(cap US${budget['cap_usd_per_attempt']:g} for this reproduction id).", "", - f"- original AC6 experiment: US${budget['original_ac6_experiment']['usd']:.2f} (fixed)", - f"- reproduction-validation: US${budget['reproduction_validation']['usd']} " - f"(cap US${budget['reproduction_validation']['cap_usd']:g})", + *ac1b_ledger.markdown(budget), "", ] lines += pilot_and_number_lines(body) diff --git a/src/tinyrouter/figures.py b/src/tinyrouter/figures.py index 9ccd76c..38f346f 100644 --- a/src/tinyrouter/figures.py +++ b/src/tinyrouter/figures.py @@ -7,6 +7,10 @@ palette, which stays distinguishable under the common colour-vision deficiencies; line styles and markers differ too, so the figures also work in greyscale. Bands and error bars are mean ± sample std over seeds 42, 43, 44. + +Each seed's final router uses the 8-way aggregation it chose on validation. +In risk_coverage a k whose seeds chose differently gets one panel per +aggregation, titled with the seeds that use it; nothing is chosen on test. """ from __future__ import annotations @@ -131,20 +135,66 @@ def learning_curves(summary: dict, plt) -> object: # noqa: ANN001 return fig -def selected_aggregation(summary: dict, group: str) -> str: +AGGREGATIONS = ("argmax", "summed") +SEEDS = (42, 43, 44) + + +def seed_aggregations(summary: dict, group: str) -> tuple[str, ...]: + """The 8-way aggregation each seed's final router chose on validation, in seed order. + + ``summary.json`` stores one string when the three seeds agree and the + per-seed list when they do not (``analysis_run.combine``). + """ chosen = summary["groups"][group]["result"]["selected_aggregation"] - if not isinstance(chosen, str): - raise ValueError(f"{group}: seeds chose different aggregations {chosen}; pick one first") - return chosen + if isinstance(chosen, str): + return (chosen,) * len(SEEDS) + if len(chosen) != len(SEEDS) or not set(chosen) <= set(AGGREGATIONS): + raise ValueError(f"{group}: unexpected selected_aggregation {chosen!r}") + return tuple(chosen) + + +def risk_coverage_panels(summary: dict) -> list[tuple[int, str, tuple[int, ...] | None]]: + """(k, aggregation, seeds whose final router uses it) per panel, None when all three do. + + A k whose seeds agree gets one panel. A k whose seeds chose different + aggregations on validation gets one panel per aggregation that a seed + chose, in the fixed order of ``AGGREGATIONS``; none is picked on test + and none is dropped. + """ + panels: list[tuple[int, str, tuple[int, ...] | None]] = [] + for k in (10, 100): + per_seed = seed_aggregations(summary, f"modernbert/k{k}") + if len(set(per_seed)) == 1: + panels.append((k, per_seed[0], None)) + continue + for aggregation in AGGREGATIONS: + seeds = tuple(s for s, a in zip(SEEDS, per_seed, strict=True) if a == aggregation) + if seeds: + panels.append((k, aggregation, seeds)) + return panels + + +def panel_title(index: int, k: int, aggregation: str, seeds: tuple[int, ...] | None) -> str: + title = f"({'abcd'[index]}) ModernBERT k={k} ({aggregation})" + if seeds is None: + return title + return f"{title}\nfinal router of seed{'s' * (len(seeds) > 1)} {', '.join(map(str, seeds))}" def risk_coverage(summary: dict, curves: dict, plt) -> object: # noqa: ANN001 - fig, axes = plt.subplots(1, 2, figsize=(6.8, 3.0), sharey=True, layout="constrained") + """Test risk-coverage of each signal; a k with mixed per-seed aggregations gets two panels. + + ``curves.json`` keeps each aggregation's curve as mean and std over all + three seeds, not per seed, so a mixed k shows both aggregations side + by side, each titled with the seeds whose final router uses it. + """ + panels = risk_coverage_panels(summary) + fig, axes = plt.subplots( + 1, len(panels), figsize=(3.4 * len(panels), 3.0), sharey=True, layout="constrained" + ) grid = [100 * c for c in curves["coverage_grid"]] - for ax, k in zip(axes, (10, 100), strict=True): - group = f"modernbert/k{k}" - aggregation = selected_aggregation(summary, group) - block = curves["groups"][group]["result"][aggregation] + for index, (ax, (k, aggregation, seeds)) in enumerate(zip(axes, panels, strict=True)): + block = curves["groups"][f"modernbert/k{k}"]["result"][aggregation] for signal, (label, colour, style, width) in SIGNALS.items(): points = block[signal]["risk_coverage"] mean = [100 * p["mean"] for p in points] @@ -166,7 +216,7 @@ def risk_coverage(summary: dict, curves: dict, plt) -> object: # noqa: ANN001 lw=0.9, label=f"{100 * float(TARGET):.0f}% target risk", ) - ax.set_title(f"({'ab'[k == 100]}) ModernBERT k={k} ({aggregation})") + ax.set_title(panel_title(index, k, aggregation, seeds)) ax.set_xlabel("coverage: queries kept by the small model (%)") ax.set_xlim(0, 100) axes[0].set_ylabel("selective risk on test (%)") diff --git a/src/tinyrouter/report.py b/src/tinyrouter/report.py index 1eacb8b..7dd8b75 100644 --- a/src/tinyrouter/report.py +++ b/src/tinyrouter/report.py @@ -18,6 +18,7 @@ from pathlib import Path from tinyrouter.cost import training_runs +from tinyrouter.figures import risk_coverage_panels BEGIN = "" END = "" @@ -436,9 +437,25 @@ def limitations(r: Results) -> list[str]: ] -def figure_lines() -> list[str]: +def mixed_aggregation_note(r: Results) -> str: + """Added to the risk-coverage caption only when a k has one panel per aggregation.""" + mixed = sorted({k for k, _, seeds in risk_coverage_panels(r.summary) if seeds is not None}) + if not mixed: + return "" + ks = " and ".join(f"k={k}" for k in mixed) + return ( + f" At {ks} the three seeds chose different 8-way aggregations on validation, so that k " + "has one panel per aggregation: each panel is the mean over all three seeds under that " + "aggregation, and its title names the seeds whose final router uses it. No aggregation " + "is chosen on test and none is left out." + ) + + +def figure_lines(r: Results) -> list[str]: lines: list[str] = [] for name, alt, caption in FIGURES: + if name == "risk_coverage.png": + caption += mixed_aggregation_note(r) lines += [f"![{alt}](results/figures/{name})", "", f"*{caption}*", ""] return lines[:-1] @@ -451,7 +468,7 @@ def readme_block(r: Results) -> str: ["### Efficiency (AC5)", "", *efficiency_table(r)], ["### Cost (RQ5)", "", *cost_lines(r)], aggregation_line(r), - ["### Figures", "", *figure_lines()], + ["### Figures", "", *figure_lines(r)], limitations(r), ] body = "\n\n".join("\n".join(s) for s in sections) diff --git a/tests/test_ac1b_ledger.py b/tests/test_ac1b_ledger.py new file mode 100644 index 0000000..2ff51a5 --- /dev/null +++ b/tests/test_ac1b_ledger.py @@ -0,0 +1,107 @@ +"""AC1b attempt ledger and budget (docs/PLAN.md 5.1, Drew 2026-09-30).""" + +import copy +import json +from pathlib import Path + +import pytest + +from tinyrouter import ac1b_ledger +from tinyrouter.ac1b_ledger import LedgerError +from tinyrouter.reproduce import Layout, full_steps + +ROOT = Path(__file__).parent.parent +EVIDENCE = ROOT / "docs" / "ac1b" + + +def committed() -> dict: + return ac1b_ledger.load(ROOT / ac1b_ledger.LEDGER) + + +def test_the_committed_ledger_records_both_failed_attempts_at_3992840(): + attempts = committed()["attempts"] + assert [(a["attempt"], a["result"], a["haiku_usd"]) for a in attempts] == [ + (1, "FAIL", 0.0), + (2, "FAIL", 3.178751), + ] + assert {a["reproduction_id"] for a in attempts} == {"3992840ddb3f"} + assert attempts[0]["reason"].startswith("infrastructure") + assert attempts[1]["reason"].startswith("program defect") + for a in attempts: + assert (ROOT / a["evidence"] / "comparison.md").is_file() + assert (ROOT / a["evidence"] / "INCIDENT.md").is_file() + + +def test_the_original_experiment_cost_and_the_cap_cannot_be_edited_in_the_ledger(): + for key, value in (("original_ac6_usd", 6.37), ("cap_usd_per_attempt", 10.0)): + ledger = copy.deepcopy(committed()) + ledger[key] = value + with pytest.raises(LedgerError): + ac1b_ledger.validate(ledger) + + +@pytest.mark.parametrize( + ("field", "value"), + [("attempt", 3), ("result", "PASSED"), ("reason", " "), ("haiku_usd", 5.01), ("haiku_usd", -1)], +) +def test_a_malformed_attempt_is_refused(field, value): + ledger = copy.deepcopy(committed()) + ledger["attempts"][1][field] = value + with pytest.raises(LedgerError): + ac1b_ledger.validate(ledger) + + +def test_the_readme_budget_blocks_match_the_ledger(): + """Rewrite with `python -m tinyrouter.ac1b_ledger` after editing attempts.json.""" + ledger = committed() + for doc in ac1b_ledger.BUDGET_DOCS: + text = (ROOT / doc).read_text(encoding="utf-8") + assert ac1b_ledger.with_block(text, ledger) == text, doc + assert "US$3.19" in text and "US$3.178751" in text + + +def test_the_total_is_only_reproduction_validation_spend(): + budget = ac1b_ledger.budget(committed()) + assert budget["original_ac6_experiment"]["usd"] == 3.19 + assert budget["reproduction_validation_total"]["usd"] == 3.178751 + assert "6.37" not in json.dumps(budget) + + +def test_the_copied_evidence_holds_no_key_or_home_path(): + for path in EVIDENCE.rglob("*.md"): + text = path.read_text(encoding="utf-8") + assert "sk-ant" not in text and "/Users/" not in text and "/private/" not in text, path + + +def test_each_new_commit_gets_its_own_journal_and_five_dollar_cap(): + """The cap is per reproduction id: a new id's Haiku step starts on an empty journal.""" + commands = {} + for run_id in ("3992840ddb3f", "abcdef012345"): + llm = next(s for s in full_steps(Layout(run_id)) if s.name == "llm") + commands[run_id] = llm.command + assert "MAX_USD=5" in llm.command + assert f"RESULTS_ROOT=reproduction/{run_id}/results" in llm.command + assert commands["3992840ddb3f"] != commands["abcdef012345"] + + +def test_spend_under_one_reproduction_id_does_not_count_against_another(tmp_path, monkeypatch): + pytest.importorskip("anthropic") + from llm_fakes import fake_client, fake_split + from tinyrouter import llm, llm_run + + rows = {"validation": 3, "test": 2} + monkeypatch.setattr(llm_run, "EXPECTED_ROWS", rows) + splits = {name: fake_split(name, n + 2) for name, n in rows.items()} + queries = llm_run.build_queries(splits, rows) + cap = 5.5 * llm.cost_usd(262, llm.MAX_TOKENS) + outcomes = [] + for run_id in ("first-attempt", "second-attempt"): + target = llm_run.full_target(tmp_path / "reproduction" / run_id / "results") + client = fake_client() + outcome = llm_run.run( + queries, client, target, max_usd=cap, sleep=lambda _: None, log=lambda _: None + ) + outcomes.append((outcome, len(client.messages.calls))) + for outcome, calls in outcomes: + assert outcome.spent_before == 0.0 + assert calls == 5 and outcome.stopped is None diff --git a/tests/test_comparison.py b/tests/test_comparison.py index eb4fe81..82dc0ce 100644 --- a/tests/test_comparison.py +++ b/tests/test_comparison.py @@ -146,13 +146,61 @@ def test_haiku_checks_fail_on_missing_rows_another_identity_or_spend_above_the_c assert not over["checks_passed"] -def test_the_two_budgets_stay_separate_and_the_original_is_fixed(): - budget = budget_section(summary(4.2)) +LEDGER = { + "original_ac6_usd": 3.19, + "cap_usd_per_attempt": 5.0, + "attempts": [ + { + "attempt": 1, + "commit": "a" * 40, + "reproduction_id": "aaaaaaaaaaaa", + "started_utc": "t0", + "finished_utc": "t1", + "result": "FAIL", + "reason": "infrastructure", + "haiku_usd": 0.0, + }, + { + "attempt": 2, + "commit": "a" * 40, + "reproduction_id": "aaaaaaaaaaaa", + "started_utc": "t2", + "finished_utc": "t3", + "result": "FAIL", + "reason": "program defect", + "haiku_usd": 3.178751, + }, + ], +} + + +def test_the_budget_has_three_parts_and_the_original_is_never_changed(): + budget = budget_section(summary(4.2), LEDGER, "bbbbbbbbbbbb") assert budget["original_ac6_experiment"]["usd"] == ORIGINAL_AC6_USD == 3.19 - assert budget["reproduction_validation"]["usd"] == 4.2 - assert budget["reproduction_validation"]["cap_usd"] == 5.0 - flat = json.dumps(budget) - assert "7.39" not in flat and "total" not in flat + assert budget["this_run_usd"] == 4.2 + attempts = budget["ac1b_attempts"] + assert [a["result"] for a in attempts] == ["FAIL", "FAIL", "PENDING"] + assert attempts[-1]["haiku_usd"] == 4.2 + assert budget["reproduction_validation_total"]["usd"] == round(3.178751 + 4.2, 6) + assert budget["cap_usd_per_attempt"] == 5.0 + # The reproduction-validation spend is never folded into the original experiment cost. + assert json.dumps(budget["original_ac6_experiment"]).count("3.19") == 1 + for spent in (3.19 + 3.178751, 3.19 + 3.178751 + 4.2): + assert f"{spent:.2f}" not in json.dumps(budget) + + +def test_a_rerun_already_in_the_ledger_is_not_counted_twice(): + budget = budget_section(summary(3.178751), LEDGER, "aaaaaaaaaaaa") + assert len(budget["ac1b_attempts"]) == 2 + assert budget["reproduction_validation_total"]["usd"] == 3.178751 + + +def test_the_markdown_budget_shows_all_three_parts(roots): + body = run_build(roots) + text = render_markdown(body) + assert "Original experiment (AC6, fixed): US$3.19" in text + assert "| attempt | reproduction id | result | reason | Haiku spend |" in text + assert "Reproduction-validation total (all AC1b attempts)" in text @pytest.fixture diff --git a/tests/test_figures.py b/tests/test_figures.py index c0b70bd..c680791 100644 --- a/tests/test_figures.py +++ b/tests/test_figures.py @@ -1,3 +1,6 @@ +import copy +import json +import shutil from pathlib import Path import pytest @@ -6,6 +9,7 @@ from tinyrouter import figures # noqa: E402 +ORIGINAL = json.loads(Path("results/analysis/summary.json").read_text(encoding="utf-8")) EXPECTED = {"learning_curves.png", "risk_coverage.png", "routers.png", "threshold_transfer.png"} @@ -18,10 +22,51 @@ def test_figures_are_drawn_from_json_and_are_byte_identical_on_rerun(tmp_path): assert b"Matplotlib version" not in a.read_bytes() -def test_risk_coverage_refuses_a_point_whose_seeds_chose_different_aggregations(): - summary = {"groups": {"x": {"result": {"selected_aggregation": ["argmax", "summed"]}}}} - with pytest.raises(ValueError, match="different aggregations"): - figures.selected_aggregation(summary, "x") +def mixed(summary: dict, group: str, per_seed: list[str]) -> dict: + """``summary`` with ``group``'s seeds choosing ``per_seed``, as analysis writes it.""" + out = copy.deepcopy(summary) + out["groups"][group]["result"]["selected_aggregation"] = per_seed + return out + + +def test_mixed_per_seed_aggregations_get_one_panel_each_titled_with_their_seeds(): + """AC1b attempt 2: ModernBERT k=100 chose argmax, summed, summed; figures crashed.""" + summary = mixed(ORIGINAL, "modernbert/k100", ["argmax", "summed", "summed"]) + assert figures.risk_coverage_panels(summary) == [ + (10, "summed", None), + (100, "argmax", (42,)), + (100, "summed", (43, 44)), + ] + plt = figures.pyplot() + fig = figures.risk_coverage(summary, figures.load(Path("results"), "analysis/curves.json"), plt) + titles = [ax.get_title() for ax in fig.axes] + assert titles == [ + "(a) ModernBERT k=10 (summed)", + "(b) ModernBERT k=100 (argmax)\nfinal router of seed 42", + "(c) ModernBERT k=100 (summed)\nfinal router of seeds 43, 44", + ] + plt.close(fig) + + +def test_unanimous_seeds_keep_the_original_two_panels(): + assert figures.risk_coverage_panels(ORIGINAL) == [(10, "summed", None), (100, "argmax", None)] + + +def test_figures_build_on_a_summary_with_mixed_aggregations(tmp_path): + root = tmp_path / "results" + (root / "analysis").mkdir(parents=True) + summary = mixed(ORIGINAL, "modernbert/k100", ["argmax", "summed", "summed"]) + summary = mixed(summary, "modernbert/k10", ["summed", "argmax", "summed"]) + (root / "analysis" / "summary.json").write_text(json.dumps(summary)) + shutil.copy(Path("results/analysis/curves.json"), root / "analysis" / "curves.json") + written = figures.build(root, root / "figures") + assert {p.name for p in written} == EXPECTED + + +def test_an_unknown_aggregation_is_refused(): + summary = mixed(ORIGINAL, "modernbert/k100", ["argmax", "median", "summed"]) + with pytest.raises(ValueError, match="unexpected selected_aggregation"): + figures.seed_aggregations(summary, "modernbert/k100") def test_threshold_transfer_skips_points_without_a_feasible_threshold_on_every_seed(): diff --git a/tests/test_report.py b/tests/test_report.py index b954b23..42c00f6 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,4 +1,5 @@ import copy +import dataclasses import json import shutil import subprocess @@ -221,3 +222,21 @@ def test_each_first_screen_number_comes_from_its_own_json_field(name): def test_k10_hybrid_counts_are_the_reviewed_absolute_numbers(): assert counts((*K10_HYBRID, "llm_call_rate")) == "1,235, 1,157, 1,548 of 5,500" assert counts((*K100_HYBRID, "llm_call_rate")) == "170, 9, 31 of 5,500" + + +def test_report_with_mixed_per_seed_aggregations_names_them_and_explains_the_panels(): + """AC1b attempt 2: ModernBERT k=100 seeds chose argmax, summed, summed.""" + r = report.Results.load(Path("results")) + summary = json.loads(json.dumps(r.summary)) + summary["groups"]["modernbert/k100"]["result"]["selected_aggregation"] = [ + "argmax", + "summed", + "summed", + ] + mixed = dataclasses.replace(r, summary=summary) + block = report.readme_block(mixed) + assert "ModernBERT-base: k=1 summed" in block + assert "k=100 per seed argmax/summed/summed" in block + assert "At k=100 the three seeds chose different 8-way aggregations on validation" in block + assert "No aggregation is chosen on test" in block + assert "chose different 8-way aggregations" not in report.readme_block(r) From a375bcee9e454cbea4c2e5dba0b22dd8007ad8d0 Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Wed, 30 Sep 2026 09:53:03 +0800 Subject: [PATCH 2/3] List per-seed aggregation choices in the AC1b comparison, not judged Every group with a final router gets a row with the original and rerun 8-way aggregation each seed chose on validation, whether they match and which seeds differ. Status is fixed at LISTED, NOT JUDGED: it adds no REVIEW REQUIRED and never changes the verdict. It explains router differences such as ModernBERT k=100 small-only, where seeds 43 and 44 switched to summed in AC1b attempt 2. Scope fixed in PLAN 5.1 before attempt 3. --- DEVLOG.md | 1 + docs/PLAN.md | 1 + src/tinyrouter/comparison.py | 72 ++++++++++++++++++++++++++++++++++++ tests/test_comparison.py | 62 +++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+) diff --git a/DEVLOG.md b/DEVLOG.md index 6b1cb1d..58daeff 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -12,6 +12,7 @@ - **AC1b 第 2 次**（續跑，2026-09-30 01:13 到 01:31 UTC）：Haiku 8,600/8,600、US$3.178751、0 parse failed、20 筆預測與原 run 不同；analysis、latency、cost 完成；`make figures` 失敗：ModernBERT k=100 三個 seed 在 validation 選了 argmax、summed、summed，`figures.selected_aggregation` 直接丟 ValueError。原始 run 三個 seed 都選 argmax，所以這條路徑從沒跑過。判 FAIL（程式缺陷）。 - **修正**：找過所有讀聚合方式的地方。`analysis` 本來就讓每個 seed 用自己 validation 選的聚合（`final`、`diagnostics` 逐 seed 算好再合併）；`report` 的聚合說明已會列出逐 seed 選擇；`cost`、`comparison`、學習曲線、router 圖、門檻圖都讀 `final` 或 `diagnostics`，不受影響。只有 risk-coverage 圖假設三個 seed 同一種。`curves.json` 對每種聚合只存三 seed 的 mean 與 std，沒有逐 seed 曲線，而補存會改動原始 `curves.json` 的位元組，所以改成：seed 一致的 k 維持一個面板；不一致的 k 每種被選到的聚合各一個面板（固定 argmax、summed 順序），標題寫出哪些 seed 的 final router 用它；README 圖說在這種情況自動加一句說明。沒有任何選擇用到 test，也沒有固定成某一種。 - **帳本**：`docs/ac1b/attempts.json` 記錄每次嘗試（commit、id、UTC 起訖、結果與原因、實際 Haiku 支出），`docs/ac1b/attempt-/` 放 comparison 與事故說明。comparison 與 README 的預算改成三部分：原始實驗 US$3.19（固定）、每次嘗試支出與原因、reproduction-validation 累計。PLAN §5.1 預算規則補一行：每次嘗試各自獨立 US$5 上限（Drew，2026-09-30）。 +- **聚合選擇只列不判**（第 3 次嘗試實跑前寫定，PLAN §5.1 實跑前補充追加一行）：comparison 新增一節，對每個有 final router 的組（25 組）列出原始與重跑每個 seed 在 validation 選的 8 類聚合與是否相同，狀態固定 `LISTED, NOT JUDGED`，不計入 REVIEW、不影響判定。用第 2 次重跑輸出實測：7 組不同，其中 ModernBERT k=100 原始 argmax/argmax/argmax、重跑 argmax/summed/summed（seeds 43、44 改選），可解釋 small-only 91.87 → 91.42 的 REVIEW；REVIEW 計數仍是 192。 - 下一步依 Drew 決定：合併後在新 commit 完整重跑 AC1b（新的一次嘗試，新 id、新 journal、獨立 US$5）；不拿第 2 次的輸出配新 commit 的圖。 ### 核心發現 / 數據 diff --git a/docs/PLAN.md b/docs/PLAN.md index 8443460..eeb7c30 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -127,6 +127,7 @@ TinyRouter 問三件事：**小模型要多少標註資料才夠？它知不知 - 判定範圍（偏離原始 mean ± std 標 `REVIEW REQUIRED`）固定為：README 首屏數字；README router 表（LLM-only，以及 ModernBERT k=10 與 k=100 的 small-only、hybrid 目標 2% 與 5%、oracle，各報 8 類準確率、OOS recall、高信心 OOS 誤派、LLM 呼叫率）；學習曲線（兩個 encoder、每個 k 的 8 類準確率與 OOS recall）；`ablation_comparison`（OOS 250 對 0 的 AUROC、AUPRC 與 router 行為）；所有門檻診斷統計。 - **只列不判**：延遲（`results/efficiency/*`）、k=100 的訓練時間與峰值記憶體、成本模型（`results/cost/cost.json`）隨機器變動。comparison 列出原始值、重跑值與差異，狀態固定為 `LISTED, NOT JUDGED`，不計入 `REVIEW REQUIRED`，也不影響判定。 +- **聚合選擇只列不判**（2026-09-30，第 3 次嘗試實跑前寫定）：每個有 final router 的組，comparison 列出原始與重跑每個 seed 在 validation 選的 8 類聚合方式與是否相同，狀態固定為 `LISTED, NOT JUDGED`，不計入 `REVIEW REQUIRED`，也不影響判定。 - 續跑只限同一身分：每一步記錄 HEAD、`uv.lock` 與 `configs/` 的 SHA-256、Python、torch 與 transformers 版本；同一個 reproduction 目錄若由不同身分開始，拒絕續跑，必須換新 id，所以一份 comparison 不會混到兩個 commit 的輸出（第 1 條的落實方式）。 **預算規則**：原始 AC6 實驗的 Haiku 花費固定是 US$3.19（完整 run US$3.18 加 smoke），這個數字不再變動。AC1b 的 Haiku 是另一次、明確標記為 reproduction-validation 的執行，有獨立的 US$5 上限，花費寫在 reproduction 目錄自己的紀錄裡；兩者不得混稱為原始實驗成本，也不相加。 diff --git a/src/tinyrouter/comparison.py b/src/tinyrouter/comparison.py index 666ac4d..323e739 100644 --- a/src/tinyrouter/comparison.py +++ b/src/tinyrouter/comparison.py @@ -224,6 +224,55 @@ def k100_training(root: Path) -> dict | None: return out +SEEDS = (42, 43, 44) + + +def seed_aggregations(summary: dict, group: str) -> list[str] | None: + """Each seed's validation-chosen 8-way aggregation, in seed order; None when absent. + + ``summary.json`` stores one string when the seeds agree, the per-seed list otherwise. + """ + chosen = at(summary, ("groups", group, "result", "selected_aggregation")) + if isinstance(chosen, str): + return [chosen] * len(SEEDS) + return list(chosen) if isinstance(chosen, list) else None + + +def aggregation_rows(before: dict, after: dict) -> list[dict]: + """Per group with a final router: original and rerun aggregation per seed. Never judged. + + Fixed before AC1b attempt 3 (docs/PLAN.md 5.1). It explains router + differences (a seed that switched aggregation routes differently) and + is ``LISTED, NOT JUDGED``: it adds no REVIEW REQUIRED and never changes + the verdict. + """ + groups = set(before.get("groups", {})) | set(after.get("groups", {})) + rows = [] + for group in sorted(g for g in groups if has_final(before, g) or has_final(after, g)): + original, rerun = seed_aggregations(before, group), seed_aggregations(after, group) + differing = ( + list(SEEDS) + if original is None or rerun is None + else [s for s, a, b in zip(SEEDS, original, rerun, strict=True) if a != b] + ) + rows.append( + { + "group": group, + "seeds": list(SEEDS), + "original": original, + "reproduced": rerun, + "same": not differing, + "differing_seeds": differing, + "status": LISTED, + } + ) + return rows + + +def has_final(summary: dict, group: str) -> bool: + return at(summary, ("groups", group, "result", "final")) is not None + + def machine_dependent_rows(original_root: Path, reproduced_root: Path) -> list[dict]: """Listed, not judged: latency, k=100 training time and peak memory, the cost model.""" rows = [] @@ -457,6 +506,7 @@ def build( "ablation": None if after is None else ablation_rows(before, after), "threshold_diagnostics": None if after is None else diagnostic_rows(before, after), "machine_dependent": machine_dependent_rows(original_root, reproduced_root), + "aggregations": None if after is None else aggregation_rows(before, after), } flow_ok = flow_passed(steps, expected_steps) return { @@ -580,6 +630,7 @@ def pilot_and_number_lines(body: dict) -> list[str]: "Latency, k=100 training time and peak memory, and the cost model vary with the " "machine; they are listed with their differences and never marked REVIEW REQUIRED.", "", + *aggregation_lines(body["aggregations"]), "## Learning curves (test, small model alone)", "", *stat_table(body["learning_curves"]), @@ -589,3 +640,24 @@ def pilot_and_number_lines(body: dict) -> list[str]: *stat_table(diag, only_review=True), "", ] + + +def aggregation_lines(rows: list[dict]) -> list[str]: + differ = sum(not row["same"] for row in rows) + lines = [ + f"## 8-way aggregation chosen on validation: {len(rows)} groups {LISTED} ({differ} differ)", + "", + "Each seed's final router uses the aggregation it chose on validation; a seed that " + "switched routes differently, which can explain a REVIEW REQUIRED row above.", + "", + "| group | original (42/43/44) | rerun (42/43/44) | same | differing seeds | status |", + "|---|---|---|---|---|---|", + ] + for row in rows: + before = "/".join(row["original"]) if row["original"] else "n/a" + after = "/".join(row["reproduced"]) if row["reproduced"] else "n/a" + seeds = ", ".join(map(str, row["differing_seeds"])) or "none" + lines.append( + f"| {row['group']} | {before} | {after} | {row['same']} | {seeds} | {row['status']} |" + ) + return [*lines, ""] diff --git a/tests/test_comparison.py b/tests/test_comparison.py index 82dc0ce..4ed9555 100644 --- a/tests/test_comparison.py +++ b/tests/test_comparison.py @@ -369,3 +369,65 @@ def test_the_flow_records_the_commit_each_step_ran_at(roots): body = run_build(roots, steps) assert body["flow"]["commits"] == ["a" * 40, "b" * 40] assert "| ac2 | PASS | aaaaaaaaaaaa |" in render_markdown(body) + + +def switch_aggregation(rerun: Path, group: str, per_seed: list[str]) -> None: + path = rerun / "analysis" / "summary.json" + body = json.loads(path.read_text()) + body["groups"][group]["result"]["selected_aggregation"] = per_seed + path.write_text(json.dumps(body)) + + +def test_aggregation_choices_are_listed_per_seed_for_every_group_with_a_final_router(roots): + body = run_build(roots) + rows = {row["group"]: row for row in body["aggregations"]} + summary = json.loads((RESULTS / "analysis" / "summary.json").read_text()) + assert set(rows) == set(summary["groups"]) + for model in ("bert", "modernbert"): + for k in (1, 5, 10, 25, 50, 100): + assert f"{model}/k{k}" in rows + assert "modernbert-oos0/k100" in rows + k100 = rows["modernbert/k100"] + assert k100["original"] == k100["reproduced"] == ["argmax"] * 3 + assert k100["same"] is True and k100["differing_seeds"] == [] + assert {row["status"] for row in rows.values()} == {LISTED} + + +def test_a_switched_aggregation_is_listed_but_neither_reviewed_nor_failed(roots): + """AC1b attempt 2: ModernBERT k=100 seeds 43 and 44 switched from argmax to summed.""" + original, rerun, _ = roots + switch_aggregation(rerun, "modernbert/k100", ["argmax", "summed", "summed"]) + body = run_build(roots) + row = next(r for r in body["aggregations"] if r["group"] == "modernbert/k100") + assert row["original"] == ["argmax", "argmax", "argmax"] + assert row["reproduced"] == ["argmax", "summed", "summed"] + assert row["same"] is False and row["differing_seeds"] == [43, 44] + assert row["status"] == LISTED + assert body["verdict"] == PASS and body["review_required"] == 0 + text = render_markdown(body) + assert ( + "| modernbert/k100 | argmax/argmax/argmax | argmax/summed/summed | False | 43, 44 |" in text + ) + assert "(1 differ)" in text + + +def test_a_group_missing_from_the_rerun_lists_every_seed_as_differing(): + before = {"groups": {"g": {"result": {"final": {}, "selected_aggregation": "summed"}}}} + rows = comparison.aggregation_rows(before, {"groups": {}}) + assert rows == [ + { + "group": "g", + "seeds": [42, 43, 44], + "original": ["summed"] * 3, + "reproduced": None, + "same": False, + "differing_seeds": [42, 43, 44], + "status": LISTED, + } + ] + + +def test_the_aggregation_section_is_never_part_of_the_review_count(): + """PLAN 5.1 (2026-09-30): aggregation choices are listed, not judged, whatever a row says.""" + assert "aggregations" not in comparison.JUDGED_SECTIONS + assert comparison.review_count({"aggregations": [{"status": REVIEW}] * 3}) == 0 From cbc699ea5ac090798c5280fd2aba0c11bd2db828 Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:10:44 +0800 Subject: [PATCH 3/3] Record attempt 1 as infrastructure-interrupted and tighten ledger results Per Drew's decision of 2026-09-29, attempt 1 (HTTP 503 during an external authentication incident) is INFRASTRUCTURE INTERRUPTED, not an AC1b failure; its comparison verdict FAIL is kept as comparison_verdict and the evidence files are unchanged. The ledger now requires failure_kind from a fixed list, allows INTERRUPTED only for infrastructure with incident evidence and the classifying decision, and requires INFRASTRUCTURE BLOCKED when the same infrastructure error recurs on the next attempt. AC1b is completed only by a PASS; the budget blocks say so. Mixed-aggregation risk-coverage titles now read "mean of 3 seeds; final router of seed ...". Docs note the attempt 1 start time source, the relayed decision wording and when the aggregation listing was added. --- DEVLOG.md | 3 +- README.md | 6 +- docs/PLAN.md | 2 +- docs/ac1b/README.md | 13 +++- docs/ac1b/attempt-2/INCIDENT.md | 2 +- docs/ac1b/attempts.json | 21 ++++-- src/tinyrouter/ac1b_ledger.py | 102 ++++++++++++++++++++++++---- src/tinyrouter/figures.py | 3 +- tests/test_ac1b_ledger.py | 115 +++++++++++++++++++++++++++++--- tests/test_figures.py | 4 +- 10 files changed, 234 insertions(+), 37 deletions(-) diff --git a/DEVLOG.md b/DEVLOG.md index 58daeff..e904f37 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -8,11 +8,12 @@ ### 本次工作 / 執行摘要 - 狀態不變：RQ1 到 RQ5 完成；**Tier 1 驗收未完成，AC1b 尚未通過**。 -- **AC1b 第 1 次**（`3992840ddb3f`，2026-09-29 07:40 到 14:24 UTC）：訓練、AC2、pilot 全過；Haiku 前的 `count_tokens` 在 Anthropic 認證事故期間連 5 次 HTTP 503，花費 US$0。依凍結標準判 FAIL；Drew 判定為基礎設施事故，同一 id 與身分續跑。 +- **AC1b 第 1 次**（`3992840ddb3f`，2026-09-29 07:40 到 14:24 UTC）：訓練、AC2、pilot 全過；Haiku 前的 `count_tokens` 在 Anthropic 認證事故期間連 5 次 HTTP 503，花費 US$0。comparison 機械判定 FAIL；Drew 判定為基礎設施事故、不視為 AC1b 失敗，帳本記為 `INFRASTRUCTURE INTERRUPTED`，同一 id 與身分續跑。 - **AC1b 第 2 次**（續跑，2026-09-30 01:13 到 01:31 UTC）：Haiku 8,600/8,600、US$3.178751、0 parse failed、20 筆預測與原 run 不同；analysis、latency、cost 完成；`make figures` 失敗：ModernBERT k=100 三個 seed 在 validation 選了 argmax、summed、summed，`figures.selected_aggregation` 直接丟 ValueError。原始 run 三個 seed 都選 argmax，所以這條路徑從沒跑過。判 FAIL（程式缺陷）。 - **修正**：找過所有讀聚合方式的地方。`analysis` 本來就讓每個 seed 用自己 validation 選的聚合（`final`、`diagnostics` 逐 seed 算好再合併）；`report` 的聚合說明已會列出逐 seed 選擇；`cost`、`comparison`、學習曲線、router 圖、門檻圖都讀 `final` 或 `diagnostics`，不受影響。只有 risk-coverage 圖假設三個 seed 同一種。`curves.json` 對每種聚合只存三 seed 的 mean 與 std，沒有逐 seed 曲線，而補存會改動原始 `curves.json` 的位元組，所以改成：seed 一致的 k 維持一個面板；不一致的 k 每種被選到的聚合各一個面板（固定 argmax、summed 順序），標題寫出哪些 seed 的 final router 用它；README 圖說在這種情況自動加一句說明。沒有任何選擇用到 test，也沒有固定成某一種。 - **帳本**：`docs/ac1b/attempts.json` 記錄每次嘗試（commit、id、UTC 起訖、結果與原因、實際 Haiku 支出），`docs/ac1b/attempt-/` 放 comparison 與事故說明。comparison 與 README 的預算改成三部分：原始實驗 US$3.19（固定）、每次嘗試支出與原因、reproduction-validation 累計。PLAN §5.1 預算規則補一行：每次嘗試各自獨立 US$5 上限（Drew，2026-09-30）。 - **聚合選擇只列不判**（第 3 次嘗試實跑前寫定，PLAN §5.1 實跑前補充追加一行）：comparison 新增一節，對每個有 final router 的組（25 組）列出原始與重跑每個 seed 在 validation 選的 8 類聚合與是否相同，狀態固定 `LISTED, NOT JUDGED`，不計入 REVIEW、不影響判定。用第 2 次重跑輸出實測：7 組不同，其中 ModernBERT k=100 原始 argmax/argmax/argmax、重跑 argmax/summed/summed（seeds 43、44 改選），可解釋 small-only 91.87 → 91.42 的 REVIEW；REVIEW 計數仍是 192。 +- **帳本結果分類（PR #20 審查後）**：依 Drew 2026-09-29 的決定（由協調者轉達），第 1 次記為 `INFRASTRUCTURE INTERRUPTED`（`failure_kind: infrastructure`），附外部事故證據（status.claude.com 事故、2026-09-29 14:21 UTC、503 credential validation failed）與決定原文；comparison 的機械判定 FAIL 另存為 `comparison_verdict`，證據檔不改寫。收緊條件：`failure_kind` 只能是 infrastructure、program defect、ac2；INTERRUPTED 只允許 infrastructure 且需證據與決定；同一 infrastructure 錯誤類型連續第二次必須是 `INFRASTRUCTURE BLOCKED`（AC1b 暫停）；程式缺陷只能是 FAIL。AC1b 最終判定仍只有 PASS 或 FAIL：INTERRUPTED 不算失敗也不算通過，只有某次嘗試 PASS 才完成驗收。risk-coverage 混合面板標題改為「mean of 3 seeds; final router of seed …」。 - 下一步依 Drew 決定：合併後在新 commit 完整重跑 AC1b（新的一次嘗試，新 id、新 journal、獨立 US$5）；不拿第 2 次的輸出配新 commit 的圖。 ### 核心發現 / 數據 diff --git a/README.md b/README.md index cde1611..8365087 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,7 @@ Training on Apple MPS is not bit-for-bit deterministic, so a rerun is not expect A clean clone is not a clean cache: the base models and the CLINC150 files come from the shared Hugging Face cache (`~/.cache/huggingface` by default). That does not affect correctness, since every model is pinned to a revision and every data file is checked against its SHA-256, but to download everything afresh point the cache at an empty directory, for example `HF_HOME=/tmp/tinyrouter-hf make reproduce`. -**Status: AC1b has not passed yet.** It has been attempted twice at `3992840ddb3f` and both attempts failed (an external authentication incident, then a defect in the figure code when seeds chose different 8-way aggregations); every attempt is recorded in [`docs/ac1b/`](docs/ac1b/README.md). Until it has passed, the status is RQ1 to RQ5 complete, Tier 1 acceptance not complete. +**Status: AC1b has not passed yet.** It has been attempted twice at `3992840ddb3f`: attempt 1 was interrupted by an external authentication incident (recorded as infrastructure-interrupted, neither a failure nor a pass), and attempt 2 failed on a defect in the figure code when seeds chose different 8-way aggregations; every attempt is recorded in [`docs/ac1b/`](docs/ac1b/README.md). Until it has passed, the status is RQ1 to RQ5 complete, Tier 1 acceptance not complete. AC1b budget. Every attempt has its own US$5 Haiku cap, enforced per reproduction id: @@ -247,12 +247,14 @@ AC1b budget. Every attempt has its own US$5 Haiku cap, enforced per reproduction | attempt | reproduction id | result | reason | Haiku spend | |---|---|---|---|---| -| 1 | `3992840ddb3f` | FAIL | infrastructure: Haiku count_tokens returned HTTP 503 credential validation failed 5 times during an Anthropic authentication incident; no classification request sent | US$0.000000 | +| 1 | `3992840ddb3f` | INFRASTRUCTURE INTERRUPTED | infrastructure: Haiku count_tokens returned HTTP 503 credential validation failed 5 times during an Anthropic authentication incident; no classification request sent | US$0.000000 | | 2 | `3992840ddb3f` | FAIL | program defect: make figures stopped because seeds chose different 8-way aggregations for modernbert/k100 (argmax, summed, summed); Haiku 8600/8600, 0 parse failures, 20 predictions differ | US$3.178751 | - Reproduction-validation total (all AC1b attempts): US$3.178751 Reproduction-validation spend is never reported as original experiment cost and does not change the AC6 conclusion (Haiku spend at most US$5). + +INFRASTRUCTURE INTERRUPTED is neither an AC1b failure nor a pass; AC1b acceptance is completed only by an attempt whose result is PASS. ## Reproducibility notes diff --git a/docs/PLAN.md b/docs/PLAN.md index eeb7c30..712837e 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -127,7 +127,7 @@ TinyRouter 問三件事：**小模型要多少標註資料才夠？它知不知 - 判定範圍（偏離原始 mean ± std 標 `REVIEW REQUIRED`）固定為：README 首屏數字；README router 表（LLM-only，以及 ModernBERT k=10 與 k=100 的 small-only、hybrid 目標 2% 與 5%、oracle，各報 8 類準確率、OOS recall、高信心 OOS 誤派、LLM 呼叫率）；學習曲線（兩個 encoder、每個 k 的 8 類準確率與 OOS recall）；`ablation_comparison`（OOS 250 對 0 的 AUROC、AUPRC 與 router 行為）；所有門檻診斷統計。 - **只列不判**：延遲（`results/efficiency/*`）、k=100 的訓練時間與峰值記憶體、成本模型（`results/cost/cost.json`）隨機器變動。comparison 列出原始值、重跑值與差異，狀態固定為 `LISTED, NOT JUDGED`，不計入 `REVIEW REQUIRED`，也不影響判定。 -- **聚合選擇只列不判**（2026-09-30，第 3 次嘗試實跑前寫定）：每個有 final router 的組，comparison 列出原始與重跑每個 seed 在 validation 選的 8 類聚合方式與是否相同，狀態固定為 `LISTED, NOT JUDGED`，不計入 `REVIEW REQUIRED`，也不影響判定。 +- **聚合選擇只列不判**（2026-09-30，第 2 次嘗試觀察到差異後追加，第 3 次嘗試實跑前寫定）：每個有 final router 的組，comparison 列出原始與重跑每個 seed 在 validation 選的 8 類聚合方式與是否相同，狀態固定為 `LISTED, NOT JUDGED`，不計入 `REVIEW REQUIRED`，也不影響判定。 - 續跑只限同一身分：每一步記錄 HEAD、`uv.lock` 與 `configs/` 的 SHA-256、Python、torch 與 transformers 版本；同一個 reproduction 目錄若由不同身分開始，拒絕續跑，必須換新 id，所以一份 comparison 不會混到兩個 commit 的輸出（第 1 條的落實方式）。 **預算規則**：原始 AC6 實驗的 Haiku 花費固定是 US$3.19（完整 run US$3.18 加 smoke），這個數字不再變動。AC1b 的 Haiku 是另一次、明確標記為 reproduction-validation 的執行，有獨立的 US$5 上限，花費寫在 reproduction 目錄自己的紀錄裡；兩者不得混稱為原始實驗成本，也不相加。 diff --git a/docs/ac1b/README.md b/docs/ac1b/README.md index d5042f0..1b7d46e 100644 --- a/docs/ac1b/README.md +++ b/docs/ac1b/README.md @@ -3,7 +3,14 @@ AC1b (docs/PLAN.md section 5.1) is the whole study rerun from a clean clone with `make setup && make reproduce`. This directory records every attempt, whether it passed or not, so that a failed attempt and its cost are never hidden by a later one. - `attempts.json`: the ledger, one entry per attempt: commit, reproduction id, start and end (UTC), result and reason, actual Haiku spend. `python -m tinyrouter.ac1b_ledger` checks it and rewrites the budget block below and in the top-level README; `tests/test_ac1b_ledger.py` fails when either is stale. -- `attempt-/`: the attempt's `comparison.md` as written by `make reproduce`, and `INCIDENT.md`, a note on why it stopped. +- `attempt-/`: the attempt's `comparison.md` as written by `make reproduce`, and `INCIDENT.md`, a note on why it stopped. Evidence files are kept as written: attempt 1's `INCIDENT.md` says it started "about 07:45" UTC; the ledger uses 07:40:02 UTC, the start of the first step in `steps.json`, which is authoritative. + +Results (validated by `tinyrouter.ac1b_ledger`): + +- `PASS` or `FAIL`: the comparison's verdict under the frozen criteria. `comparison_verdict` always records that mechanical verdict, and the evidence is never rewritten. +- `INFRASTRUCTURE INTERRUPTED`: allowed only with `failure_kind: infrastructure`, external incident evidence (incident, time, source, error type) and the decision that classified it. INFRASTRUCTURE INTERRUPTED is neither an AC1b failure nor a pass; AC1b acceptance is completed only by an attempt whose result is PASS. Attempt 1 is recorded this way under Drew's decision of 2026-09-29, while its comparison verdict stays FAIL. +- `INFRASTRUCTURE BLOCKED`: required when the same infrastructure error type recurs on the next attempt. AC1b is then paused, not resumed again. +- `failure_kind` is one of `infrastructure`, `program defect`, `ac2`; a program defect can only be `FAIL`. Budget rules (Drew, 2026-09-29 and 2026-09-30): @@ -17,10 +24,12 @@ Budget rules (Drew, 2026-09-29 and 2026-09-30): | attempt | reproduction id | result | reason | Haiku spend | |---|---|---|---|---| -| 1 | `3992840ddb3f` | FAIL | infrastructure: Haiku count_tokens returned HTTP 503 credential validation failed 5 times during an Anthropic authentication incident; no classification request sent | US$0.000000 | +| 1 | `3992840ddb3f` | INFRASTRUCTURE INTERRUPTED | infrastructure: Haiku count_tokens returned HTTP 503 credential validation failed 5 times during an Anthropic authentication incident; no classification request sent | US$0.000000 | | 2 | `3992840ddb3f` | FAIL | program defect: make figures stopped because seeds chose different 8-way aggregations for modernbert/k100 (argmax, summed, summed); Haiku 8600/8600, 0 parse failures, 20 predictions differ | US$3.178751 | - Reproduction-validation total (all AC1b attempts): US$3.178751 Reproduction-validation spend is never reported as original experiment cost and does not change the AC6 conclusion (Haiku spend at most US$5). + +INFRASTRUCTURE INTERRUPTED is neither an AC1b failure nor a pass; AC1b acceptance is completed only by an attempt whose result is PASS. diff --git a/docs/ac1b/attempt-2/INCIDENT.md b/docs/ac1b/attempt-2/INCIDENT.md index dc77ab1..7a6b7b2 100644 --- a/docs/ac1b/attempt-2/INCIDENT.md +++ b/docs/ac1b/attempt-2/INCIDENT.md @@ -8,4 +8,4 @@ Written 2026-09-30 from the attempt's `run.log`, `steps.json` and `comparison.md - Haiku spend: US$3.178751, under the US$5 cap of this reproduction id (attempt 1 spent US$0 on the same journal). - Failed step: `figures`. `figures.selected_aggregation` raised `ValueError: modernbert/k100: seeds chose different aggregations ['argmax', 'summed', 'summed']; pick one first`. In the original run all three ModernBERT k=100 seeds chose argmax on validation, so this path had never run. - Verdict under the frozen criteria (docs/PLAN.md 5.1): FAIL (the flow did not complete). This is a defect in the program, not an infrastructure failure. -- Drew's decision (2026-09-30): fix the defect, then run AC1b again in full at the new commit as a new attempt with its own US$5 cap. The outputs of this attempt are not combined with figures or a report from another commit. +- Drew's decision, relayed by the coordinator (2026-09-30): fix the defect, then run AC1b again in full at the new commit as a new attempt with its own US$5 cap. The outputs of this attempt are not combined with figures or a report from another commit. diff --git a/docs/ac1b/attempts.json b/docs/ac1b/attempts.json index c328b43..cb27c97 100644 --- a/docs/ac1b/attempts.json +++ b/docs/ac1b/attempts.json @@ -11,11 +11,22 @@ "started_utc": "2026-09-29T07:40:02Z", "finished_utc": "2026-09-29T14:24:22Z", "time_source": "steps.json: start of the first step, end of the failed llm step", - "result": "FAIL", + "result": "INFRASTRUCTURE INTERRUPTED", "failure_kind": "infrastructure", + "comparison_verdict": "FAIL", "reason": "infrastructure: Haiku count_tokens returned HTTP 503 credential validation failed 5 times during an Anthropic authentication incident; no classification request sent", + "evidence": { + "incident": "Elevated errors on claude.ai, Claude Code and Claude Cowork", + "time_utc": "2026-09-29 14:21 UTC, status Investigating", + "source": "status.claude.com", + "error_type": "HTTP 503 credential validation failed (messages.count_tokens)" + }, + "decision": { + "text": "503 是外部服務／認證可用性事故，不代表訓練或實驗協定失效……若同一認證問題反覆發生，才暫停並把 AC1b 標為 infrastructure-blocked；目前不應視為 AC1b 失敗。", + "source": "Drew's decision, relayed by the coordinator (2026-09-29)" + }, "haiku_usd": 0.0, - "evidence": "docs/ac1b/attempt-1/" + "evidence_dir": "docs/ac1b/attempt-1/" }, { "attempt": 2, @@ -26,9 +37,11 @@ "time_source": "steps.json: start of the resumed llm step, end of the failed figures step", "result": "FAIL", "failure_kind": "program defect", + "comparison_verdict": "FAIL", "reason": "program defect: make figures stopped because seeds chose different 8-way aggregations for modernbert/k100 (argmax, summed, summed); Haiku 8600/8600, 0 parse failures, 20 predictions differ", "haiku_usd": 3.178751, - "evidence": "docs/ac1b/attempt-2/" + "evidence_dir": "docs/ac1b/attempt-2/" } - ] + ], + "result_rule": "INFRASTRUCTURE INTERRUPTED is neither an AC1b failure nor a pass; AC1b acceptance is completed only by an attempt whose result is PASS. comparison_verdict keeps the comparison's own mechanical verdict unchanged." } diff --git a/src/tinyrouter/ac1b_ledger.py b/src/tinyrouter/ac1b_ledger.py index 58abdf4..1829a6a 100644 --- a/src/tinyrouter/ac1b_ledger.py +++ b/src/tinyrouter/ac1b_ledger.py @@ -22,7 +22,16 @@ REPO = Path(__file__).resolve().parents[2] ORIGINAL_AC6_USD = 3.19 CAP_USD_PER_ATTEMPT = 5.0 -RESULTS = ("PASS", "FAIL", "INFRASTRUCTURE INTERRUPTED") +PASS = "PASS" +FAIL = "FAIL" +INTERRUPTED = "INFRASTRUCTURE INTERRUPTED" +BLOCKED = "INFRASTRUCTURE BLOCKED" +RESULTS = (PASS, FAIL, INTERRUPTED, BLOCKED) +INFRASTRUCTURE = "infrastructure" +FAILURE_KINDS = (INFRASTRUCTURE, "program defect", "ac2") +VERDICTS = (PASS, FAIL) +EVIDENCE_FIELDS = ("incident", "time_utc", "source", "error_type") +DECISION_FIELDS = ("text", "source") FIELDS = ( "attempt", "commit", @@ -30,9 +39,15 @@ "started_utc", "finished_utc", "result", + "failure_kind", + "comparison_verdict", "reason", "haiku_usd", ) +FINAL_RULE = ( + "INFRASTRUCTURE INTERRUPTED is neither an AC1b failure nor a pass; AC1b acceptance is " + "completed only by an attempt whose result is PASS." +) class LedgerError(ValueError): @@ -40,7 +55,14 @@ class LedgerError(ValueError): def validate(ledger: dict) -> dict: - """Attempts numbered 1..n, a known result, a reason, spend within the per-attempt cap.""" + """Numbered attempts, known results and kinds, spend within the per-attempt cap. + + An infrastructure result (INTERRUPTED or BLOCKED) needs failure_kind + ``infrastructure``, external incident evidence and, for INTERRUPTED, the + decision that classified it. A second consecutive infrastructure + attempt with the same error type must be BLOCKED: AC1b pauses instead + of resuming again. The comparison's own verdict is kept as recorded. + """ if ledger.get("original_ac6_usd") != ORIGINAL_AC6_USD: raise LedgerError(f"original_ac6_usd must stay {ORIGINAL_AC6_USD}") if ledger.get("cap_usd_per_attempt") != CAP_USD_PER_ATTEMPT: @@ -48,22 +70,71 @@ def validate(ledger: dict) -> dict: attempts = ledger.get("attempts") if not isinstance(attempts, list) or not attempts: raise LedgerError("no attempts recorded") + previous: dict | None = None for number, entry in enumerate(attempts, start=1): - missing = [f for f in FIELDS if f not in entry] - if missing: - raise LedgerError(f"attempt {number}: missing {missing}") - if entry["attempt"] != number: - raise LedgerError(f"attempt numbers must run 1..n; found {entry['attempt']}") - if entry["result"] not in RESULTS: - raise LedgerError(f"attempt {number}: result {entry['result']!r} not in {RESULTS}") - if not str(entry["reason"]).strip(): - raise LedgerError(f"attempt {number}: a result needs a reason") - spent = entry["haiku_usd"] - if not isinstance(spent, int | float) or not 0 <= spent <= CAP_USD_PER_ATTEMPT: - raise LedgerError(f"attempt {number}: haiku_usd {spent!r} outside 0..cap") + check_attempt(number, entry) + check_repeat(number, entry, previous) + previous = entry return ledger +def check_attempt(number: int, entry: dict) -> None: + missing = [f for f in FIELDS if f not in entry] + if missing: + raise LedgerError(f"attempt {number}: missing {missing}") + if entry["attempt"] != number: + raise LedgerError(f"attempt numbers must run 1..n; found {entry['attempt']}") + result, kind = entry["result"], entry["failure_kind"] + if result not in RESULTS: + raise LedgerError(f"attempt {number}: result {result!r} not in {RESULTS}") + if entry["comparison_verdict"] not in VERDICTS: + raise LedgerError(f"attempt {number}: comparison_verdict must be one of {VERDICTS}") + if result == PASS and (kind is not None or entry["comparison_verdict"] != PASS): + raise LedgerError(f"attempt {number}: PASS needs a PASS comparison and no failure_kind") + if result != PASS and kind not in FAILURE_KINDS: + raise LedgerError(f"attempt {number}: failure_kind {kind!r} not in {FAILURE_KINDS}") + if result in (INTERRUPTED, BLOCKED): + check_infrastructure(number, entry) + if not str(entry["reason"]).strip(): + raise LedgerError(f"attempt {number}: a result needs a reason") + spent = entry["haiku_usd"] + if not isinstance(spent, int | float) or not 0 <= spent <= CAP_USD_PER_ATTEMPT: + raise LedgerError(f"attempt {number}: haiku_usd {spent!r} outside 0..cap") + + +def check_infrastructure(number: int, entry: dict) -> None: + if entry["failure_kind"] != INFRASTRUCTURE: + raise LedgerError(f"attempt {number}: {entry['result']} is for infrastructure only") + if not filled(entry.get("evidence"), EVIDENCE_FIELDS): + raise LedgerError(f"attempt {number}: {entry['result']} needs evidence {EVIDENCE_FIELDS}") + if entry["result"] == INTERRUPTED and not filled(entry.get("decision"), DECISION_FIELDS): + raise LedgerError(f"attempt {number}: INTERRUPTED needs the decision {DECISION_FIELDS}") + + +def check_repeat(number: int, entry: dict, previous: dict | None) -> None: + """The same infrastructure error twice in a row pauses AC1b (BLOCKED).""" + repeated = ( + previous is not None + and previous["failure_kind"] == INFRASTRUCTURE == entry["failure_kind"] + and error_type(previous) is not None + and error_type(previous) == error_type(entry) + ) + if repeated and entry["result"] != BLOCKED: + raise LedgerError( + f"attempt {number}: the infrastructure error of attempt {number - 1} recurred; " + f"the result must be {BLOCKED} (AC1b paused)" + ) + + +def error_type(entry: dict) -> str | None: + evidence = entry.get("evidence") + return evidence.get("error_type") if isinstance(evidence, dict) else None + + +def filled(value: object, fields: tuple[str, ...]) -> bool: + return isinstance(value, dict) and all(str(value.get(f) or "").strip() for f in fields) + + def load(path: Path = REPO / LEDGER) -> dict: return validate(json.loads(path.read_text(encoding="utf-8"))) @@ -107,6 +178,7 @@ def budget(ledger: dict, current: dict | None = None) -> dict[str, object]: "note": "sum over AC1b attempts only; never added to the original experiment cost", }, "cap_usd_per_attempt": CAP_USD_PER_ATTEMPT, + "final_rule": FINAL_RULE, "rule": ( "reproduction-validation spend is never reported as original experiment cost and " "does not change the AC6 US$5 conclusion" @@ -137,6 +209,8 @@ def markdown(section: dict) -> list[str]: "", "Reproduction-validation spend is never reported as original experiment cost and does " "not change the AC6 conclusion (Haiku spend at most US$5).", + "", + FINAL_RULE, ] diff --git a/src/tinyrouter/figures.py b/src/tinyrouter/figures.py index 38f346f..5afc0e6 100644 --- a/src/tinyrouter/figures.py +++ b/src/tinyrouter/figures.py @@ -178,7 +178,8 @@ def panel_title(index: int, k: int, aggregation: str, seeds: tuple[int, ...] | N title = f"({'abcd'[index]}) ModernBERT k={k} ({aggregation})" if seeds is None: return title - return f"{title}\nfinal router of seed{'s' * (len(seeds) > 1)} {', '.join(map(str, seeds))}" + plural = "s" * (len(seeds) > 1) + return f"{title}\nmean of 3 seeds; final router of seed{plural} {', '.join(map(str, seeds))}" def risk_coverage(summary: dict, curves: dict, plt) -> object: # noqa: ANN001 diff --git a/tests/test_ac1b_ledger.py b/tests/test_ac1b_ledger.py index 2ff51a5..0b18e57 100644 --- a/tests/test_ac1b_ledger.py +++ b/tests/test_ac1b_ledger.py @@ -18,18 +18,108 @@ def committed() -> dict: return ac1b_ledger.load(ROOT / ac1b_ledger.LEDGER) -def test_the_committed_ledger_records_both_failed_attempts_at_3992840(): +def test_the_committed_ledger_records_both_attempts_at_3992840(): attempts = committed()["attempts"] - assert [(a["attempt"], a["result"], a["haiku_usd"]) for a in attempts] == [ - (1, "FAIL", 0.0), - (2, "FAIL", 3.178751), + assert [ + (a["attempt"], a["result"], a["comparison_verdict"], a["haiku_usd"]) for a in attempts + ] == [ + (1, "INFRASTRUCTURE INTERRUPTED", "FAIL", 0.0), + (2, "FAIL", "FAIL", 3.178751), ] + assert [a["failure_kind"] for a in attempts] == ["infrastructure", "program defect"] assert {a["reproduction_id"] for a in attempts} == {"3992840ddb3f"} - assert attempts[0]["reason"].startswith("infrastructure") - assert attempts[1]["reason"].startswith("program defect") + assert attempts[0]["decision"]["source"] == ( + "Drew's decision, relayed by the coordinator (2026-09-29)" + ) + assert "status.claude.com" in attempts[0]["evidence"]["source"] for a in attempts: - assert (ROOT / a["evidence"] / "comparison.md").is_file() - assert (ROOT / a["evidence"] / "INCIDENT.md").is_file() + assert (ROOT / a["evidence_dir"] / "comparison.md").is_file() + assert (ROOT / a["evidence_dir"] / "INCIDENT.md").is_file() + + +def test_the_comparison_verdict_in_the_evidence_is_kept_as_written(): + """Attempt 1 is INTERRUPTED in the ledger; its comparison still says FAIL.""" + text = (EVIDENCE / "attempt-1" / "comparison.md").read_text(encoding="utf-8") + assert text.startswith("# AC1b comparison\n\n**Verdict: FAIL**") + + +def interrupted() -> dict: + return copy.deepcopy(committed()["attempts"][0]) + + +def with_attempts(*attempts: dict) -> dict: + ledger = copy.deepcopy(committed()) + ledger["attempts"] = [{**a, "attempt": i} for i, a in enumerate(attempts, start=1)] + return ledger + + +@pytest.mark.parametrize( + "change", + [ + {"evidence": None}, + {"evidence": {"incident": "x", "time_utc": "t", "source": "s", "error_type": " "}}, + {"decision": None}, + {"failure_kind": "program defect"}, + {"failure_kind": "ac2"}, + {"failure_kind": None}, + ], + ids=["no evidence", "blank error type", "no decision", "program defect", "ac2", "no kind"], +) +def test_infrastructure_interrupted_is_refused_without_its_conditions(change): + with pytest.raises(LedgerError): + ac1b_ledger.validate(with_attempts({**interrupted(), **change})) + + +def test_a_program_defect_can_only_be_fail(): + defect = copy.deepcopy(committed()["attempts"][1]) + for result in ("INFRASTRUCTURE INTERRUPTED", "INFRASTRUCTURE BLOCKED", "PASS"): + with pytest.raises(LedgerError): + ac1b_ledger.validate(with_attempts({**defect, "result": result})) + + +def test_failure_kind_is_an_enumeration(): + defect = copy.deepcopy(committed()["attempts"][1]) + with pytest.raises(LedgerError, match="failure_kind"): + ac1b_ledger.validate(with_attempts({**defect, "failure_kind": "flaky"})) + + +def test_the_same_infrastructure_error_twice_in_a_row_must_be_blocked(): + first = interrupted() + with pytest.raises(LedgerError, match="INFRASTRUCTURE BLOCKED"): + ac1b_ledger.validate(with_attempts(first, interrupted())) + with pytest.raises(LedgerError, match="INFRASTRUCTURE BLOCKED"): + ac1b_ledger.validate(with_attempts(first, {**interrupted(), "result": "FAIL"})) + blocked = {**interrupted(), "result": "INFRASTRUCTURE BLOCKED"} + assert ac1b_ledger.validate(with_attempts(first, blocked)) + + +def test_a_different_infrastructure_error_may_be_interrupted_again(): + other = interrupted() + other["evidence"] = {**other["evidence"], "error_type": "HTTP 529 overloaded"} + assert ac1b_ledger.validate(with_attempts(interrupted(), other)) + + +def test_blocked_needs_infrastructure_evidence_too(): + blocked = {**interrupted(), "result": "INFRASTRUCTURE BLOCKED", "evidence": None} + with pytest.raises(LedgerError): + ac1b_ledger.validate(with_attempts(blocked)) + + +def test_pass_needs_a_pass_comparison_and_no_failure_kind(): + good = {**copy.deepcopy(committed()["attempts"][1]), "result": "PASS"} + with pytest.raises(LedgerError): + ac1b_ledger.validate(with_attempts({**good, "failure_kind": None})) + assert ac1b_ledger.validate( + with_attempts({**good, "failure_kind": None, "comparison_verdict": "PASS"}) + ) + + +def test_the_budget_states_that_only_a_pass_completes_ac1b(): + text = "\n".join(ac1b_ledger.markdown(ac1b_ledger.budget(committed()))) + assert ac1b_ledger.FINAL_RULE in text + assert "neither an AC1b failure nor a pass" in ac1b_ledger.FINAL_RULE + for doc in ac1b_ledger.BUDGET_DOCS: + assert ac1b_ledger.FINAL_RULE in (ROOT / doc).read_text(encoding="utf-8") def test_the_original_experiment_cost_and_the_cap_cannot_be_edited_in_the_ledger(): @@ -42,7 +132,14 @@ def test_the_original_experiment_cost_and_the_cap_cannot_be_edited_in_the_ledger @pytest.mark.parametrize( ("field", "value"), - [("attempt", 3), ("result", "PASSED"), ("reason", " "), ("haiku_usd", 5.01), ("haiku_usd", -1)], + [ + ("attempt", 3), + ("result", "PASSED"), + ("reason", " "), + ("haiku_usd", 5.01), + ("haiku_usd", -1), + ("comparison_verdict", "NOT STARTED"), + ], ) def test_a_malformed_attempt_is_refused(field, value): ledger = copy.deepcopy(committed()) diff --git a/tests/test_figures.py b/tests/test_figures.py index c680791..885ce32 100644 --- a/tests/test_figures.py +++ b/tests/test_figures.py @@ -42,8 +42,8 @@ def test_mixed_per_seed_aggregations_get_one_panel_each_titled_with_their_seeds( titles = [ax.get_title() for ax in fig.axes] assert titles == [ "(a) ModernBERT k=10 (summed)", - "(b) ModernBERT k=100 (argmax)\nfinal router of seed 42", - "(c) ModernBERT k=100 (summed)\nfinal router of seeds 43, 44", + "(b) ModernBERT k=100 (argmax)\nmean of 3 seeds; final router of seed 42", + "(c) ModernBERT k=100 (summed)\nmean of 3 seeds; final router of seeds 43, 44", ] plt.close(fig)