From 0fc0b45312b51bf6f26ce230b4fd7d0fbc005496 Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:19:07 +0800 Subject: [PATCH 1/5] Add offline RQ2 to RQ4 analysis from stored logits and Haiku predictions make analysis reads the 75 logits archives and the 8,600 stored Haiku predictions, trains nothing and calls no API, and writes results/analysis/{summary,curves,haiku}.json. Temperature, 8-way aggregation, deferral threshold, hybrid signal and operating-curve thresholds are chosen on validation only; every choosing function raises LeakageError on any other split. Inputs are checked before anything runs (curve indexes, archive count, Haiku rows, gold labels equal across archives and Haiku), and outputs are read back and checked before the completion line is printed. --- DEVLOG.md | 37 + Makefile | 11 +- README.md | 5 + results/analysis/curves.json | 1 + results/analysis/haiku.json | 112 + results/analysis/summary.json | 64051 +++++++++++++++++++++++++++++++ src/tinyrouter/analysis.py | 347 + src/tinyrouter/analysis_run.py | 370 + src/tinyrouter/haiku.py | 143 + src/tinyrouter/selective.py | 207 + tests/test_analysis.py | 222 + tests/test_analysis_run.py | 248 + tests/test_haiku.py | 83 + tests/test_selective.py | 149 + 14 files changed, 65985 insertions(+), 1 deletion(-) create mode 100644 results/analysis/curves.json create mode 100644 results/analysis/haiku.json create mode 100644 results/analysis/summary.json create mode 100644 src/tinyrouter/analysis.py create mode 100644 src/tinyrouter/analysis_run.py create mode 100644 src/tinyrouter/haiku.py create mode 100644 src/tinyrouter/selective.py create mode 100644 tests/test_analysis.py create mode 100644 tests/test_analysis_run.py create mode 100644 tests/test_haiku.py create mode 100644 tests/test_selective.py diff --git a/DEVLOG.md b/DEVLOG.md index b9e1c5c..183c844 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -4,6 +4,43 @@ --- +## 2026-09-29（深夜）：步驟 4 後半，RQ2 到 RQ4 離線分析 + +### 本次工作 / 執行摘要 +- 新增 `make analysis`（`analysis_run.py`、`analysis.py`、`selective.py`、`haiku.py`）：只讀 75 個 logits archive 與 Haiku 8,600 筆逐筆預測，不訓練、不打 API。輸出 `results/analysis/summary.json`、`curves.json`、`haiku.json`，兩次執行逐位元組相同（沒有任何亂數）。 +- 開跑前比照 completeness.py 驗證：四個 curve index 重跑 `verify_index`（BERT、ModernBERT 各 18、消融 3、基準 36；BERT k=100 即 AC2 三個 run），archive 總數 75 與 Haiku 列數 8,600 都是字面值；每個 split 內所有 archive 的 labels 必須相同，Haiku 的 `gold_intent` 必須與之逐列相等。輸出先寫暫存檔、讀回檢查（25 組、每組 3 seeds、兩個 Haiku split）才換上，最後印 `completed analysis (75/75 archives, 8600/8600 llm rows, 25 groups)`。 +- 只用 validation 決定、程式層級守住（傳 test 就 `LeakageError`，有測試）：溫度、8 類聚合（validation 8 類準確率高者，平手取 argmax）、每個訊號在每個目標 risk 的門檻、hybrid 用哪個訊號（validation coverage 大者，再比 validation AURC）、操作曲線上的門檻。另有一條測試把 test labels 打亂，確認所有選擇都不變。 +- 門檻：候選是 validation 上所有不同的分數，取「selective risk 的單側 95% Wilson 上界（z = 1.645）≤ 目標」中 coverage 最大者；同分的列永遠一起收或一起退。沒有門檻可行時，router 全部交給 LLM（coverage 0），並記 `feasible: false`。目標 2% 與 5% 兩檔。 +- 訊號（越大越有信心）：`msp`、`entropy`（負熵）在 T = 1；`msp_t` 在擬合的 T；`margin` 是擬合 T 下前兩名 log 機率的差，argmax 聚合時等於 (z1 − z2)/T，排序與原始分數 margin 相同。argmax 聚合的訊號取自 151 類分布，summed 取自 8 類分布。TF-IDF 只報 `msp_t` 與 `margin`、只報校準後 ECE；多數類不擬合 T、不報訊號（PLAN §4.1）。 +- oracle：只把小模型 8 類判錯的查詢交給 Haiku。錯誤回收率 = 被交出去且 Haiku 救回的小模型錯誤 ÷ 小模型全部錯誤；「抓到的可回收錯誤」= 同一個分子 ÷ Haiku 會救回的小模型錯誤（oracle 的分子）。 +- 並列：LLM-only、small-only、hybrid（每個目標 risk）、oracle。std 是 3 seeds 的樣本標準差（ddof=1）。 +- 舊專案解析規則（子字串、取集合迭代到的第一個）重做成 `legacy_parse`，順序改用 `AGENTS` 固定，並另計「含兩個以上標籤、在舊程式裡答案不固定」的列數。 + +### 核心發現 / 數據 +（全部取自 `results/analysis/*.json`，test，百分比，mean ± std） +- **Haiku 4.5 zero-shot**：8 類準確率 82.1，OOS recall 56.8（Wilson 95% 53.7 到 59.8），每 1K 查詢 US$0.369。8,600 筆回覆全是 8 個標籤之一，parse_failed 0 列，新舊解析規則不一致 0 列，順序相依 0 列。 +- **ModernBERT k=100（主模型，validation 選 argmax）**：small-only 91.9 ± 0.1，OOS recall 61.1 ± 0.4；oracle 95.1 ± 0.1、LLM 呼叫 8.1 ± 0.1。 +- **validation 選的門檻在 test 上守不住目標 risk**。目標 2%：ModernBERT k=100 test selective risk 7.3 ± 0.9（coverage 98.7）、k=25 為 5.3 ± 0.5；目標 5% 在 k ≥ 25 時 coverage 幾乎 100%，test selective risk 8.0 到 13.5。原因是 CLINC150 的 validation 只有 100/3,100（3.2%）OOS，test 是 1,000/5,500（18.2%）；小模型的錯誤大多是 OOS，validation 上錯誤率本來就低，Wilson 上界再保守也看不到 test 的 OOS 比例。這不是程式錯，是協定（只用 validation）在這個資料集上的已知代價，需要 Drew 決定要不要處理（見 Blockers）。 +- 小資料量時 hybrid 的價值最清楚：ModernBERT k=10 在目標 2% 下 88.0 ± 0.4，LLM 呼叫 23.9 ± 3.8，高於 Haiku 單獨的 82.1 與 small-only 的 81.5 ± 0.7。 +- **OOS 0 筆消融**：small-only OOS recall 0.0（從不預測 oos），但只靠不確定性當 OOS 分數時 AUROC 反而更高：msp_t 98.0 ± 0.1、AUPRC 92.2 ± 0.4，對照同點 OOS 250 筆的 91.9 ± 0.5、72.0 ± 0.7。注意這個比較對有 OOS 訓練的模型不利：它有信心地預測成 oos 的查詢，在「低信心 = OOS」的分數下反而被當成最不像 OOS。以 router 實際行為比較：消融在目標 2% 下 hybrid OOS recall 40.6 ± 1.0、高信心 OOS 誤派 37.7 ± 2.9；目標 5% 時門檻放到全收，高信心 OOS 誤派 99.3 ± 0.2。 +- 聚合方式：ModernBERT k ≤ 10 三個 seed 都選 summed，k ≥ 50 都選 argmax；BERT 在 k=10、50、100 三個 seed 選得不一致，兩者差距都在 1pp 內。兩種的 test 數字都在 JSON。 +- 多數類：argmax 聚合永遠猜 oos（k-shot 樣本裡 oos 是單一最大 intent），summed 永遠猜 finance；validation 選 summed，test 8 類 20.7。 + +### Blockers / 遇到的問題 +- **待 Drew 決定**：validation 與 test 的 OOS 比例差 5.7 倍，使「只用 validation 選門檻」在 test 上系統性地超出目標 risk。可選：(a) 照實報，當成部署時要重新校準門檻的證據；(b) 另報一個敏感度版本，validation 的 OOS 列依「假設的部署 OOS 比例」加權（比例是事先宣告的參數，不從 test 估），這會動到 PLAN 的協定，所以這次沒做。 +- 「不確定性當 OOS 分數」對有 OOS 訓練的模型不公平（上面的消融段）；若要公平比較，可另加「預測為 oos 或低信心」的組合分數。這次照任務只報四個訊號。 + +### Next +- [ ] Drew 決定上面兩點 +- [ ] 步驟 5：README 與圖（risk-coverage、操作曲線、reliability），成本表（RQ5），數字由 `make report` 從 `results/analysis/*.json` 產生 + +### Files / Budget +- 新增：`src/tinyrouter/analysis.py`、`analysis_run.py`、`selective.py`、`haiku.py`；`tests/test_analysis.py`、`test_analysis_run.py`、`test_selective.py`、`test_haiku.py`；`results/analysis/summary.json`、`curves.json`、`haiku.json` +- 修改：`Makefile`（`make analysis`）、`README.md`（指令表與目錄）、`DEVLOG.md` +- API 花費：US$0 + +--- + ## 2026-09-29（夜）：llm-smoke 第一次實跑失敗，temperature 改走 extra_body ### 本次工作 / 執行摘要 diff --git a/Makefile b/Makefile index 03bec17..cf174c4 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: setup lint format test test-network smoke train evaluate ac2 pilot-lr pilot-steps baselines \ - curve oos-ablation verify-logits llm-smoke llm verify-llm report clean-checkpoints + curve oos-ablation verify-logits llm-smoke llm verify-llm analysis report clean-checkpoints CONFIG ?= configs/bert-base.yaml SEED ?= 42 @@ -132,6 +132,15 @@ llm: verify-llm: uv run python -m tinyrouter.llm_run --verify +# RQ2 to RQ4 from the stored logits and Haiku predictions (docs/PLAN.md sections 3 +# and 4, AC3, AC4, AC6): no training, no API calls. Needs the archives in +# results/logits/ and results/llm/haiku-8way.jsonl (GitHub Releases; check them +# with verify-logits and verify-llm). Writes results/analysis/{summary,curves,haiku}.json. +# Done means the whole last line `completed analysis (75/75 archives, 8600/8600 llm +# rows, 25 groups)`. +analysis: + uv run python -m tinyrouter.analysis_run --quiet + report: uv run python -m tinyrouter.report diff --git a/README.md b/README.md index 63b3237..016269a 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Other targets: | `make llm` | Claude Haiku on validation 3,100 + test 5,500 in the 8-way space, one stored record per query; resumes; stops before passing `MAX_USD` (default 5); writes `results/llm/haiku-8way.{jsonl,json}` and `results/llm-manifest.json` | | `make verify-llm` | check `results/llm/haiku-8way.jsonl`: every row once, SHA-256 equal in the file, the summary and the manifest | | `make verify-logits` | check every archive in `results/logits/` against `results/logits-manifest.json` | +| `make analysis` | RQ2 to RQ4 from the stored logits and Haiku predictions, no training and no API calls; every temperature, aggregation, threshold and signal is chosen on validation; writes `results/analysis/{summary,curves,haiku}.json` and ends with `completed analysis (75/75 archives, 8600/8600 llm rows, 25 groups)` | | `make report` | build `results/summary.md` from `results/runs/*.json` | | `make clean-checkpoints` | delete all trained weights | @@ -88,6 +89,10 @@ src/tinyrouter/ ac2.py AC2 run over three seeds and PASS/FAIL verdict llm.py Claude Haiku zero-shot router: prompt, retries, pricing (baseline and fallback) llm_run.py Haiku over validation + test: journal, resume, cost cap, completion check + selective.py AURC, risk-coverage, AUROC, AUPRC, Wilson-bound threshold choice (validation only) + haiku.py stored Haiku predictions as arrays; old and new reply parsers + analysis.py one archive: 8-way aggregation, OOS, signals, ECE, fallback, oracle + analysis_run.py `make analysis`: completion checks, label cross-checks, mean and std over seeds report.py results/*.json -> results/summary.md smoke.py end-to-end wiring check tests/ pytest; `network` marker for Hub downloads diff --git a/results/analysis/curves.json b/results/analysis/curves.json new file mode 100644 index 0000000..309b75e --- /dev/null +++ b/results/analysis/curves.json @@ -0,0 +1 @@ +{"format_version":1,"inputs":{"archives":75,"logits_manifest_sha256":"2635659e9e95348b7b0de34d73f91fc3957e228b33491e6a41708265260b08db","curve_index_sha256":{"bert.json":"35b46d514e6b47728501678dd6e029fb6a670e9b57d4514d6a2e73827aa987a5","modernbert.json":"515c153596bfb0115b39b2a6166999135014ef2ca68361a35764aa1866e9ffc4","oos-ablation.json":"4d8bb6aa6e4cf64d17a15e8b0453825b946da948bcd6cddbe256afa7099e081b","baselines.json":"f6a483c306e2ea10c35c3515e39428e5654b8d1e2303fbadd5f0b56d4b7177d3"},"haiku_predictions_sha256":"cac660f25f06bad317573c14aa55c0d7d977619c3cd6471c97654d94ec3432a0","llm_rows":8600},"coverage_grid":[0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,0.7,0.75,0.8,0.85,0.9,0.95,1.0],"groups":{"bert/k1":{"model":"bert","k":1,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["bert-base-uncased-k1-seed42","bert-base-uncased-k1-seed43","bert-base-uncased-k1-seed44"],"logits_sha256":["33464fd83ba26e6de38f2df4a347659dc396c57d98ce9fca16b78ab31101da2d","de95f070e64b443b4761f3ca786da624343fe8a5dc20e3f341220333e586cff3","db5595875d86d5b3476689bb5932f72bdb7651effb3686d8636cb308e8279881"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.118788,"std":0.066091},{"mean":0.144848,"std":0.059037},{"mean":0.169697,"std":0.041479},{"mean":0.200606,"std":0.028113},{"mean":0.230303,"std":0.030183},{"mean":0.254545,"std":0.02422},{"mean":0.274632,"std":0.025541},{"mean":0.290606,"std":0.026521},{"mean":0.311246,"std":0.021924},{"mean":0.328242,"std":0.02007},{"mean":0.3427,"std":0.020055},{"mean":0.358384,"std":0.020544},{"mean":0.371375,"std":0.017226},{"mean":0.386407,"std":0.015431},{"mean":0.400646,"std":0.016558},{"mean":0.413636,"std":0.014309},{"mean":0.426453,"std":0.013458},{"mean":0.440269,"std":0.012617},{"mean":0.453652,"std":0.011738},{"mean":0.466121,"std":0.012365}],"operating":{"tau":[{"mean":0.169048,"std":0.022902},{"mean":0.116553,"std":0.012689},{"mean":0.093793,"std":0.009367},{"mean":0.079422,"std":0.007147},{"mean":0.070336,"std":0.005337},{"mean":0.063115,"std":0.004823},{"mean":0.057372,"std":0.004518},{"mean":0.052561,"std":0.003865},{"mean":0.048845,"std":0.003264},{"mean":0.045425,"std":0.003081},{"mean":0.042417,"std":0.00264},{"mean":0.039844,"std":0.002348},{"mean":0.037637,"std":0.002131},{"mean":0.035566,"std":0.00175},{"mean":0.033441,"std":0.00145},{"mean":0.031316,"std":0.001332},{"mean":0.029284,"std":0.001004},{"mean":0.027034,"std":0.000866},{"mean":0.024308,"std":0.000588},{"mean":0.016374,"std":0.000181}],"coverage":[{"mean":0.041333,"std":0.001111},{"mean":0.089879,"std":0.005173},{"mean":0.13297,"std":0.009959},{"mean":0.181212,"std":0.006696},{"mean":0.222364,"std":0.005464},{"mean":0.267576,"std":0.010147},{"mean":0.316848,"std":0.015577},{"mean":0.366545,"std":0.014213},{"mean":0.418121,"std":0.0123},{"mean":0.473697,"std":0.019246},{"mean":0.524303,"std":0.016255},{"mean":0.577091,"std":0.015753},{"mean":0.625697,"std":0.020883},{"mean":0.673879,"std":0.011644},{"mean":0.727455,"std":0.00756},{"mean":0.782848,"std":0.008249},{"mean":0.835091,"std":0.001898},{"mean":0.892485,"std":0.005558},{"mean":0.944242,"std":0.001001},{"mean":0.999697,"std":0.000525}],"selective_risk":[{"mean":0.11108,"std":0.069621},{"mean":0.137258,"std":0.056609},{"mean":0.160937,"std":0.045831},{"mean":0.18733,"std":0.032971},{"mean":0.213198,"std":0.028907},{"mean":0.238026,"std":0.026832},{"mean":0.264797,"std":0.023096},{"mean":0.279718,"std":0.026606},{"mean":0.29959,"std":0.023896},{"mean":0.318281,"std":0.023164},{"mean":0.337086,"std":0.019626},{"mean":0.350117,"std":0.019978},{"mean":0.3664,"std":0.018769},{"mean":0.378365,"std":0.017446},{"mean":0.394056,"std":0.01775},{"mean":0.408578,"std":0.015991},{"mean":0.422075,"std":0.01402},{"mean":0.438227,"std":0.011359},{"mean":0.452309,"std":0.01206},{"mean":0.465963,"std":0.012131}],"accuracy_8":[{"mean":0.820121,"std":0.002317},{"mean":0.819455,"std":0.002834},{"mean":0.816545,"std":0.002578},{"mean":0.811091,"std":0.002745},{"mean":0.803939,"std":0.00165},{"mean":0.794788,"std":0.002244},{"mean":0.78497,"std":0.004964},{"mean":0.77503,"std":0.006833},{"mean":0.761394,"std":0.007395},{"mean":0.745939,"std":0.008615},{"mean":0.729394,"std":0.008608},{"mean":0.71303,"std":0.01072},{"mean":0.695515,"std":0.011553},{"mean":0.678303,"std":0.011422},{"mean":0.657333,"std":0.011747},{"mean":0.633636,"std":0.013517},{"mean":0.611394,"std":0.010824},{"mean":0.585273,"std":0.008099},{"mean":0.56,"std":0.011495},{"mean":0.534182,"std":0.011918}],"high_conf_oos_misroute_rate":[{"mean":0.005667,"std":0.002082},{"mean":0.016667,"std":0.005508},{"mean":0.030333,"std":0.001528},{"mean":0.052,"std":0.006928},{"mean":0.068667,"std":0.012423},{"mean":0.092333,"std":0.016289},{"mean":0.124333,"std":0.034385},{"mean":0.159,"std":0.04078},{"mean":0.199333,"std":0.040452},{"mean":0.238333,"std":0.051033},{"mean":0.280667,"std":0.054308},{"mean":0.322,"std":0.068169},{"mean":0.369333,"std":0.071305},{"mean":0.407667,"std":0.072252},{"mean":0.472667,"std":0.078704},{"mean":0.527667,"std":0.079431},{"mean":0.587667,"std":0.084884},{"mean":0.669,"std":0.086816},{"mean":0.739333,"std":0.087535},{"mean":0.827,"std":0.08561}],"oos_misroute_rate":[{"mean":0.429667,"std":0.004726},{"mean":0.429,"std":0.008},{"mean":0.431,"std":0.012767},{"mean":0.434667,"std":0.01914},{"mean":0.438667,"std":0.024007},{"mean":0.445333,"std":0.024786},{"mean":0.457,"std":0.038691},{"mean":0.47,"std":0.047159},{"mean":0.486333,"std":0.048336},{"mean":0.502667,"std":0.053163},{"mean":0.524333,"std":0.058011},{"mean":0.547,"std":0.068352},{"mean":0.570333,"std":0.070713},{"mean":0.586667,"std":0.070607},{"mean":0.622667,"std":0.075798},{"mean":0.654333,"std":0.074895},{"mean":0.687333,"std":0.079589},{"mean":0.733333,"std":0.084358},{"mean":0.774667,"std":0.086633},{"mean":0.827,"std":0.08561}],"llm_call_rate":[{"mean":0.958667,"std":0.001111},{"mean":0.910121,"std":0.005173},{"mean":0.86703,"std":0.009959},{"mean":0.818788,"std":0.006696},{"mean":0.777636,"std":0.005464},{"mean":0.732424,"std":0.010147},{"mean":0.683152,"std":0.015577},{"mean":0.633455,"std":0.014213},{"mean":0.581879,"std":0.0123},{"mean":0.526303,"std":0.019246},{"mean":0.475697,"std":0.016255},{"mean":0.422909,"std":0.015753},{"mean":0.374303,"std":0.020883},{"mean":0.326121,"std":0.011644},{"mean":0.272545,"std":0.00756},{"mean":0.217152,"std":0.008249},{"mean":0.164909,"std":0.001898},{"mean":0.107515,"std":0.005558},{"mean":0.055758,"std":0.001001},{"mean":0.000303,"std":0.000525}]}},"entropy":{"risk_coverage":[{"mean":0.13697,"std":0.095819},{"mean":0.192121,"std":0.062703},{"mean":0.231919,"std":0.049717},{"mean":0.258182,"std":0.05005},{"mean":0.286788,"std":0.047991},{"mean":0.306667,"std":0.046501},{"mean":0.322597,"std":0.04184},{"mean":0.338485,"std":0.036171},{"mean":0.355152,"std":0.032283},{"mean":0.371515,"std":0.030285},{"mean":0.383691,"std":0.026472},{"mean":0.391818,"std":0.022723},{"mean":0.402051,"std":0.022587},{"mean":0.411082,"std":0.019517},{"mean":0.42101,"std":0.019938},{"mean":0.431894,"std":0.017776},{"mean":0.441426,"std":0.01359},{"mean":0.45037,"std":0.012229},{"mean":0.456715,"std":0.012825},{"mean":0.466121,"std":0.012365}],"operating":{"tau":[{"mean":-4.387107,"std":0.085749},{"mean":-4.519868,"std":0.05109},{"mean":-4.585229,"std":0.036766},{"mean":-4.626706,"std":0.029885},{"mean":-4.655781,"std":0.025973},{"mean":-4.678065,"std":0.024614},{"mean":-4.69736,"std":0.022327},{"mean":-4.713906,"std":0.020537},{"mean":-4.729415,"std":0.019542},{"mean":-4.742843,"std":0.020639},{"mean":-4.755626,"std":0.018811},{"mean":-4.768742,"std":0.016882},{"mean":-4.780695,"std":0.01712},{"mean":-4.792576,"std":0.016384},{"mean":-4.804274,"std":0.0158},{"mean":-4.8173,"std":0.015705},{"mean":-4.830448,"std":0.01503},{"mean":-4.845671,"std":0.014053},{"mean":-4.866747,"std":0.011277},{"mean":-4.936085,"std":0.009787}],"coverage":[{"mean":0.043394,"std":0.003177},{"mean":0.083697,"std":0.006113},{"mean":0.13103,"std":0.013515},{"mean":0.175818,"std":0.018},{"mean":0.220606,"std":0.019109},{"mean":0.267818,"std":0.020449},{"mean":0.318606,"std":0.020651},{"mean":0.366606,"std":0.023515},{"mean":0.416848,"std":0.023092},{"mean":0.466121,"std":0.024778},{"mean":0.516848,"std":0.02148},{"mean":0.571273,"std":0.015143},{"mean":0.621273,"std":0.018273},{"mean":0.674,"std":0.018175},{"mean":0.727394,"std":0.016275},{"mean":0.783455,"std":0.02156},{"mean":0.835273,"std":0.018299},{"mean":0.887394,"std":0.013091},{"mean":0.945697,"std":0.002302},{"mean":0.999939,"std":0.000105}],"selective_risk":[{"mean":0.134284,"std":0.101578},{"mean":0.171974,"std":0.080465},{"mean":0.210499,"std":0.05754},{"mean":0.247391,"std":0.048421},{"mean":0.270958,"std":0.054244},{"mean":0.296068,"std":0.050868},{"mean":0.316008,"std":0.046805},{"mean":0.327155,"std":0.044907},{"mean":0.343731,"std":0.039354},{"mean":0.359746,"std":0.035083},{"mean":0.375213,"std":0.031249},{"mean":0.388209,"std":0.025826},{"mean":0.394864,"std":0.024383},{"mean":0.406564,"std":0.022973},{"mean":0.41692,"std":0.021238},{"mean":0.427631,"std":0.020018},{"mean":0.438748,"std":0.014658},{"mean":0.448321,"std":0.012916},{"mean":0.456096,"std":0.013331},{"mean":0.46609,"std":0.012318}],"accuracy_8":[{"mean":0.818909,"std":0.003367},{"mean":0.816061,"std":0.005395},{"mean":0.810303,"std":0.004195},{"mean":0.803152,"std":0.006023},{"mean":0.794727,"std":0.010914},{"mean":0.783091,"std":0.013106},{"mean":0.770788,"std":0.014829},{"mean":0.760424,"std":0.01787},{"mean":0.746485,"std":0.019182},{"mean":0.730909,"std":0.020298},{"mean":0.712545,"std":0.018898},{"mean":0.694364,"std":0.017273},{"mean":0.679515,"std":0.018113},{"mean":0.660182,"std":0.02092},{"mean":0.641515,"std":0.019566},{"mean":0.620788,"std":0.018086},{"mean":0.600545,"std":0.013807},{"mean":0.579333,"std":0.012232},{"mean":0.557515,"std":0.013461},{"mean":0.533939,"std":0.012276}],"high_conf_oos_misroute_rate":[{"mean":0.005667,"std":0.000577},{"mean":0.017333,"std":0.002887},{"mean":0.033667,"std":0.012662},{"mean":0.06,"std":0.024759},{"mean":0.088667,"std":0.038657},{"mean":0.120667,"std":0.045457},{"mean":0.155,"std":0.058043},{"mean":0.187667,"std":0.068821},{"mean":0.229,"std":0.080616},{"mean":0.266,"std":0.08747},{"mean":0.306667,"std":0.090853},{"mean":0.350667,"std":0.089366},{"mean":0.393333,"std":0.102002},{"mean":0.447333,"std":0.11339},{"mean":0.497,"std":0.117235},{"mean":0.555333,"std":0.119538},{"mean":0.614,"std":0.112641},{"mean":0.675667,"std":0.113742},{"mean":0.748667,"std":0.094129},{"mean":0.827333,"std":0.086176}],"oos_misroute_rate":[{"mean":0.428667,"std":0.005508},{"mean":0.429333,"std":0.010017},{"mean":0.432,"std":0.023516},{"mean":0.440667,"std":0.036529},{"mean":0.450667,"std":0.049166},{"mean":0.464,"std":0.052849},{"mean":0.48,"std":0.06353},{"mean":0.491667,"std":0.066711},{"mean":0.507,"std":0.073021},{"mean":0.522333,"std":0.083936},{"mean":0.545,"std":0.08743},{"mean":0.564,"std":0.091033},{"mean":0.585333,"std":0.095699},{"mean":0.614,"std":0.109302},{"mean":0.643,"std":0.113133},{"mean":0.672,"std":0.113618},{"mean":0.702333,"std":0.107342},{"mean":0.735,"std":0.103436},{"mean":0.776333,"std":0.092414},{"mean":0.827333,"std":0.086176}],"llm_call_rate":[{"mean":0.956606,"std":0.003177},{"mean":0.916303,"std":0.006113},{"mean":0.86897,"std":0.013515},{"mean":0.824182,"std":0.018},{"mean":0.779394,"std":0.019109},{"mean":0.732182,"std":0.020449},{"mean":0.681394,"std":0.020651},{"mean":0.633394,"std":0.023515},{"mean":0.583152,"std":0.023092},{"mean":0.533879,"std":0.024778},{"mean":0.483152,"std":0.02148},{"mean":0.428727,"std":0.015143},{"mean":0.378727,"std":0.018273},{"mean":0.326,"std":0.018175},{"mean":0.272606,"std":0.016275},{"mean":0.216545,"std":0.02156},{"mean":0.164727,"std":0.018299},{"mean":0.112606,"std":0.013091},{"mean":0.054303,"std":0.002302},{"mean":6.1e-05,"std":0.000105}]}},"margin":{"risk_coverage":[{"mean":0.095758,"std":0.024752},{"mean":0.131515,"std":0.021692},{"mean":0.17697,"std":0.015475},{"mean":0.206667,"std":0.008447},{"mean":0.238545,"std":0.011337},{"mean":0.264242,"std":0.014252},{"mean":0.289004,"std":0.01587},{"mean":0.312273,"std":0.019354},{"mean":0.327138,"std":0.018771},{"mean":0.346303,"std":0.018847},{"mean":0.363416,"std":0.018864},{"mean":0.380303,"std":0.01549},{"mean":0.392634,"std":0.016147},{"mean":0.404502,"std":0.014623},{"mean":0.416081,"std":0.014337},{"mean":0.428788,"std":0.014259},{"mean":0.440071,"std":0.013504},{"mean":0.451246,"std":0.01328},{"mean":0.458947,"std":0.011034},{"mean":0.466121,"std":0.012365}],"operating":{"tau":[{"mean":3.824925,"std":0.197376},{"mean":2.819114,"std":0.15041},{"mean":2.261501,"std":0.126888},{"mean":1.847171,"std":0.061069},{"mean":1.570111,"std":0.062708},{"mean":1.3399,"std":0.066517},{"mean":1.152882,"std":0.055307},{"mean":0.989255,"std":0.050834},{"mean":0.855392,"std":0.044671},{"mean":0.732497,"std":0.028999},{"mean":0.629081,"std":0.022806},{"mean":0.534014,"std":0.021161},{"mean":0.447916,"std":0.016407},{"mean":0.369932,"std":0.011978},{"mean":0.298262,"std":0.011744},{"mean":0.23081,"std":0.006248},{"mean":0.16733,"std":0.003842},{"mean":0.109333,"std":0.002088},{"mean":0.050244,"std":0.001525},{"mean":0.000289,"std":0.000226}],"coverage":[{"mean":0.043333,"std":0.006767},{"mean":0.091455,"std":0.009096},{"mean":0.136788,"std":0.009886},{"mean":0.188,"std":0.011277},{"mean":0.23303,"std":0.008826},{"mean":0.282909,"std":0.009764},{"mean":0.33097,"std":0.005407},{"mean":0.384303,"std":0.005407},{"mean":0.432667,"std":0.003815},{"mean":0.484909,"std":0.006771},{"mean":0.534424,"std":0.004818},{"mean":0.584545,"std":0.008358},{"mean":0.635212,"std":0.008321},{"mean":0.687333,"std":0.006731},{"mean":0.735636,"std":0.00189},{"mean":0.785212,"std":0.010094},{"mean":0.836364,"std":0.013546},{"mean":0.887152,"std":0.011558},{"mean":0.945394,"std":0.002783},{"mean":0.999758,"std":0.000105}],"selective_risk":[{"mean":0.080786,"std":0.013191},{"mean":0.125217,"std":0.009515},{"mean":0.163824,"std":0.019714},{"mean":0.200015,"std":0.009008},{"mean":0.227177,"std":0.011397},{"mean":0.256207,"std":0.014119},{"mean":0.277334,"std":0.015065},{"mean":0.305123,"std":0.019453},{"mean":0.322312,"std":0.016637},{"mean":0.341597,"std":0.017703},{"mean":0.358218,"std":0.017742},{"mean":0.375326,"std":0.01236},{"mean":0.389082,"std":0.015464},{"mean":0.403024,"std":0.013883},{"mean":0.413022,"std":0.014149},{"mean":0.424286,"std":0.013436},{"mean":0.437198,"std":0.014391},{"mean":0.448099,"std":0.012483},{"mean":0.458174,"std":0.011582},{"mean":0.466114,"std":0.012501}],"accuracy_8":[{"mean":0.821758,"std":0.000105},{"mean":0.82103,"std":0.000555},{"mean":0.816303,"std":0.001458},{"mean":0.809758,"std":0.002783},{"mean":0.803152,"std":0.002877},{"mean":0.790788,"std":0.005623},{"mean":0.779394,"std":0.006113},{"mean":0.764545,"std":0.007086},{"mean":0.751697,"std":0.004462},{"mean":0.733758,"std":0.005675},{"mean":0.718061,"std":0.005567},{"mean":0.697939,"std":0.004001},{"mean":0.679576,"std":0.005339},{"mean":0.660545,"std":0.00526},{"mean":0.643273,"std":0.005747},{"mean":0.623879,"std":0.009073},{"mean":0.60097,"std":0.013923},{"mean":0.579636,"std":0.012597},{"mean":0.555152,"std":0.011716},{"mean":0.533879,"std":0.012455}],"high_conf_oos_misroute_rate":[{"mean":0.006,"std":0.001},{"mean":0.018,"std":0.002646},{"mean":0.035667,"std":0.006429},{"mean":0.058,"std":0.013892},{"mean":0.084,"std":0.015875},{"mean":0.112333,"std":0.019732},{"mean":0.148333,"std":0.018583},{"mean":0.197,"std":0.021794},{"mean":0.241667,"std":0.024338},{"mean":0.29,"std":0.032741},{"mean":0.338333,"std":0.034675},{"mean":0.386667,"std":0.041405},{"mean":0.441667,"std":0.053163},{"mean":0.495667,"std":0.059475},{"mean":0.543333,"std":0.068879},{"mean":0.591,"std":0.072519},{"mean":0.644667,"std":0.086864},{"mean":0.708333,"std":0.079431},{"mean":0.770333,"std":0.078996},{"mean":0.827667,"std":0.086743}],"oos_misroute_rate":[{"mean":0.431,"std":0.003606},{"mean":0.431333,"std":0.010116},{"mean":0.436,"std":0.01562},{"mean":0.439,"std":0.022113},{"mean":0.446,"std":0.022113},{"mean":0.459667,"std":0.027429},{"mean":0.476333,"std":0.032517},{"mean":0.496333,"std":0.032808},{"mean":0.518333,"std":0.033322},{"mean":0.545,"std":0.039281},{"mean":0.565,"std":0.044844},{"mean":0.593333,"std":0.049803},{"mean":0.623333,"std":0.058184},{"mean":0.650667,"std":0.063311},{"mean":0.675,"std":0.068432},{"mean":0.700333,"std":0.070925},{"mean":0.732667,"std":0.085442},{"mean":0.766,"std":0.082656},{"mean":0.798333,"std":0.086373},{"mean":0.827667,"std":0.086743}],"llm_call_rate":[{"mean":0.956667,"std":0.006767},{"mean":0.908545,"std":0.009096},{"mean":0.863212,"std":0.009886},{"mean":0.812,"std":0.011277},{"mean":0.76697,"std":0.008826},{"mean":0.717091,"std":0.009764},{"mean":0.66903,"std":0.005407},{"mean":0.615697,"std":0.005407},{"mean":0.567333,"std":0.003815},{"mean":0.515091,"std":0.006771},{"mean":0.465576,"std":0.004818},{"mean":0.415455,"std":0.008358},{"mean":0.364788,"std":0.008321},{"mean":0.312667,"std":0.006731},{"mean":0.264364,"std":0.00189},{"mean":0.214788,"std":0.010094},{"mean":0.163636,"std":0.013546},{"mean":0.112848,"std":0.011558},{"mean":0.054606,"std":0.002783},{"mean":0.000242,"std":0.000105}]}},"msp_t":{"risk_coverage":[{"mean":0.106667,"std":0.053608},{"mean":0.144242,"std":0.045175},{"mean":0.167273,"std":0.036985},{"mean":0.196364,"std":0.022044},{"mean":0.225697,"std":0.021158},{"mean":0.251919,"std":0.024762},{"mean":0.26961,"std":0.022981},{"mean":0.290455,"std":0.023315},{"mean":0.309899,"std":0.020669},{"mean":0.325455,"std":0.017253},{"mean":0.342039,"std":0.015369},{"mean":0.355354,"std":0.016582},{"mean":0.37035,"std":0.020722},{"mean":0.384935,"std":0.017534},{"mean":0.398061,"std":0.014907},{"mean":0.411894,"std":0.013565},{"mean":0.42574,"std":0.011658},{"mean":0.439529,"std":0.011556},{"mean":0.452951,"std":0.012571},{"mean":0.466121,"std":0.012365}],"operating":{"tau":[{"mean":0.835905,"std":0.029873},{"mean":0.675324,"std":0.047485},{"mean":0.545187,"std":0.04542},{"mean":0.45461,"std":0.039336},{"mean":0.387175,"std":0.038089},{"mean":0.333321,"std":0.02918},{"mean":0.287645,"std":0.022153},{"mean":0.252071,"std":0.021353},{"mean":0.223902,"std":0.016597},{"mean":0.199667,"std":0.012246},{"mean":0.178963,"std":0.010956},{"mean":0.162175,"std":0.008505},{"mean":0.148006,"std":0.009022},{"mean":0.134128,"std":0.008645},{"mean":0.120775,"std":0.008477},{"mean":0.108724,"std":0.007963},{"mean":0.09681,"std":0.007276},{"mean":0.083996,"std":0.005977},{"mean":0.070396,"std":0.00436},{"mean":0.036244,"std":0.00322}],"coverage":[{"mean":0.04303,"std":0.003022},{"mean":0.088667,"std":0.006102},{"mean":0.13703,"std":0.009608},{"mean":0.179576,"std":0.007458},{"mean":0.224848,"std":0.004746},{"mean":0.269636,"std":0.011343},{"mean":0.318727,"std":0.01412},{"mean":0.375818,"std":0.010956},{"mean":0.425818,"std":0.018175},{"mean":0.477515,"std":0.01712},{"mean":0.526909,"std":0.013072},{"mean":0.576182,"std":0.016714},{"mean":0.624,"std":0.01106},{"mean":0.673879,"std":0.010035},{"mean":0.729455,"std":0.002678},{"mean":0.781212,"std":0.006748},{"mean":0.833212,"std":0.004967},{"mean":0.892848,"std":0.005095},{"mean":0.943939,"std":0.006183},{"mean":0.999576,"std":0.000458}],"selective_risk":[{"mean":0.098163,"std":0.051401},{"mean":0.136372,"std":0.040966},{"mean":0.158304,"std":0.036812},{"mean":0.185519,"std":0.029735},{"mean":0.211994,"std":0.021135},{"mean":0.235667,"std":0.020611},{"mean":0.25758,"std":0.022965},{"mean":0.28028,"std":0.021257},{"mean":0.300203,"std":0.022344},{"mean":0.318523,"std":0.019616},{"mean":0.333603,"std":0.01572},{"mean":0.349282,"std":0.016086},{"mean":0.363169,"std":0.019342},{"mean":0.378621,"std":0.017641},{"mean":0.393952,"std":0.015865},{"mean":0.406733,"std":0.01643},{"mean":0.421463,"std":0.014371},{"mean":0.437582,"std":0.010466},{"mean":0.451447,"std":0.011667},{"mean":0.465897,"std":0.012232}],"accuracy_8":[{"mean":0.820788,"std":0.00163},{"mean":0.819576,"std":0.001412},{"mean":0.817333,"std":0.002099},{"mean":0.811636,"std":0.002446},{"mean":0.804606,"std":0.00042},{"mean":0.796121,"std":0.002723},{"mean":0.786061,"std":0.004622},{"mean":0.772848,"std":0.004714},{"mean":0.759818,"std":0.007252},{"mean":0.744545,"std":0.008286},{"mean":0.729515,"std":0.007164},{"mean":0.713758,"std":0.008095},{"mean":0.697273,"std":0.009226},{"mean":0.678,"std":0.008561},{"mean":0.656848,"std":0.009117},{"mean":0.635333,"std":0.010068},{"mean":0.612364,"std":0.011789},{"mean":0.585394,"std":0.006748},{"mean":0.561212,"std":0.009724},{"mean":0.534242,"std":0.012011}],"high_conf_oos_misroute_rate":[{"mean":0.005333,"std":0.000577},{"mean":0.017667,"std":0.004509},{"mean":0.032667,"std":0.001528},{"mean":0.051,"std":0.009539},{"mean":0.071667,"std":0.007638},{"mean":0.095333,"std":0.01963},{"mean":0.125,"std":0.027875},{"mean":0.163667,"std":0.031943},{"mean":0.204333,"std":0.045764},{"mean":0.245333,"std":0.04572},{"mean":0.281,"std":0.044193},{"mean":0.325667,"std":0.058046},{"mean":0.372,"std":0.061539},{"mean":0.418,"std":0.066461},{"mean":0.474,"std":0.060324},{"mean":0.528,"std":0.082438},{"mean":0.592667,"std":0.079223},{"mean":0.667333,"std":0.080158},{"mean":0.740333,"std":0.082106},{"mean":0.826667,"std":0.085798}],"oos_misroute_rate":[{"mean":0.430333,"std":0.004041},{"mean":0.43,"std":0.008185},{"mean":0.430667,"std":0.01222},{"mean":0.436333,"std":0.021733},{"mean":0.440333,"std":0.019218},{"mean":0.446667,"std":0.026951},{"mean":0.455667,"std":0.035838},{"mean":0.473667,"std":0.040377},{"mean":0.488667,"std":0.047585},{"mean":0.507667,"std":0.0506},{"mean":0.527333,"std":0.048542},{"mean":0.548333,"std":0.058501},{"mean":0.572,"std":0.061},{"mean":0.596333,"std":0.066003},{"mean":0.622333,"std":0.062915},{"mean":0.655333,"std":0.072947},{"mean":0.690667,"std":0.076357},{"mean":0.733,"std":0.076603},{"mean":0.774667,"std":0.081002},{"mean":0.826667,"std":0.085798}],"llm_call_rate":[{"mean":0.95697,"std":0.003022},{"mean":0.911333,"std":0.006102},{"mean":0.86297,"std":0.009608},{"mean":0.820424,"std":0.007458},{"mean":0.775152,"std":0.004746},{"mean":0.730364,"std":0.011343},{"mean":0.681273,"std":0.01412},{"mean":0.624182,"std":0.010956},{"mean":0.574182,"std":0.018175},{"mean":0.522485,"std":0.01712},{"mean":0.473091,"std":0.013072},{"mean":0.423818,"std":0.016714},{"mean":0.376,"std":0.01106},{"mean":0.326121,"std":0.010035},{"mean":0.270545,"std":0.002678},{"mean":0.218788,"std":0.006748},{"mean":0.166788,"std":0.004967},{"mean":0.107152,"std":0.005095},{"mean":0.056061,"std":0.006183},{"mean":0.000424,"std":0.000458}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.038788,"std":0.020028},{"mean":0.061818,"std":0.017344},{"mean":0.095758,"std":0.005284},{"mean":0.140303,"std":0.004199},{"mean":0.195636,"std":0.010982},{"mean":0.250101,"std":0.012514},{"mean":0.297489,"std":0.014641},{"mean":0.336061,"std":0.017758},{"mean":0.366061,"std":0.014551},{"mean":0.391152,"std":0.012961},{"mean":0.408705,"std":0.011144},{"mean":0.424949,"std":0.007904},{"mean":0.435897,"std":0.006162},{"mean":0.441732,"std":0.008049},{"mean":0.447758,"std":0.008387},{"mean":0.45,"std":0.007843},{"mean":0.449697,"std":0.007472},{"mean":0.44835,"std":0.006609},{"mean":0.447528,"std":0.008474},{"mean":0.450727,"std":0.013235}],"operating":{"tau":[{"mean":0.39614,"std":0.015725},{"mean":0.367506,"std":0.008067},{"mean":0.347775,"std":0.007356},{"mean":0.331109,"std":0.006162},{"mean":0.314995,"std":0.004906},{"mean":0.30132,"std":0.004705},{"mean":0.289955,"std":0.004366},{"mean":0.280499,"std":0.00475},{"mean":0.272774,"std":0.004361},{"mean":0.265963,"std":0.003544},{"mean":0.259803,"std":0.00315},{"mean":0.25395,"std":0.00236},{"mean":0.248421,"std":0.001264},{"mean":0.243329,"std":0.000879},{"mean":0.238028,"std":0.000963},{"mean":0.232839,"std":0.001881},{"mean":0.227268,"std":0.003302},{"mean":0.221344,"std":0.004075},{"mean":0.213134,"std":0.004918},{"mean":0.179601,"std":0.008756}],"coverage":[{"mean":0.039091,"std":0.004329},{"mean":0.082303,"std":0.009186},{"mean":0.126061,"std":0.006587},{"mean":0.171273,"std":0.006309},{"mean":0.220727,"std":0.004738},{"mean":0.274303,"std":0.005043},{"mean":0.325818,"std":0.006456},{"mean":0.380727,"std":0.010832},{"mean":0.435697,"std":0.00861},{"mean":0.485273,"std":0.011192},{"mean":0.53697,"std":0.014004},{"mean":0.590424,"std":0.008197},{"mean":0.645333,"std":0.008912},{"mean":0.693697,"std":0.016081},{"mean":0.748364,"std":0.010432},{"mean":0.797636,"std":0.012677},{"mean":0.848,"std":0.012479},{"mean":0.897152,"std":0.01299},{"mean":0.946667,"std":0.008301},{"mean":0.999879,"std":0.00021}],"selective_risk":[{"mean":0.03883,"std":0.020139},{"mean":0.051418,"std":0.013951},{"mean":0.078247,"std":0.004987},{"mean":0.116726,"std":0.002452},{"mean":0.163545,"std":0.00707},{"mean":0.222535,"std":0.014739},{"mean":0.275787,"std":0.020437},{"mean":0.322869,"std":0.024045},{"mean":0.357084,"std":0.021297},{"mean":0.384118,"std":0.019265},{"mean":0.403901,"std":0.015907},{"mean":0.422609,"std":0.010451},{"mean":0.435054,"std":0.007133},{"mean":0.43987,"std":0.007306},{"mean":0.447083,"std":0.007792},{"mean":0.449355,"std":0.007178},{"mean":0.449825,"std":0.006966},{"mean":0.448622,"std":0.006682},{"mean":0.447292,"std":0.007878},{"mean":0.45078,"std":0.013145}],"accuracy_8":[{"mean":0.82103,"std":0.000458},{"mean":0.821091,"std":0.000656},{"mean":0.819455,"std":0.000909},{"mean":0.816545,"std":0.002097},{"mean":0.808242,"std":0.003182},{"mean":0.792848,"std":0.005227},{"mean":0.774303,"std":0.007578},{"mean":0.752182,"std":0.009588},{"mean":0.730667,"std":0.010045},{"mean":0.711455,"std":0.011852},{"mean":0.691273,"std":0.011927},{"mean":0.670545,"std":0.009133},{"mean":0.651576,"std":0.007723},{"mean":0.637455,"std":0.006598},{"mean":0.618485,"std":0.006518},{"mean":0.602788,"std":0.005993},{"mean":0.590364,"std":0.005178},{"mean":0.579333,"std":0.003379},{"mean":0.566545,"std":0.006395},{"mean":0.549273,"std":0.013235}],"high_conf_oos_misroute_rate":[{"mean":0.005667,"std":0.001155},{"mean":0.015333,"std":0.005859},{"mean":0.029667,"std":0.005686},{"mean":0.058333,"std":0.005508},{"mean":0.097667,"std":0.018903},{"mean":0.152,"std":0.042579},{"mean":0.204667,"std":0.052444},{"mean":0.272,"std":0.075941},{"mean":0.332333,"std":0.083213},{"mean":0.388,"std":0.085082},{"mean":0.442,"std":0.089599},{"mean":0.508333,"std":0.075791},{"mean":0.569667,"std":0.065041},{"mean":0.614,"std":0.060671},{"mean":0.669,"std":0.049},{"mean":0.715,"std":0.03297},{"mean":0.762667,"std":0.029023},{"mean":0.803333,"std":0.03247},{"mean":0.845667,"std":0.037018},{"mean":0.901,"std":0.061733}],"oos_misroute_rate":[{"mean":0.432,"std":0.001},{"mean":0.433,"std":0.001},{"mean":0.437,"std":0.003},{"mean":0.444,"std":0.008185},{"mean":0.46,"std":0.014731},{"mean":0.484,"std":0.027731},{"mean":0.511,"std":0.036497},{"mean":0.547333,"std":0.048604},{"mean":0.580667,"std":0.054794},{"mean":0.603667,"std":0.060211},{"mean":0.633667,"std":0.061175},{"mean":0.669333,"std":0.053267},{"mean":0.702667,"std":0.046264},{"mean":0.727,"std":0.040632},{"mean":0.761667,"std":0.030892},{"mean":0.792667,"std":0.019655},{"mean":0.818333,"std":0.016166},{"mean":0.844333,"std":0.021079},{"mean":0.871667,"std":0.034269},{"mean":0.901,"std":0.061733}],"llm_call_rate":[{"mean":0.960909,"std":0.004329},{"mean":0.917697,"std":0.009186},{"mean":0.873939,"std":0.006587},{"mean":0.828727,"std":0.006309},{"mean":0.779273,"std":0.004738},{"mean":0.725697,"std":0.005043},{"mean":0.674182,"std":0.006456},{"mean":0.619273,"std":0.010832},{"mean":0.564303,"std":0.00861},{"mean":0.514727,"std":0.011192},{"mean":0.46303,"std":0.014004},{"mean":0.409576,"std":0.008197},{"mean":0.354667,"std":0.008912},{"mean":0.306303,"std":0.016081},{"mean":0.251636,"std":0.010432},{"mean":0.202364,"std":0.012677},{"mean":0.152,"std":0.012479},{"mean":0.102848,"std":0.01299},{"mean":0.053333,"std":0.008301},{"mean":0.000121,"std":0.00021}]}},"entropy":{"risk_coverage":[{"mean":0.052121,"std":0.033788},{"mean":0.098788,"std":0.039094},{"mean":0.139798,"std":0.032442},{"mean":0.176061,"std":0.039838},{"mean":0.219152,"std":0.031945},{"mean":0.259596,"std":0.026154},{"mean":0.283636,"std":0.023016},{"mean":0.31,"std":0.018159},{"mean":0.329966,"std":0.020738},{"mean":0.348727,"std":0.020663},{"mean":0.364848,"std":0.019401},{"mean":0.378182,"std":0.014568},{"mean":0.389091,"std":0.009933},{"mean":0.40026,"std":0.008164},{"mean":0.40897,"std":0.004682},{"mean":0.415303,"std":0.002585},{"mean":0.424742,"std":0.005796},{"mean":0.434545,"std":0.010722},{"mean":0.444912,"std":0.013324},{"mean":0.450727,"std":0.013235}],"operating":{"tau":[{"mean":-1.721089,"std":0.017346},{"mean":-1.755334,"std":0.011405},{"mean":-1.776767,"std":0.007196},{"mean":-1.793579,"std":0.005688},{"mean":-1.808153,"std":0.005628},{"mean":-1.819437,"std":0.006387},{"mean":-1.829646,"std":0.007173},{"mean":-1.838745,"std":0.006862},{"mean":-1.847906,"std":0.004976},{"mean":-1.855211,"std":0.004402},{"mean":-1.862241,"std":0.004342},{"mean":-1.868894,"std":0.003375},{"mean":-1.875986,"std":0.002241},{"mean":-1.883211,"std":0.001862},{"mean":-1.891354,"std":0.000614},{"mean":-1.899906,"std":0.001406},{"mean":-1.90943,"std":0.003357},{"mean":-1.921619,"std":0.006092},{"mean":-1.940475,"std":0.011949},{"mean":-2.012126,"std":0.008437}],"coverage":[{"mean":0.039091,"std":0.007578},{"mean":0.080182,"std":0.007154},{"mean":0.127697,"std":0.009815},{"mean":0.174182,"std":0.007825},{"mean":0.220121,"std":0.005733},{"mean":0.265273,"std":0.004049},{"mean":0.312061,"std":0.000933},{"mean":0.358545,"std":0.003196},{"mean":0.41497,"std":0.014152},{"mean":0.463091,"std":0.013407},{"mean":0.517333,"std":0.006367},{"mean":0.566182,"std":0.008278},{"mean":0.618121,"std":0.008249},{"mean":0.669939,"std":0.008637},{"mean":0.724,"std":0.009805},{"mean":0.774485,"std":0.01246},{"mean":0.828121,"std":0.013708},{"mean":0.885818,"std":0.011717},{"mean":0.939333,"std":0.011923},{"mean":0.99903,"std":0.000735}],"selective_risk":[{"mean":0.046607,"std":0.035375},{"mean":0.079389,"std":0.038553},{"mean":0.117413,"std":0.044164},{"mean":0.158453,"std":0.040187},{"mean":0.193541,"std":0.03653},{"mean":0.231776,"std":0.035836},{"mean":0.265443,"std":0.024107},{"mean":0.289502,"std":0.022372},{"mean":0.315699,"std":0.024841},{"mean":0.333332,"std":0.022051},{"mean":0.3547,"std":0.01941},{"mean":0.368218,"std":0.016373},{"mean":0.381954,"std":0.012092},{"mean":0.394307,"std":0.008251},{"mean":0.403837,"std":0.004705},{"mean":0.412362,"std":0.004216},{"mean":0.420555,"std":0.004607},{"mean":0.432406,"std":0.010442},{"mean":0.44366,"std":0.0143},{"mean":0.450805,"std":0.013229}],"accuracy_8":[{"mean":0.820606,"std":0.000378},{"mean":0.819152,"std":0.000757},{"mean":0.816606,"std":0.002107},{"mean":0.81103,"std":0.002917},{"mean":0.803333,"std":0.00469},{"mean":0.792788,"std":0.005587},{"mean":0.779879,"std":0.004226},{"mean":0.767939,"std":0.005147},{"mean":0.752485,"std":0.009239},{"mean":0.73897,"std":0.010302},{"mean":0.720788,"std":0.006924},{"mean":0.705212,"std":0.00548},{"mean":0.689576,"std":0.003022},{"mean":0.672364,"std":0.001135},{"mean":0.654,"std":0.003186},{"mean":0.636,"std":0.00581},{"mean":0.617212,"std":0.009207},{"mean":0.592909,"std":0.014281},{"mean":0.570061,"std":0.018636},{"mean":0.549333,"std":0.013}],"high_conf_oos_misroute_rate":[{"mean":0.005333,"std":0.002309},{"mean":0.016333,"std":0.007506},{"mean":0.033,"std":0.01833},{"mean":0.054333,"std":0.014503},{"mean":0.085333,"std":0.006658},{"mean":0.119333,"std":0.011846},{"mean":0.156333,"std":0.012662},{"mean":0.192,"std":0.015716},{"mean":0.239,"std":0.022913},{"mean":0.282333,"std":0.028113},{"mean":0.332,"std":0.040447},{"mean":0.370667,"std":0.038991},{"mean":0.421333,"std":0.040501},{"mean":0.474667,"std":0.035346},{"mean":0.537,"std":0.031177},{"mean":0.597667,"std":0.040501},{"mean":0.667333,"std":0.039552},{"mean":0.751333,"std":0.039004},{"mean":0.841,"std":0.043715},{"mean":0.901,"std":0.061733}],"oos_misroute_rate":[{"mean":0.432333,"std":0.000577},{"mean":0.435333,"std":0.002517},{"mean":0.437667,"std":0.005508},{"mean":0.441667,"std":0.004726},{"mean":0.451667,"std":0.004933},{"mean":0.464,"std":0.004},{"mean":0.478667,"std":0.004041},{"mean":0.494333,"std":0.010017},{"mean":0.516667,"std":0.010214},{"mean":0.54,"std":0.0151},{"mean":0.567333,"std":0.024028},{"mean":0.59,"std":0.023643},{"mean":0.617,"std":0.025515},{"mean":0.650667,"std":0.018009},{"mean":0.688,"std":0.013115},{"mean":0.727,"std":0.01044},{"mean":0.771667,"std":0.015011},{"mean":0.824333,"std":0.025502},{"mean":0.883667,"std":0.034933},{"mean":0.902333,"std":0.060682}],"llm_call_rate":[{"mean":0.960909,"std":0.007578},{"mean":0.919818,"std":0.007154},{"mean":0.872303,"std":0.009815},{"mean":0.825818,"std":0.007825},{"mean":0.779879,"std":0.005733},{"mean":0.734727,"std":0.004049},{"mean":0.687939,"std":0.000933},{"mean":0.641455,"std":0.003196},{"mean":0.58503,"std":0.014152},{"mean":0.536909,"std":0.013407},{"mean":0.482667,"std":0.006367},{"mean":0.433818,"std":0.008278},{"mean":0.381879,"std":0.008249},{"mean":0.330061,"std":0.008637},{"mean":0.276,"std":0.009805},{"mean":0.225515,"std":0.01246},{"mean":0.171879,"std":0.013708},{"mean":0.114182,"std":0.011717},{"mean":0.060667,"std":0.011923},{"mean":0.00097,"std":0.000735}]}},"margin":{"risk_coverage":[{"mean":0.065455,"std":0.022709},{"mean":0.099394,"std":0.026494},{"mean":0.111111,"std":0.023295},{"mean":0.124545,"std":0.009226},{"mean":0.144242,"std":0.010598},{"mean":0.161818,"std":0.007373},{"mean":0.181818,"std":0.003922},{"mean":0.203939,"std":0.005007},{"mean":0.228283,"std":0.007148},{"mean":0.253212,"std":0.006629},{"mean":0.278678,"std":0.005876},{"mean":0.297576,"std":0.007838},{"mean":0.320932,"std":0.011155},{"mean":0.34303,"std":0.011197},{"mean":0.361293,"std":0.012646},{"mean":0.382955,"std":0.014193},{"mean":0.401783,"std":0.013221},{"mean":0.420067,"std":0.013335},{"mean":0.435343,"std":0.013944},{"mean":0.450727,"std":0.013235}],"operating":{"tau":[{"mean":3.128212,"std":0.334279},{"mean":2.356827,"std":0.279102},{"mean":1.953989,"std":0.21942},{"mean":1.662878,"std":0.156006},{"mean":1.432512,"std":0.127619},{"mean":1.247055,"std":0.095869},{"mean":1.085519,"std":0.091455},{"mean":0.948222,"std":0.07645},{"mean":0.820208,"std":0.063971},{"mean":0.712065,"std":0.058757},{"mean":0.614375,"std":0.054041},{"mean":0.522193,"std":0.044029},{"mean":0.442605,"std":0.031479},{"mean":0.36266,"std":0.024062},{"mean":0.290356,"std":0.020602},{"mean":0.224414,"std":0.016893},{"mean":0.163521,"std":0.015463},{"mean":0.109293,"std":0.007036},{"mean":0.055541,"std":0.003962},{"mean":0.000226,"std":7.7e-05}],"coverage":[{"mean":0.042848,"std":0.003494},{"mean":0.088788,"std":0.004528},{"mean":0.133818,"std":0.01092},{"mean":0.179455,"std":0.011754},{"mean":0.224364,"std":0.006988},{"mean":0.272364,"std":0.01113},{"mean":0.324061,"std":0.010176},{"mean":0.372424,"std":0.006496},{"mean":0.423455,"std":0.009914},{"mean":0.473879,"std":0.010289},{"mean":0.523879,"std":0.014474},{"mean":0.575758,"std":0.013489},{"mean":0.625212,"std":0.019991},{"mean":0.68097,"std":0.020391},{"mean":0.736364,"std":0.018425},{"mean":0.788848,"std":0.013589},{"mean":0.843939,"std":0.007175},{"mean":0.891212,"std":0.006403},{"mean":0.944424,"std":0.007532},{"mean":0.999697,"std":0.000278}],"selective_risk":[{"mean":0.060376,"std":0.020394},{"mean":0.090982,"std":0.030122},{"mean":0.10578,"std":0.022901},{"mean":0.120467,"std":0.017591},{"mean":0.136114,"std":0.010809},{"mean":0.149735,"std":0.005974},{"mean":0.173568,"std":0.003042},{"mean":0.191044,"std":0.001034},{"mean":0.214233,"std":0.004296},{"mean":0.239569,"std":0.002352},{"mean":0.265467,"std":0.003171},{"mean":0.287292,"std":0.002408},{"mean":0.308391,"std":0.00223},{"mean":0.334835,"std":0.004239},{"mean":0.356074,"std":0.006792},{"mean":0.378542,"std":0.00768},{"mean":0.40006,"std":0.011351},{"mean":0.416646,"std":0.011073},{"mean":0.433484,"std":0.012103},{"mean":0.45062,"std":0.013406}],"accuracy_8":[{"mean":0.822061,"std":0.000278},{"mean":0.822485,"std":0.000278},{"mean":0.820606,"std":0.000735},{"mean":0.819212,"std":0.001412},{"mean":0.815879,"std":0.000458},{"mean":0.811758,"std":0.002011},{"mean":0.804424,"std":0.00164},{"mean":0.797818,"std":0.001734},{"mean":0.787879,"std":0.003279},{"mean":0.774727,"std":0.002641},{"mean":0.759091,"std":0.004524},{"mean":0.742788,"std":0.002795},{"mean":0.726,"std":0.004589},{"mean":0.701152,"std":0.004742},{"mean":0.678788,"std":0.002699},{"mean":0.654788,"std":0.002735},{"mean":0.628545,"std":0.00756},{"mean":0.605333,"std":0.007502},{"mean":0.578848,"std":0.009733},{"mean":0.549333,"std":0.013339}],"high_conf_oos_misroute_rate":[{"mean":0.005,"std":0.001},{"mean":0.013,"std":0.002},{"mean":0.029333,"std":0.007767},{"mean":0.046667,"std":0.013204},{"mean":0.066333,"std":0.010408},{"mean":0.089333,"std":0.011015},{"mean":0.125333,"std":0.01914},{"mean":0.158667,"std":0.024826},{"mean":0.202,"std":0.02816},{"mean":0.249667,"std":0.030436},{"mean":0.304667,"std":0.038109},{"mean":0.358333,"std":0.043936},{"mean":0.413333,"std":0.042004},{"mean":0.478,"std":0.045211},{"mean":0.545667,"std":0.044881},{"mean":0.615,"std":0.05011},{"mean":0.698,"std":0.044933},{"mean":0.763667,"std":0.046715},{"mean":0.827667,"std":0.054372},{"mean":0.900667,"std":0.062067}],"oos_misroute_rate":[{"mean":0.430667,"std":0.003055},{"mean":0.429667,"std":0.005686},{"mean":0.434,"std":0.010392},{"mean":0.438,"std":0.014},{"mean":0.442333,"std":0.012741},{"mean":0.452,"std":0.016093},{"mean":0.467333,"std":0.022546},{"mean":0.479333,"std":0.026102},{"mean":0.498,"std":0.025239},{"mean":0.524,"std":0.031321},{"mean":0.553333,"std":0.033561},{"mean":0.586,"std":0.03985},{"mean":0.610667,"std":0.044061},{"mean":0.650333,"std":0.04236},{"mean":0.691,"std":0.039154},{"mean":0.729333,"std":0.049014},{"mean":0.779,"std":0.047085},{"mean":0.817,"std":0.050587},{"mean":0.856,"std":0.058043},{"mean":0.901,"std":0.061733}],"llm_call_rate":[{"mean":0.957152,"std":0.003494},{"mean":0.911212,"std":0.004528},{"mean":0.866182,"std":0.01092},{"mean":0.820545,"std":0.011754},{"mean":0.775636,"std":0.006988},{"mean":0.727636,"std":0.01113},{"mean":0.675939,"std":0.010176},{"mean":0.627576,"std":0.006496},{"mean":0.576545,"std":0.009914},{"mean":0.526121,"std":0.010289},{"mean":0.476121,"std":0.014474},{"mean":0.424242,"std":0.013489},{"mean":0.374788,"std":0.019991},{"mean":0.31903,"std":0.020391},{"mean":0.263636,"std":0.018425},{"mean":0.211152,"std":0.013589},{"mean":0.156061,"std":0.007175},{"mean":0.108788,"std":0.006403},{"mean":0.055576,"std":0.007532},{"mean":0.000303,"std":0.000278}]}},"msp_t":{"risk_coverage":[{"mean":0.084848,"std":0.041354},{"mean":0.103636,"std":0.034064},{"mean":0.117172,"std":0.029748},{"mean":0.135152,"std":0.020537},{"mean":0.150303,"std":0.011275},{"mean":0.163232,"std":0.004899},{"mean":0.181299,"std":0.004252},{"mean":0.202424,"std":0.006808},{"mean":0.225993,"std":0.004127},{"mean":0.246545,"std":0.005091},{"mean":0.270303,"std":0.005691},{"mean":0.291313,"std":0.00431},{"mean":0.314312,"std":0.006721},{"mean":0.335758,"std":0.007965},{"mean":0.356525,"std":0.007819},{"mean":0.375833,"std":0.008983},{"mean":0.39508,"std":0.008915},{"mean":0.413401,"std":0.010883},{"mean":0.432026,"std":0.012167},{"mean":0.450727,"std":0.013235}],"operating":{"tau":[{"mean":0.88426,"std":0.03203},{"mean":0.778626,"std":0.052967},{"mean":0.697548,"std":0.051093},{"mean":0.633217,"std":0.047498},{"mean":0.581189,"std":0.037315},{"mean":0.5387,"std":0.033183},{"mean":0.503324,"std":0.028115},{"mean":0.46938,"std":0.023727},{"mean":0.438961,"std":0.020567},{"mean":0.41346,"std":0.020279},{"mean":0.389635,"std":0.01871},{"mean":0.368472,"std":0.016758},{"mean":0.348258,"std":0.014365},{"mean":0.329715,"std":0.011587},{"mean":0.311246,"std":0.011224},{"mean":0.29525,"std":0.010206},{"mean":0.277812,"std":0.008301},{"mean":0.258026,"std":0.006041},{"mean":0.235874,"std":0.003259},{"mean":0.174611,"std":0.007033}],"coverage":[{"mean":0.041091,"std":0.001585},{"mean":0.086606,"std":0.004091},{"mean":0.132848,"std":0.010137},{"mean":0.179758,"std":0.010107},{"mean":0.224485,"std":0.011442},{"mean":0.27,"std":0.006844},{"mean":0.318667,"std":0.008925},{"mean":0.369394,"std":0.01103},{"mean":0.418364,"std":0.013047},{"mean":0.46703,"std":0.009568},{"mean":0.517333,"std":0.007732},{"mean":0.567273,"std":0.009759},{"mean":0.617636,"std":0.010769},{"mean":0.669212,"std":0.013339},{"mean":0.724182,"std":0.005464},{"mean":0.774424,"std":0.006711},{"mean":0.83097,"std":0.006823},{"mean":0.889697,"std":0.005779},{"mean":0.944545,"std":0.006433},{"mean":0.999333,"std":0.000735}],"selective_risk":[{"mean":0.076324,"std":0.04308},{"mean":0.106244,"std":0.039238},{"mean":0.113707,"std":0.033439},{"mean":0.126669,"std":0.030858},{"mean":0.141454,"std":0.013821},{"mean":0.155128,"std":0.009354},{"mean":0.169236,"std":0.003121},{"mean":0.18818,"std":0.000861},{"mean":0.211737,"std":0.004382},{"mean":0.232283,"std":0.001286},{"mean":0.254651,"std":0.007119},{"mean":0.277205,"std":0.005831},{"mean":0.299149,"std":0.006437},{"mean":0.322299,"std":0.004626},{"mean":0.345903,"std":0.006483},{"mean":0.365655,"std":0.005158},{"mean":0.388847,"std":0.007416},{"mean":0.409164,"std":0.008312},{"mean":0.429557,"std":0.009364},{"mean":0.450365,"std":0.012968}],"accuracy_8":[{"mean":0.821515,"std":0.001315},{"mean":0.820848,"std":0.00147},{"mean":0.820061,"std":0.001514},{"mean":0.817697,"std":0.002366},{"mean":0.814788,"std":0.001096},{"mean":0.811091,"std":0.000656},{"mean":0.805212,"std":0.002044},{"mean":0.798303,"std":0.002421},{"mean":0.788303,"std":0.004644},{"mean":0.778121,"std":0.003574},{"mean":0.764182,"std":0.004882},{"mean":0.749212,"std":0.004547},{"mean":0.731879,"std":0.004536},{"mean":0.712364,"std":0.004589},{"mean":0.689394,"std":0.003274},{"mean":0.668,"std":0.001967},{"mean":0.640606,"std":0.003182},{"mean":0.612424,"std":0.004742},{"mean":0.581939,"std":0.006696},{"mean":0.549879,"std":0.012782}],"high_conf_oos_misroute_rate":[{"mean":0.005,"std":0.001},{"mean":0.014333,"std":0.003786},{"mean":0.027667,"std":0.004163},{"mean":0.043333,"std":0.008963},{"mean":0.063333,"std":0.017098},{"mean":0.080667,"std":0.016073},{"mean":0.106,"std":0.020518},{"mean":0.139667,"std":0.020033},{"mean":0.181667,"std":0.01963},{"mean":0.225333,"std":0.027301},{"mean":0.266667,"std":0.027737},{"mean":0.315333,"std":0.027062},{"mean":0.367,"std":0.041725},{"mean":0.423333,"std":0.040772},{"mean":0.486,"std":0.033601},{"mean":0.546667,"std":0.04236},{"mean":0.622333,"std":0.040624},{"mean":0.705333,"std":0.043386},{"mean":0.793667,"std":0.050856},{"mean":0.898333,"std":0.058774}],"oos_misroute_rate":[{"mean":0.430333,"std":0.003215},{"mean":0.43,"std":0.005568},{"mean":0.433333,"std":0.008386},{"mean":0.436,"std":0.010583},{"mean":0.44,"std":0.014799},{"mean":0.444333,"std":0.017616},{"mean":0.455333,"std":0.021502},{"mean":0.468333,"std":0.019757},{"mean":0.487,"std":0.021378},{"mean":0.504667,"std":0.028711},{"mean":0.522667,"std":0.03107},{"mean":0.542333,"std":0.030238},{"mean":0.570333,"std":0.040079},{"mean":0.605333,"std":0.039119},{"mean":0.641,"std":0.032741},{"mean":0.675333,"std":0.040377},{"mean":0.716667,"std":0.03876},{"mean":0.764,"std":0.046163},{"mean":0.825667,"std":0.054243},{"mean":0.898667,"std":0.059349}],"llm_call_rate":[{"mean":0.958909,"std":0.001585},{"mean":0.913394,"std":0.004091},{"mean":0.867152,"std":0.010137},{"mean":0.820242,"std":0.010107},{"mean":0.775515,"std":0.011442},{"mean":0.73,"std":0.006844},{"mean":0.681333,"std":0.008925},{"mean":0.630606,"std":0.01103},{"mean":0.581636,"std":0.013047},{"mean":0.53297,"std":0.009568},{"mean":0.482667,"std":0.007732},{"mean":0.432727,"std":0.009759},{"mean":0.382364,"std":0.010769},{"mean":0.330788,"std":0.013339},{"mean":0.275818,"std":0.005464},{"mean":0.225576,"std":0.006711},{"mean":0.16903,"std":0.006823},{"mean":0.110303,"std":0.005779},{"mean":0.055455,"std":0.006433},{"mean":0.000667,"std":0.000735}]}}}}},"bert/k10":{"model":"bert","k":10,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["bert-base-uncased-k10-seed42","bert-base-uncased-k10-seed43","bert-base-uncased-k10-seed44"],"logits_sha256":["51df91b493b56ae7d49fdd3195bc1d53d55099ec5b1cf011dfa9579a8a6320a3","113c11c8b72a8f20ac46e70a1c8492ee1a4d0571b1b86caba1b583cc85d0faf8","b60ca5253fc8ff28108fcfa32d4aa79139357e9f5639d187b2270a266c594adb"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.012121,"std":0.011109},{"mean":0.011515,"std":0.003785},{"mean":0.011717,"std":0.0007},{"mean":0.012121,"std":0.00105},{"mean":0.012364,"std":0.00126},{"mean":0.014747,"std":0.004549},{"mean":0.017143,"std":0.007027},{"mean":0.020303,"std":0.007529},{"mean":0.024781,"std":0.00891},{"mean":0.027152,"std":0.007464},{"mean":0.030964,"std":0.007748},{"mean":0.03596,"std":0.008711},{"mean":0.041212,"std":0.007133},{"mean":0.047965,"std":0.008709},{"mean":0.059152,"std":0.009222},{"mean":0.0725,"std":0.009804},{"mean":0.090909,"std":0.012211},{"mean":0.115354,"std":0.014948},{"mean":0.145455,"std":0.017416},{"mean":0.177697,"std":0.017591}],"operating":{"tau":[{"mean":0.177774,"std":0.012723},{"mean":0.153852,"std":0.007467},{"mean":0.13943,"std":0.00555},{"mean":0.129199,"std":0.005388},{"mean":0.120494,"std":0.005446},{"mean":0.111667,"std":0.005055},{"mean":0.10394,"std":0.004711},{"mean":0.097189,"std":0.00485},{"mean":0.091317,"std":0.005026},{"mean":0.086027,"std":0.004592},{"mean":0.080905,"std":0.004631},{"mean":0.07587,"std":0.004619},{"mean":0.070926,"std":0.004426},{"mean":0.06622,"std":0.003713},{"mean":0.061523,"std":0.003606},{"mean":0.056424,"std":0.003341},{"mean":0.049923,"std":0.002855},{"mean":0.04348,"std":0.001867},{"mean":0.035148,"std":0.000905},{"mean":0.017002,"std":0.000316}],"coverage":[{"mean":0.041939,"std":0.003374},{"mean":0.085576,"std":0.003854},{"mean":0.125273,"std":0.001845},{"mean":0.162606,"std":0.005431},{"mean":0.201455,"std":0.007184},{"mean":0.245273,"std":0.005773},{"mean":0.287515,"std":0.005498},{"mean":0.334121,"std":0.008625},{"mean":0.379273,"std":0.009538},{"mean":0.425636,"std":0.004237},{"mean":0.467697,"std":0.007057},{"mean":0.511333,"std":0.001238},{"mean":0.555697,"std":0.003527},{"mean":0.602182,"std":0.004194},{"mean":0.646545,"std":0.004237},{"mean":0.697879,"std":0.007301},{"mean":0.753818,"std":0.009609},{"mean":0.811152,"std":0.00444},{"mean":0.885152,"std":0.00696},{"mean":0.99897,"std":0.00021}],"selective_risk":[{"mean":0.013158,"std":0.011992},{"mean":0.011423,"std":0.003745},{"mean":0.012573,"std":0.000673},{"mean":0.011531,"std":0.001376},{"mean":0.012345,"std":0.000689},{"mean":0.012344,"std":0.000897},{"mean":0.014293,"std":0.003225},{"mean":0.016798,"std":0.00572},{"mean":0.018658,"std":0.007872},{"mean":0.023263,"std":0.008578},{"mean":0.025439,"std":0.007572},{"mean":0.028081,"std":0.007292},{"mean":0.031268,"std":0.008196},{"mean":0.036134,"std":0.008804},{"mean":0.040845,"std":0.007812},{"mean":0.047399,"std":0.009314},{"mean":0.060332,"std":0.010674},{"mean":0.075848,"std":0.009625},{"mean":0.107611,"std":0.010758},{"mean":0.176971,"std":0.017491}],"accuracy_8":[{"mean":0.821758,"std":0.00084},{"mean":0.823879,"std":0.000278},{"mean":0.827152,"std":0.000458},{"mean":0.831697,"std":0.002482},{"mean":0.835636,"std":0.003848},{"mean":0.840667,"std":0.004777},{"mean":0.844364,"std":0.00553},{"mean":0.849273,"std":0.005357},{"mean":0.853636,"std":0.003632},{"mean":0.85703,"std":0.003754},{"mean":0.860485,"std":0.003611},{"mean":0.864182,"std":0.002182},{"mean":0.868303,"std":0.003611},{"mean":0.872242,"std":0.00447},{"mean":0.874909,"std":0.00387},{"mean":0.877273,"std":0.005894},{"mean":0.876242,"std":0.006385},{"mean":0.874727,"std":0.007441},{"mean":0.86503,"std":0.01152},{"mean":0.822909,"std":0.017582}],"high_conf_oos_misroute_rate":[{"mean":0.002333,"std":0.002517},{"mean":0.004333,"std":0.002517},{"mean":0.007,"std":0.001732},{"mean":0.008333,"std":0.001155},{"mean":0.010333,"std":0.002517},{"mean":0.012,"std":0.002646},{"mean":0.015667,"std":0.004726},{"mean":0.020333,"std":0.007095},{"mean":0.022667,"std":0.007095},{"mean":0.029667,"std":0.008083},{"mean":0.035333,"std":0.007371},{"mean":0.042667,"std":0.010066},{"mean":0.053333,"std":0.012503},{"mean":0.068333,"std":0.015044},{"mean":0.086333,"std":0.013577},{"mean":0.111333,"std":0.021825},{"mean":0.157667,"std":0.03139},{"mean":0.219,"std":0.033511},{"mean":0.359333,"std":0.039017},{"mean":0.735,"std":0.074081}],"oos_misroute_rate":[{"mean":0.433,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434333,"std":0.000577},{"mean":0.434333,"std":0.000577},{"mean":0.434667,"std":0.001155},{"mean":0.436667,"std":0.001155},{"mean":0.438,"std":0.001732},{"mean":0.438,"std":0.001732},{"mean":0.440667,"std":0.000577},{"mean":0.443,"std":0.001},{"mean":0.444333,"std":0.002082},{"mean":0.446333,"std":0.004619},{"mean":0.450333,"std":0.005508},{"mean":0.455,"std":0.007937},{"mean":0.461667,"std":0.01823},{"mean":0.478667,"std":0.027025},{"mean":0.497,"std":0.033779},{"mean":0.544333,"std":0.049943},{"mean":0.736333,"std":0.07446}],"llm_call_rate":[{"mean":0.958061,"std":0.003374},{"mean":0.914424,"std":0.003854},{"mean":0.874727,"std":0.001845},{"mean":0.837394,"std":0.005431},{"mean":0.798545,"std":0.007184},{"mean":0.754727,"std":0.005773},{"mean":0.712485,"std":0.005498},{"mean":0.665879,"std":0.008625},{"mean":0.620727,"std":0.009538},{"mean":0.574364,"std":0.004237},{"mean":0.532303,"std":0.007057},{"mean":0.488667,"std":0.001238},{"mean":0.444303,"std":0.003527},{"mean":0.397818,"std":0.004194},{"mean":0.353455,"std":0.004237},{"mean":0.302121,"std":0.007301},{"mean":0.246182,"std":0.009609},{"mean":0.188848,"std":0.00444},{"mean":0.114848,"std":0.00696},{"mean":0.00103,"std":0.00021}]}},"entropy":{"risk_coverage":[{"mean":0.012121,"std":0.011109},{"mean":0.013333,"std":0.012106},{"mean":0.022222,"std":0.007406},{"mean":0.025455,"std":0.007216},{"mean":0.025212,"std":0.006185},{"mean":0.027273,"std":0.007348},{"mean":0.027359,"std":0.007658},{"mean":0.03,"std":0.007767},{"mean":0.032593,"std":0.006469},{"mean":0.038424,"std":0.005684},{"mean":0.043967,"std":0.007067},{"mean":0.049192,"std":0.007578},{"mean":0.057902,"std":0.006518},{"mean":0.06632,"std":0.007053},{"mean":0.07903,"std":0.007926},{"mean":0.092273,"std":0.00908},{"mean":0.108021,"std":0.012588},{"mean":0.128822,"std":0.013096},{"mean":0.152217,"std":0.0157},{"mean":0.177697,"std":0.017591}],"operating":{"tau":[{"mean":-4.356248,"std":0.042895},{"mean":-4.423087,"std":0.022854},{"mean":-4.452543,"std":0.01952},{"mean":-4.474997,"std":0.017213},{"mean":-4.494102,"std":0.017106},{"mean":-4.513231,"std":0.018566},{"mean":-4.532654,"std":0.017798},{"mean":-4.551615,"std":0.017218},{"mean":-4.569143,"std":0.017854},{"mean":-4.585181,"std":0.019302},{"mean":-4.600776,"std":0.020134},{"mean":-4.617986,"std":0.023756},{"mean":-4.636857,"std":0.026729},{"mean":-4.655966,"std":0.027771},{"mean":-4.679017,"std":0.024361},{"mean":-4.703458,"std":0.020094},{"mean":-4.730994,"std":0.020018},{"mean":-4.763128,"std":0.01863},{"mean":-4.807891,"std":0.013246},{"mean":-4.930832,"std":0.004619}],"coverage":[{"mean":0.03903,"std":0.004714},{"mean":0.085273,"std":0.005091},{"mean":0.125455,"std":0.00634},{"mean":0.166061,"std":0.00873},{"mean":0.209394,"std":0.007972},{"mean":0.252606,"std":0.012286},{"mean":0.295939,"std":0.008455},{"mean":0.344485,"std":0.00347},{"mean":0.390182,"std":0.006203},{"mean":0.430364,"std":0.00513},{"mean":0.467879,"std":0.005398},{"mean":0.510061,"std":0.003951},{"mean":0.553091,"std":0.006426},{"mean":0.598121,"std":0.015546},{"mean":0.647515,"std":0.014373},{"mean":0.697273,"std":0.013473},{"mean":0.755394,"std":0.012704},{"mean":0.815152,"std":0.015794},{"mean":0.891091,"std":0.008153},{"mean":0.997758,"std":0.001568}],"selective_risk":[{"mean":0.015103,"std":0.016331},{"mean":0.012757,"std":0.012299},{"mean":0.01826,"std":0.008742},{"mean":0.023969,"std":0.006043},{"mean":0.025292,"std":0.007052},{"mean":0.024969,"std":0.007208},{"mean":0.026489,"std":0.008152},{"mean":0.027296,"std":0.007387},{"mean":0.029265,"std":0.006992},{"mean":0.030861,"std":0.00665},{"mean":0.035257,"std":0.005339},{"mean":0.039084,"std":0.00632},{"mean":0.043773,"std":0.007486},{"mean":0.049088,"std":0.009063},{"mean":0.057244,"std":0.009226},{"mean":0.06623,"std":0.008536},{"mean":0.079948,"std":0.011277},{"mean":0.097733,"std":0.014742},{"mean":0.124989,"std":0.015672},{"mean":0.176572,"std":0.017798}],"accuracy_8":[{"mean":0.822182,"std":0.002691},{"mean":0.824606,"std":0.003642},{"mean":0.827091,"std":0.00413},{"mean":0.829333,"std":0.005198},{"mean":0.833333,"std":0.005864},{"mean":0.837333,"std":0.004312},{"mean":0.840909,"std":0.004158},{"mean":0.845394,"std":0.003279},{"mean":0.849273,"std":0.003308},{"mean":0.852303,"std":0.003098},{"mean":0.854545,"std":0.002065},{"mean":0.858061,"std":0.001839},{"mean":0.859515,"std":0.00105},{"mean":0.862,"std":0.001553},{"mean":0.864545,"std":0.003037},{"mean":0.866182,"std":0.005818},{"mean":0.866303,"std":0.008095},{"mean":0.862848,"std":0.011751},{"mean":0.853576,"std":0.014541},{"mean":0.823333,"std":0.01761}],"high_conf_oos_misroute_rate":[{"mean":0.002333,"std":0.003215},{"mean":0.003,"std":0.003},{"mean":0.005333,"std":0.004041},{"mean":0.007667,"std":0.005686},{"mean":0.012,"std":0.003464},{"mean":0.015333,"std":0.002517},{"mean":0.021,"std":0.001732},{"mean":0.025667,"std":0.001528},{"mean":0.032333,"std":0.004041},{"mean":0.037667,"std":0.00611},{"mean":0.047333,"std":0.006028},{"mean":0.058333,"std":0.005859},{"mean":0.072,"std":0.009165},{"mean":0.09,"std":0.017321},{"mean":0.121667,"std":0.018448},{"mean":0.158333,"std":0.018877},{"mean":0.217333,"std":0.031501},{"mean":0.297667,"std":0.045236},{"mean":0.430333,"std":0.054455},{"mean":0.732,"std":0.076151}],"oos_misroute_rate":[{"mean":0.432667,"std":0.001155},{"mean":0.433,"std":0.001},{"mean":0.433333,"std":0.000577},{"mean":0.434,"std":0.001},{"mean":0.434333,"std":0.000577},{"mean":0.435,"std":0.001},{"mean":0.436,"std":0.001},{"mean":0.436667,"std":0.001528},{"mean":0.437667,"std":0.001528},{"mean":0.438333,"std":0.002082},{"mean":0.440333,"std":0.004041},{"mean":0.442667,"std":0.004509},{"mean":0.447667,"std":0.008505},{"mean":0.454667,"std":0.01106},{"mean":0.465667,"std":0.014978},{"mean":0.479,"std":0.019},{"mean":0.500333,"std":0.032868},{"mean":0.531333,"std":0.046576},{"mean":0.585,"std":0.055561},{"mean":0.734,"std":0.07494}],"llm_call_rate":[{"mean":0.96097,"std":0.004714},{"mean":0.914727,"std":0.005091},{"mean":0.874545,"std":0.00634},{"mean":0.833939,"std":0.00873},{"mean":0.790606,"std":0.007972},{"mean":0.747394,"std":0.012286},{"mean":0.704061,"std":0.008455},{"mean":0.655515,"std":0.00347},{"mean":0.609818,"std":0.006203},{"mean":0.569636,"std":0.00513},{"mean":0.532121,"std":0.005398},{"mean":0.489939,"std":0.003951},{"mean":0.446909,"std":0.006426},{"mean":0.401879,"std":0.015546},{"mean":0.352485,"std":0.014373},{"mean":0.302727,"std":0.013473},{"mean":0.244606,"std":0.012704},{"mean":0.184848,"std":0.015794},{"mean":0.108909,"std":0.008153},{"mean":0.002242,"std":0.001568}]}},"margin":{"risk_coverage":[{"mean":0.018182,"std":0.007273},{"mean":0.016364,"std":0.00481},{"mean":0.01899,"std":0.004257},{"mean":0.020303,"std":0.003674},{"mean":0.019879,"std":0.004842},{"mean":0.023232,"std":0.004257},{"mean":0.025628,"std":0.004231},{"mean":0.028333,"std":0.002585},{"mean":0.032323,"std":0.002799},{"mean":0.039152,"std":0.004308},{"mean":0.048264,"std":0.005522},{"mean":0.061111,"std":0.007275},{"mean":0.07338,"std":0.009379},{"mean":0.087706,"std":0.010685},{"mean":0.102141,"std":0.013572},{"mean":0.118409,"std":0.016734},{"mean":0.132906,"std":0.019343},{"mean":0.147811,"std":0.020184},{"mean":0.162616,"std":0.018927},{"mean":0.177697,"std":0.017591}],"operating":{"tau":[{"mean":7.521112,"std":0.307058},{"mean":6.558235,"std":0.224371},{"mean":5.852202,"std":0.20471},{"mean":5.307327,"std":0.211006},{"mean":4.829162,"std":0.207041},{"mean":4.401084,"std":0.203146},{"mean":3.964442,"std":0.136104},{"mean":3.573348,"std":0.094842},{"mean":3.207621,"std":0.101739},{"mean":2.85116,"std":0.087256},{"mean":2.51461,"std":0.108038},{"mean":2.141173,"std":0.082093},{"mean":1.791454,"std":0.053017},{"mean":1.482585,"std":0.009159},{"mean":1.177887,"std":0.035978},{"mean":0.905903,"std":0.045863},{"mean":0.643508,"std":0.034738},{"mean":0.418571,"std":0.030268},{"mean":0.19843,"std":0.021213},{"mean":0.001406,"std":0.000828}],"coverage":[{"mean":0.043394,"std":0.004536},{"mean":0.081333,"std":0.003028},{"mean":0.127818,"std":0.002993},{"mean":0.169939,"std":0.002373},{"mean":0.21303,"std":0.003588},{"mean":0.255455,"std":0.007578},{"mean":0.303697,"std":0.005519},{"mean":0.345455,"std":0.002751},{"mean":0.390121,"std":0.010212},{"mean":0.439455,"std":0.01118},{"mean":0.481818,"std":0.015506},{"mean":0.534121,"std":0.013787},{"mean":0.59097,"std":0.013167},{"mean":0.640788,"std":0.011275},{"mean":0.693636,"std":0.011562},{"mean":0.750424,"std":0.009841},{"mean":0.808848,"std":0.010249},{"mean":0.866848,"std":0.010729},{"mean":0.934909,"std":0.006938},{"mean":0.999212,"std":0.000757}],"selective_risk":[{"mean":0.018599,"std":0.006392},{"mean":0.019447,"std":0.00501},{"mean":0.018535,"std":0.006327},{"mean":0.02,"std":0.003534},{"mean":0.02053,"std":0.004104},{"mean":0.020728,"std":0.004447},{"mean":0.023005,"std":0.004607},{"mean":0.025811,"std":0.004288},{"mean":0.027529,"std":0.001876},{"mean":0.031896,"std":0.002279},{"mean":0.0355,"std":0.001507},{"mean":0.04551,"std":0.005062},{"mean":0.058807,"std":0.008198},{"mean":0.070685,"std":0.007174},{"mean":0.085538,"std":0.011786},{"mean":0.102975,"std":0.015066},{"mean":0.121906,"std":0.018484},{"mean":0.137922,"std":0.021267},{"mean":0.157822,"std":0.021058},{"mean":0.177415,"std":0.01725}],"accuracy_8":[{"mean":0.822848,"std":0.001747},{"mean":0.824909,"std":0.001845},{"mean":0.827515,"std":0.001689},{"mean":0.831152,"std":0.001277},{"mean":0.834606,"std":0.00134},{"mean":0.839152,"std":0.001821},{"mean":0.844788,"std":0.001034},{"mean":0.848848,"std":0.001481},{"mean":0.853333,"std":0.002154},{"mean":0.857333,"std":0.002842},{"mean":0.860848,"std":0.002003},{"mean":0.864364,"std":0.001734},{"mean":0.86497,"std":0.002259},{"mean":0.865455,"std":0.003887},{"mean":0.863212,"std":0.005976},{"mean":0.857212,"std":0.009556},{"mean":0.851333,"std":0.013435},{"mean":0.84497,"std":0.016494},{"mean":0.834667,"std":0.019244},{"mean":0.822424,"std":0.017486}],"high_conf_oos_misroute_rate":[{"mean":0.003333,"std":0.001528},{"mean":0.007,"std":0.001},{"mean":0.010333,"std":0.002517},{"mean":0.015333,"std":0.000577},{"mean":0.019667,"std":0.002309},{"mean":0.022667,"std":0.004041},{"mean":0.030667,"std":0.006658},{"mean":0.038333,"std":0.009866},{"mean":0.045333,"std":0.003786},{"mean":0.059333,"std":0.005686},{"mean":0.071,"std":0.006245},{"mean":0.099,"std":0.022271},{"mean":0.143,"std":0.037643},{"mean":0.187,"std":0.036014},{"mean":0.245,"std":0.052716},{"mean":0.32,"std":0.065368},{"mean":0.406333,"std":0.076788},{"mean":0.493667,"std":0.090362},{"mean":0.609667,"std":0.094299},{"mean":0.738667,"std":0.073364}],"oos_misroute_rate":[{"mean":0.433333,"std":0.001155},{"mean":0.434333,"std":0.001155},{"mean":0.436,"std":0.001},{"mean":0.437,"std":0.0},{"mean":0.437667,"std":0.000577},{"mean":0.439667,"std":0.000577},{"mean":0.440667,"std":0.003786},{"mean":0.441,"std":0.005},{"mean":0.442,"std":0.007},{"mean":0.445,"std":0.009165},{"mean":0.447333,"std":0.011676},{"mean":0.453333,"std":0.019218},{"mean":0.471333,"std":0.028919},{"mean":0.488,"std":0.030512},{"mean":0.513333,"std":0.040017},{"mean":0.551,"std":0.054111},{"mean":0.586667,"std":0.063066},{"mean":0.623333,"std":0.0777},{"mean":0.679333,"std":0.084607},{"mean":0.739667,"std":0.074467}],"llm_call_rate":[{"mean":0.956606,"std":0.004536},{"mean":0.918667,"std":0.003028},{"mean":0.872182,"std":0.002993},{"mean":0.830061,"std":0.002373},{"mean":0.78697,"std":0.003588},{"mean":0.744545,"std":0.007578},{"mean":0.696303,"std":0.005519},{"mean":0.654545,"std":0.002751},{"mean":0.609879,"std":0.010212},{"mean":0.560545,"std":0.01118},{"mean":0.518182,"std":0.015506},{"mean":0.465879,"std":0.013787},{"mean":0.40903,"std":0.013167},{"mean":0.359212,"std":0.011275},{"mean":0.306364,"std":0.011562},{"mean":0.249576,"std":0.009841},{"mean":0.191152,"std":0.010249},{"mean":0.133152,"std":0.010729},{"mean":0.065091,"std":0.006938},{"mean":0.000788,"std":0.000757}]}},"msp_t":{"risk_coverage":[{"mean":0.013333,"std":0.011689},{"mean":0.016364,"std":0.003636},{"mean":0.015354,"std":0.0007},{"mean":0.016061,"std":0.004099},{"mean":0.018182,"std":0.003636},{"mean":0.018182,"std":0.00437},{"mean":0.019221,"std":0.002699},{"mean":0.021818,"std":0.002981},{"mean":0.02532,"std":0.003436},{"mean":0.029697,"std":0.003279},{"mean":0.038347,"std":0.005649},{"mean":0.045051,"std":0.00648},{"mean":0.054079,"std":0.006944},{"mean":0.063983,"std":0.009615},{"mean":0.073293,"std":0.011415},{"mean":0.085833,"std":0.014234},{"mean":0.102032,"std":0.014849},{"mean":0.123636,"std":0.015994},{"mean":0.14756,"std":0.017296},{"mean":0.177697,"std":0.017591}],"operating":{"tau":[{"mean":0.997726,"std":0.000214},{"mean":0.994963,"std":0.000523},{"mean":0.99092,"std":0.001639},{"mean":0.985911,"std":0.00323},{"mean":0.977799,"std":0.004085},{"mean":0.96658,"std":0.005278},{"mean":0.951989,"std":0.006148},{"mean":0.931925,"std":0.00889},{"mean":0.908379,"std":0.01074},{"mean":0.876278,"std":0.015339},{"mean":0.835725,"std":0.013746},{"mean":0.787674,"std":0.018566},{"mean":0.733097,"std":0.023702},{"mean":0.670925,"std":0.025376},{"mean":0.606158,"std":0.02295},{"mean":0.544608,"std":0.025427},{"mean":0.478318,"std":0.02973},{"mean":0.399771,"std":0.039726},{"mean":0.295357,"std":0.026312},{"mean":0.08379,"std":0.017504}],"coverage":[{"mean":0.042909,"std":0.002993},{"mean":0.083515,"std":0.00691},{"mean":0.126848,"std":0.003674},{"mean":0.166727,"std":0.001585},{"mean":0.210606,"std":0.004514},{"mean":0.255394,"std":0.004261},{"mean":0.298364,"std":0.002751},{"mean":0.343455,"std":0.006576},{"mean":0.383394,"std":0.008682},{"mean":0.427152,"std":0.0147},{"mean":0.476788,"std":0.012636},{"mean":0.523212,"std":0.013829},{"mean":0.570545,"std":0.012298},{"mean":0.620727,"std":0.012871},{"mean":0.671758,"std":0.010867},{"mean":0.719697,"std":0.011897},{"mean":0.776303,"std":0.014168},{"mean":0.837273,"std":0.016012},{"mean":0.90303,"std":0.011679},{"mean":0.998909,"std":0.000656}],"selective_risk":[{"mean":0.016192,"std":0.014204},{"mean":0.016746,"std":0.006302},{"mean":0.015273,"std":0.001315},{"mean":0.014905,"std":0.001675},{"mean":0.016093,"std":0.00347},{"mean":0.01777,"std":0.003355},{"mean":0.018259,"std":0.004256},{"mean":0.019264,"std":0.002776},{"mean":0.02089,"std":0.00218},{"mean":0.024004,"std":0.003417},{"mean":0.027465,"std":0.00366},{"mean":0.032589,"std":0.003379},{"mean":0.041755,"std":0.004212},{"mean":0.049258,"std":0.008134},{"mean":0.057869,"std":0.00951},{"mean":0.06804,"std":0.013491},{"mean":0.079872,"std":0.014464},{"mean":0.097433,"std":0.017786},{"mean":0.123768,"std":0.017638},{"mean":0.176862,"std":0.017339}],"accuracy_8":[{"mean":0.822667,"std":0.001821},{"mean":0.824909,"std":0.00142},{"mean":0.828485,"std":0.002107},{"mean":0.831394,"std":0.002573},{"mean":0.835758,"std":0.003674},{"mean":0.841273,"std":0.002641},{"mean":0.846242,"std":0.001001},{"mean":0.850788,"std":0.001944},{"mean":0.85503,"std":0.002184},{"mean":0.85903,"std":0.00268},{"mean":0.863455,"std":0.002049},{"mean":0.866909,"std":0.001443},{"mean":0.868485,"std":0.001096},{"mean":0.871576,"std":0.003},{"mean":0.872121,"std":0.005},{"mean":0.872242,"std":0.007863},{"mean":0.871394,"std":0.009044},{"mean":0.867515,"std":0.011684},{"mean":0.855576,"std":0.014275},{"mean":0.822909,"std":0.017367}],"high_conf_oos_misroute_rate":[{"mean":0.002667,"std":0.003055},{"mean":0.006,"std":0.0},{"mean":0.008667,"std":0.001528},{"mean":0.011,"std":0.002646},{"mean":0.015,"std":0.003606},{"mean":0.02,"std":0.003606},{"mean":0.024333,"std":0.006429},{"mean":0.029333,"std":0.005686},{"mean":0.035,"std":0.005292},{"mean":0.044,"std":0.008},{"mean":0.055333,"std":0.010066},{"mean":0.069333,"std":0.010504},{"mean":0.093667,"std":0.014844},{"mean":0.120667,"std":0.023671},{"mean":0.148667,"std":0.028361},{"mean":0.186333,"std":0.046705},{"mean":0.237,"std":0.056152},{"mean":0.313,"std":0.074505},{"mean":0.439333,"std":0.081929},{"mean":0.734667,"std":0.073078}],"oos_misroute_rate":[{"mean":0.433333,"std":0.001528},{"mean":0.433667,"std":0.001155},{"mean":0.435,"std":0.001732},{"mean":0.435667,"std":0.001528},{"mean":0.436667,"std":0.000577},{"mean":0.437667,"std":0.000577},{"mean":0.439333,"std":0.001528},{"mean":0.440667,"std":0.003055},{"mean":0.441333,"std":0.003215},{"mean":0.441667,"std":0.005508},{"mean":0.443667,"std":0.008145},{"mean":0.448,"std":0.009644},{"mean":0.455667,"std":0.014364},{"mean":0.463667,"std":0.018717},{"mean":0.472667,"std":0.022745},{"mean":0.489667,"std":0.039068},{"mean":0.506667,"std":0.049943},{"mean":0.532333,"std":0.060781},{"mean":0.586667,"std":0.072831},{"mean":0.736667,"std":0.073364}],"llm_call_rate":[{"mean":0.957091,"std":0.002993},{"mean":0.916485,"std":0.00691},{"mean":0.873152,"std":0.003674},{"mean":0.833273,"std":0.001585},{"mean":0.789394,"std":0.004514},{"mean":0.744606,"std":0.004261},{"mean":0.701636,"std":0.002751},{"mean":0.656545,"std":0.006576},{"mean":0.616606,"std":0.008682},{"mean":0.572848,"std":0.0147},{"mean":0.523212,"std":0.012636},{"mean":0.476788,"std":0.013829},{"mean":0.429455,"std":0.012298},{"mean":0.379273,"std":0.012871},{"mean":0.328242,"std":0.010867},{"mean":0.280303,"std":0.011897},{"mean":0.223697,"std":0.014168},{"mean":0.162727,"std":0.016012},{"mean":0.09697,"std":0.011679},{"mean":0.001091,"std":0.000656}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.050909,"std":0.019242},{"mean":0.044848,"std":0.005845},{"mean":0.052525,"std":0.010999},{"mean":0.057576,"std":0.006053},{"mean":0.060848,"std":0.002939},{"mean":0.062222,"std":0.006676},{"mean":0.067186,"std":0.006454},{"mean":0.068182,"std":0.008344},{"mean":0.072593,"std":0.009933},{"mean":0.078788,"std":0.012708},{"mean":0.082424,"std":0.013046},{"mean":0.089495,"std":0.009022},{"mean":0.095291,"std":0.00844},{"mean":0.105714,"std":0.008774},{"mean":0.115879,"std":0.008069},{"mean":0.127197,"std":0.00975},{"mean":0.142816,"std":0.010206},{"mean":0.158788,"std":0.012323},{"mean":0.171164,"std":0.012489},{"mean":0.185273,"std":0.015458}],"operating":{"tau":[{"mean":0.514505,"std":0.023238},{"mean":0.483216,"std":0.025561},{"mean":0.450359,"std":0.024558},{"mean":0.417587,"std":0.016112},{"mean":0.38616,"std":0.010693},{"mean":0.366673,"std":0.008604},{"mean":0.35263,"std":0.006863},{"mean":0.341196,"std":0.007293},{"mean":0.330057,"std":0.008636},{"mean":0.320354,"std":0.00832},{"mean":0.309678,"std":0.008645},{"mean":0.29978,"std":0.007724},{"mean":0.290452,"std":0.005227},{"mean":0.280874,"std":0.004411},{"mean":0.272111,"std":0.003496},{"mean":0.262649,"std":0.003268},{"mean":0.252966,"std":0.002606},{"mean":0.24185,"std":0.00264},{"mean":0.228812,"std":0.002569},{"mean":0.180439,"std":0.006757}],"coverage":[{"mean":0.038909,"std":0.005671},{"mean":0.085818,"std":0.00413},{"mean":0.128545,"std":0.001666},{"mean":0.171455,"std":0.003938},{"mean":0.215152,"std":0.004312},{"mean":0.25903,"std":0.00164},{"mean":0.306788,"std":0.004967},{"mean":0.35097,"std":0.006566},{"mean":0.400182,"std":0.00964},{"mean":0.441576,"std":0.009254},{"mean":0.486061,"std":0.008874},{"mean":0.536121,"std":0.008663},{"mean":0.582606,"std":0.003574},{"mean":0.634364,"std":0.005695},{"mean":0.682909,"std":0.002545},{"mean":0.736606,"std":0.004914},{"mean":0.788485,"std":0.00469},{"mean":0.844364,"std":0.012456},{"mean":0.903879,"std":0.017049},{"mean":0.999394,"std":0.000555}],"selective_risk":[{"mean":0.047709,"std":0.022228},{"mean":0.043572,"std":0.009336},{"mean":0.048023,"std":0.012397},{"mean":0.054656,"std":0.008897},{"mean":0.058529,"std":0.004801},{"mean":0.060585,"std":0.004158},{"mean":0.062756,"std":0.007012},{"mean":0.066741,"std":0.007425},{"mean":0.068016,"std":0.008375},{"mean":0.071764,"std":0.012168},{"mean":0.077021,"std":0.013545},{"mean":0.081245,"std":0.014349},{"mean":0.087337,"std":0.011136},{"mean":0.093863,"std":0.008555},{"mean":0.102221,"std":0.009359},{"mean":0.112686,"std":0.008381},{"mean":0.124014,"std":0.01115},{"mean":0.141552,"std":0.014062},{"mean":0.159387,"std":0.016297},{"mean":0.185017,"std":0.01554}],"accuracy_8":[{"mean":0.820788,"std":0.00021},{"mean":0.82097,"std":0.000458},{"mean":0.821273,"std":0.000727},{"mean":0.822424,"std":0.001412},{"mean":0.824303,"std":0.002922},{"mean":0.826909,"std":0.003004},{"mean":0.830182,"std":0.003037},{"mean":0.833515,"std":0.003193},{"mean":0.835818,"std":0.003677},{"mean":0.838242,"std":0.003203},{"mean":0.84097,"std":0.004203},{"mean":0.844545,"std":0.003938},{"mean":0.84703,"std":0.004006},{"mean":0.850606,"std":0.002475},{"mean":0.851758,"std":0.005993},{"mean":0.850242,"std":0.004091},{"mean":0.847636,"std":0.006147},{"mean":0.840182,"std":0.009559},{"mean":0.831333,"std":0.012539},{"mean":0.814788,"std":0.015563}],"high_conf_oos_misroute_rate":[{"mean":0.01,"std":0.005292},{"mean":0.019333,"std":0.005033},{"mean":0.031667,"std":0.007506},{"mean":0.046,"std":0.007937},{"mean":0.061333,"std":0.006028},{"mean":0.073333,"std":0.005132},{"mean":0.089,"std":0.007937},{"mean":0.108333,"std":0.010017},{"mean":0.124667,"std":0.014012},{"mean":0.146333,"std":0.024007},{"mean":0.171,"std":0.028844},{"mean":0.197333,"std":0.038371},{"mean":0.228333,"std":0.03147},{"mean":0.263333,"std":0.029023},{"mean":0.303667,"std":0.02829},{"mean":0.354333,"std":0.031501},{"mean":0.410333,"std":0.042395},{"mean":0.495667,"std":0.055537},{"mean":0.605,"std":0.071084},{"mean":0.787,"std":0.068462}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.433667,"std":0.001155},{"mean":0.435333,"std":0.000577},{"mean":0.439,"std":0.001},{"mean":0.442667,"std":0.000577},{"mean":0.447333,"std":0.005132},{"mean":0.455,"std":0.004583},{"mean":0.462667,"std":0.00611},{"mean":0.471333,"std":0.009866},{"mean":0.481667,"std":0.015308},{"mean":0.490667,"std":0.02318},{"mean":0.501,"std":0.020785},{"mean":0.514333,"std":0.016862},{"mean":0.531,"std":0.023388},{"mean":0.555333,"std":0.025891},{"mean":0.582667,"std":0.039678},{"mean":0.626333,"std":0.047753},{"mean":0.682667,"std":0.058011},{"mean":0.788,"std":0.067735}],"llm_call_rate":[{"mean":0.961091,"std":0.005671},{"mean":0.914182,"std":0.00413},{"mean":0.871455,"std":0.001666},{"mean":0.828545,"std":0.003938},{"mean":0.784848,"std":0.004312},{"mean":0.74097,"std":0.00164},{"mean":0.693212,"std":0.004967},{"mean":0.64903,"std":0.006566},{"mean":0.599818,"std":0.00964},{"mean":0.558424,"std":0.009254},{"mean":0.513939,"std":0.008874},{"mean":0.463879,"std":0.008663},{"mean":0.417394,"std":0.003574},{"mean":0.365636,"std":0.005695},{"mean":0.317091,"std":0.002545},{"mean":0.263394,"std":0.004914},{"mean":0.211515,"std":0.00469},{"mean":0.155636,"std":0.012456},{"mean":0.096121,"std":0.017049},{"mean":0.000606,"std":0.000555}]}},"entropy":{"risk_coverage":[{"mean":0.046061,"std":0.021307},{"mean":0.044848,"std":0.00757},{"mean":0.044444,"std":0.007887},{"mean":0.051515,"std":0.004576},{"mean":0.052364,"std":0.006464},{"mean":0.057374,"std":0.010433},{"mean":0.060606,"std":0.01356},{"mean":0.065152,"std":0.014843},{"mean":0.069091,"std":0.014461},{"mean":0.073212,"std":0.015536},{"mean":0.080771,"std":0.010785},{"mean":0.086667,"std":0.006061},{"mean":0.093986,"std":0.007289},{"mean":0.101905,"std":0.007854},{"mean":0.108283,"std":0.009232},{"mean":0.119924,"std":0.011174},{"mean":0.135116,"std":0.012188},{"mean":0.154882,"std":0.009889},{"mean":0.175247,"std":0.012318},{"mean":0.185273,"std":0.015458}],"operating":{"tau":[{"mean":-1.536803,"std":0.051465},{"mean":-1.588578,"std":0.05051},{"mean":-1.632952,"std":0.044524},{"mean":-1.668676,"std":0.040086},{"mean":-1.702547,"std":0.026859},{"mean":-1.732284,"std":0.016589},{"mean":-1.752277,"std":0.012783},{"mean":-1.76757,"std":0.01066},{"mean":-1.781567,"std":0.008387},{"mean":-1.793546,"std":0.007394},{"mean":-1.803953,"std":0.005974},{"mean":-1.813857,"std":0.004598},{"mean":-1.82361,"std":0.00228},{"mean":-1.833765,"std":0.002349},{"mean":-1.844519,"std":0.003361},{"mean":-1.857598,"std":0.00525},{"mean":-1.873336,"std":0.005865},{"mean":-1.892001,"std":0.009106},{"mean":-1.912438,"std":0.005761},{"mean":-2.009396,"std":0.003096}],"coverage":[{"mean":0.039818,"std":0.005571},{"mean":0.085939,"std":0.003315},{"mean":0.128121,"std":0.001689},{"mean":0.168485,"std":0.004572},{"mean":0.212606,"std":0.007117},{"mean":0.261697,"std":0.006881},{"mean":0.305576,"std":0.006642},{"mean":0.351939,"std":0.004784},{"mean":0.395273,"std":0.004841},{"mean":0.439636,"std":0.007716},{"mean":0.482,"std":0.004892},{"mean":0.524485,"std":0.005661},{"mean":0.568667,"std":0.003745},{"mean":0.617333,"std":0.002373},{"mean":0.667758,"std":0.008447},{"mean":0.720909,"std":0.006576},{"mean":0.775939,"std":0.004721},{"mean":0.831939,"std":0.012129},{"mean":0.890121,"std":0.00714},{"mean":0.998545,"std":0.000793}],"selective_risk":[{"mean":0.046725,"std":0.021668},{"mean":0.045799,"std":0.005625},{"mean":0.044952,"std":0.005167},{"mean":0.046718,"std":0.003935},{"mean":0.051755,"std":0.005868},{"mean":0.054512,"std":0.008644},{"mean":0.057985,"std":0.011586},{"mean":0.061004,"std":0.014405},{"mean":0.064441,"std":0.015543},{"mean":0.067578,"std":0.014041},{"mean":0.07263,"std":0.01559},{"mean":0.077447,"std":0.013314},{"mean":0.082776,"std":0.008815},{"mean":0.089326,"std":0.007282},{"mean":0.095166,"std":0.005846},{"mean":0.104373,"std":0.007802},{"mean":0.113553,"std":0.011241},{"mean":0.128956,"std":0.015564},{"mean":0.150197,"std":0.01393},{"mean":0.185477,"std":0.015495}],"accuracy_8":[{"mean":0.820788,"std":0.00021},{"mean":0.820848,"std":0.000278},{"mean":0.821152,"std":0.00042},{"mean":0.823152,"std":0.002373},{"mean":0.825576,"std":0.002123},{"mean":0.829152,"std":0.00163},{"mean":0.832182,"std":0.001311},{"mean":0.834848,"std":0.000278},{"mean":0.836485,"std":0.001034},{"mean":0.840303,"std":0.001546},{"mean":0.842788,"std":0.001927},{"mean":0.846788,"std":0.002302},{"mean":0.850485,"std":0.002131},{"mean":0.853939,"std":0.003044},{"mean":0.857697,"std":0.002854},{"mean":0.857818,"std":0.003831},{"mean":0.857091,"std":0.005715},{"mean":0.851515,"std":0.008952},{"mean":0.840061,"std":0.011706},{"mean":0.814727,"std":0.015458}],"high_conf_oos_misroute_rate":[{"mean":0.010333,"std":0.005508},{"mean":0.020333,"std":0.003215},{"mean":0.029,"std":0.002},{"mean":0.037333,"std":0.004509},{"mean":0.050667,"std":0.009609},{"mean":0.061,"std":0.013748},{"mean":0.073,"std":0.0151},{"mean":0.089333,"std":0.017616},{"mean":0.106333,"std":0.025423},{"mean":0.124667,"std":0.023245},{"mean":0.148333,"std":0.024906},{"mean":0.172,"std":0.023431},{"mean":0.196333,"std":0.017926},{"mean":0.225667,"std":0.016862},{"mean":0.255333,"std":0.005774},{"mean":0.298333,"std":0.013279},{"mean":0.351667,"std":0.02797},{"mean":0.428,"std":0.056824},{"mean":0.539667,"std":0.05862},{"mean":0.788333,"std":0.067174}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.433333,"std":0.000577},{"mean":0.434333,"std":0.001155},{"mean":0.436667,"std":0.002082},{"mean":0.438667,"std":0.002082},{"mean":0.440333,"std":0.000577},{"mean":0.446,"std":0.001732},{"mean":0.450667,"std":0.003512},{"mean":0.456,"std":0.005292},{"mean":0.464,"std":0.006245},{"mean":0.473,"std":0.010392},{"mean":0.480333,"std":0.013429},{"mean":0.489,"std":0.017349},{"mean":0.499,"std":0.014799},{"mean":0.517333,"std":0.01914},{"mean":0.543333,"std":0.028024},{"mean":0.577667,"std":0.040415},{"mean":0.638,"std":0.050468},{"mean":0.788333,"std":0.067174}],"llm_call_rate":[{"mean":0.960182,"std":0.005571},{"mean":0.914061,"std":0.003315},{"mean":0.871879,"std":0.001689},{"mean":0.831515,"std":0.004572},{"mean":0.787394,"std":0.007117},{"mean":0.738303,"std":0.006881},{"mean":0.694424,"std":0.006642},{"mean":0.648061,"std":0.004784},{"mean":0.604727,"std":0.004841},{"mean":0.560364,"std":0.007716},{"mean":0.518,"std":0.004892},{"mean":0.475515,"std":0.005661},{"mean":0.431333,"std":0.003745},{"mean":0.382667,"std":0.002373},{"mean":0.332242,"std":0.008447},{"mean":0.279091,"std":0.006576},{"mean":0.224061,"std":0.004721},{"mean":0.168061,"std":0.012129},{"mean":0.109879,"std":0.00714},{"mean":0.001455,"std":0.000793}]}},"margin":{"risk_coverage":[{"mean":0.015758,"std":0.010497},{"mean":0.013939,"std":0.004199},{"mean":0.012525,"std":0.00305},{"mean":0.012424,"std":0.001892},{"mean":0.013333,"std":0.002222},{"mean":0.016162,"std":0.001525},{"mean":0.019394,"std":0.00334},{"mean":0.021061,"std":0.003472},{"mean":0.023569,"std":0.003291},{"mean":0.026788,"std":0.005446},{"mean":0.030964,"std":0.006099},{"mean":0.035859,"std":0.004737},{"mean":0.040653,"std":0.005929},{"mean":0.049351,"std":0.004595},{"mean":0.060364,"std":0.004903},{"mean":0.075909,"std":0.006707},{"mean":0.097897,"std":0.010687},{"mean":0.125185,"std":0.015545},{"mean":0.155343,"std":0.015058},{"mean":0.185273,"std":0.015458}],"operating":{"tau":[{"mean":8.182472,"std":0.276067},{"mean":7.426385,"std":0.374285},{"mean":6.919238,"std":0.330182},{"mean":6.536316,"std":0.294592},{"mean":6.21271,"std":0.252122},{"mean":5.877776,"std":0.219455},{"mean":5.567429,"std":0.177388},{"mean":5.243661,"std":0.13068},{"mean":4.925056,"std":0.114914},{"mean":4.635341,"std":0.092185},{"mean":4.339825,"std":0.077169},{"mean":4.025504,"std":0.018627},{"mean":3.730732,"std":0.030466},{"mean":3.39809,"std":0.015147},{"mean":3.073221,"std":0.031172},{"mean":2.672952,"std":0.06993},{"mean":2.19348,"std":0.062454},{"mean":1.531263,"std":0.118919},{"mean":0.729408,"std":0.114961},{"mean":0.001543,"std":0.000727}],"coverage":[{"mean":0.040606,"std":0.004604},{"mean":0.082788,"std":0.009113},{"mean":0.120848,"std":0.006587},{"mean":0.164485,"std":0.007196},{"mean":0.205758,"std":0.005739},{"mean":0.252,"std":0.005715},{"mean":0.294121,"std":0.004034},{"mean":0.337818,"std":0.004166},{"mean":0.383758,"std":0.002911},{"mean":0.423636,"std":0.00365},{"mean":0.470364,"std":0.008511},{"mean":0.515697,"std":0.009144},{"mean":0.558121,"std":0.006252},{"mean":0.601818,"std":0.004738},{"mean":0.642,"std":0.004329},{"mean":0.693576,"std":0.00701},{"mean":0.74897,"std":0.00191},{"mean":0.818,"std":0.003938},{"mean":0.903636,"std":0.007461},{"mean":0.999939,"std":0.000105}],"selective_risk":[{"mean":0.01544,"std":0.013428},{"mean":0.012708,"std":0.006676},{"mean":0.013351,"std":0.00529},{"mean":0.013198,"std":0.002299},{"mean":0.012351,"std":0.001489},{"mean":0.013439,"std":0.002265},{"mean":0.016087,"std":0.00182},{"mean":0.018867,"std":0.003576},{"mean":0.019891,"std":0.003228},{"mean":0.021723,"std":0.003993},{"mean":0.024331,"std":0.004076},{"mean":0.027826,"std":0.005864},{"mean":0.03156,"std":0.006874},{"mean":0.035766,"std":0.004444},{"mean":0.039768,"std":0.005627},{"mean":0.048153,"std":0.004539},{"mean":0.060043,"std":0.005338},{"mean":0.082834,"std":0.008486},{"mean":0.127868,"std":0.019761},{"mean":0.185222,"std":0.015545}],"accuracy_8":[{"mean":0.821939,"std":0.000105},{"mean":0.824424,"std":0.001766},{"mean":0.827091,"std":0.002799},{"mean":0.830485,"std":0.003413},{"mean":0.832061,"std":0.003546},{"mean":0.834545,"std":0.004919},{"mean":0.837212,"std":0.005108},{"mean":0.839394,"std":0.005007},{"mean":0.843758,"std":0.003993},{"mean":0.847758,"std":0.003546},{"mean":0.852182,"std":0.002446},{"mean":0.857636,"std":0.00284},{"mean":0.861515,"std":0.002169},{"mean":0.864424,"std":0.001728},{"mean":0.868061,"std":0.000735},{"mean":0.871394,"std":0.00082},{"mean":0.874121,"std":0.002612},{"mean":0.870606,"std":0.001412},{"mean":0.850545,"std":0.011076},{"mean":0.814788,"std":0.015563}],"high_conf_oos_misroute_rate":[{"mean":0.002667,"std":0.003055},{"mean":0.004667,"std":0.003055},{"mean":0.007,"std":0.002},{"mean":0.01,"std":0.001},{"mean":0.011667,"std":0.002517},{"mean":0.014667,"std":0.002082},{"mean":0.021333,"std":0.004163},{"mean":0.029667,"std":0.006429},{"mean":0.036667,"std":0.008083},{"mean":0.045,"std":0.009849},{"mean":0.055,"std":0.01249},{"mean":0.069667,"std":0.015308},{"mean":0.086,"std":0.018},{"mean":0.102333,"std":0.012897},{"mean":0.119333,"std":0.018148},{"mean":0.153333,"std":0.019399},{"mean":0.201667,"std":0.030022},{"mean":0.295667,"std":0.042194},{"mean":0.498667,"std":0.088081},{"mean":0.788,"std":0.067735}],"oos_misroute_rate":[{"mean":0.433,"std":0.001},{"mean":0.433333,"std":0.001528},{"mean":0.433333,"std":0.001528},{"mean":0.433667,"std":0.001528},{"mean":0.433667,"std":0.001528},{"mean":0.435333,"std":0.002082},{"mean":0.436333,"std":0.001528},{"mean":0.439333,"std":0.002082},{"mean":0.440667,"std":0.002517},{"mean":0.442333,"std":0.003786},{"mean":0.443333,"std":0.004726},{"mean":0.446333,"std":0.005508},{"mean":0.450667,"std":0.009018},{"mean":0.456,"std":0.007},{"mean":0.458,"std":0.009539},{"mean":0.467667,"std":0.012702},{"mean":0.480667,"std":0.020257},{"mean":0.519,"std":0.031},{"mean":0.625333,"std":0.065271},{"mean":0.788,"std":0.067735}],"llm_call_rate":[{"mean":0.959394,"std":0.004604},{"mean":0.917212,"std":0.009113},{"mean":0.879152,"std":0.006587},{"mean":0.835515,"std":0.007196},{"mean":0.794242,"std":0.005739},{"mean":0.748,"std":0.005715},{"mean":0.705879,"std":0.004034},{"mean":0.662182,"std":0.004166},{"mean":0.616242,"std":0.002911},{"mean":0.576364,"std":0.00365},{"mean":0.529636,"std":0.008511},{"mean":0.484303,"std":0.009144},{"mean":0.441879,"std":0.006252},{"mean":0.398182,"std":0.004738},{"mean":0.358,"std":0.004329},{"mean":0.306424,"std":0.00701},{"mean":0.25103,"std":0.00191},{"mean":0.182,"std":0.003938},{"mean":0.096364,"std":0.007461},{"mean":6.1e-05,"std":0.000105}]}},"msp_t":{"risk_coverage":[{"mean":0.015758,"std":0.010497},{"mean":0.012121,"std":0.004199},{"mean":0.011717,"std":0.003896},{"mean":0.012121,"std":0.001892},{"mean":0.012606,"std":0.002753},{"mean":0.015354,"std":0.003943},{"mean":0.017143,"std":0.002749},{"mean":0.01803,"std":0.003812},{"mean":0.021145,"std":0.003754},{"mean":0.02497,"std":0.004488},{"mean":0.027218,"std":0.005805},{"mean":0.032828,"std":0.006423},{"mean":0.038228,"std":0.004799},{"mean":0.046061,"std":0.00339},{"mean":0.055677,"std":0.003549},{"mean":0.07,"std":0.006575},{"mean":0.094332,"std":0.012743},{"mean":0.120875,"std":0.014648},{"mean":0.150877,"std":0.01431},{"mean":0.185273,"std":0.015458}],"operating":{"tau":[{"mean":0.999283,"std":0.000214},{"mean":0.998544,"std":0.000449},{"mean":0.997755,"std":0.000696},{"mean":0.996729,"std":0.000772},{"mean":0.995481,"std":0.001046},{"mean":0.994007,"std":0.00129},{"mean":0.991954,"std":0.001317},{"mean":0.989247,"std":0.001694},{"mean":0.985894,"std":0.001856},{"mean":0.981294,"std":0.002522},{"mean":0.975223,"std":0.002272},{"mean":0.966902,"std":0.002525},{"mean":0.956425,"std":0.002906},{"mean":0.941125,"std":0.003581},{"mean":0.920632,"std":0.003343},{"mean":0.89221,"std":0.002751},{"mean":0.841294,"std":0.004426},{"mean":0.738186,"std":0.013112},{"mean":0.553049,"std":0.015573},{"mean":0.243117,"std":0.042494}],"coverage":[{"mean":0.041333,"std":0.0029},{"mean":0.082364,"std":0.005235},{"mean":0.122242,"std":0.00861},{"mean":0.163273,"std":0.006801},{"mean":0.206,"std":0.006533},{"mean":0.248788,"std":0.008728},{"mean":0.292545,"std":0.002234},{"mean":0.338364,"std":0.002364},{"mean":0.378545,"std":0.003154},{"mean":0.424424,"std":0.002945},{"mean":0.468788,"std":0.007749},{"mean":0.514242,"std":0.007191},{"mean":0.55697,"std":0.006531},{"mean":0.599879,"std":0.002309},{"mean":0.641818,"std":0.005695},{"mean":0.687455,"std":0.003938},{"mean":0.742,"std":0.003581},{"mean":0.809212,"std":0.00105},{"mean":0.890727,"std":0.003887},{"mean":0.995758,"std":0.004622}],"selective_risk":[{"mean":0.01709,"std":0.010837},{"mean":0.013036,"std":0.00524},{"mean":0.011753,"std":0.003237},{"mean":0.011811,"std":0.003528},{"mean":0.011754,"std":0.00166},{"mean":0.012402,"std":0.002699},{"mean":0.014725,"std":0.003854},{"mean":0.016486,"std":0.003153},{"mean":0.0176,"std":0.00283},{"mean":0.019415,"std":0.002295},{"mean":0.022588,"std":0.003858},{"mean":0.025307,"std":0.005007},{"mean":0.027939,"std":0.006244},{"mean":0.032736,"std":0.006294},{"mean":0.036922,"std":0.004599},{"mean":0.04338,"std":0.003405},{"mean":0.054406,"std":0.00235},{"mean":0.074,"std":0.007718},{"mean":0.11637,"std":0.016814},{"mean":0.182255,"std":0.012882}],"accuracy_8":[{"mean":0.821455,"std":0.000481},{"mean":0.823758,"std":0.00129},{"mean":0.826788,"std":0.00168},{"mean":0.829333,"std":0.002624},{"mean":0.832121,"std":0.00371},{"mean":0.834727,"std":0.005282},{"mean":0.837576,"std":0.006225},{"mean":0.840667,"std":0.004721},{"mean":0.844,"std":0.003938},{"mean":0.849152,"std":0.004859},{"mean":0.854424,"std":0.004388},{"mean":0.85903,"std":0.003555},{"mean":0.862848,"std":0.003119},{"mean":0.865879,"std":0.002273},{"mean":0.870364,"std":0.000727},{"mean":0.87503,"std":0.002729},{"mean":0.876788,"std":0.003193},{"mean":0.874485,"std":0.002068},{"mean":0.859212,"std":0.010447},{"mean":0.817879,"std":0.012639}],"high_conf_oos_misroute_rate":[{"mean":0.003,"std":0.002646},{"mean":0.004667,"std":0.002887},{"mean":0.006333,"std":0.002309},{"mean":0.008667,"std":0.002082},{"mean":0.011,"std":0.002},{"mean":0.013,"std":0.001732},{"mean":0.019333,"std":0.007767},{"mean":0.026,"std":0.005292},{"mean":0.031667,"std":0.007234},{"mean":0.04,"std":0.005292},{"mean":0.050667,"std":0.009504},{"mean":0.062,"std":0.013077},{"mean":0.074667,"std":0.017039},{"mean":0.091667,"std":0.018583},{"mean":0.107667,"std":0.016197},{"mean":0.134333,"std":0.019399},{"mean":0.178333,"std":0.016773},{"mean":0.253,"std":0.035679},{"mean":0.424333,"std":0.065912},{"mean":0.77,"std":0.049487}],"oos_misroute_rate":[{"mean":0.433,"std":0.001},{"mean":0.433333,"std":0.001528},{"mean":0.433333,"std":0.001528},{"mean":0.433333,"std":0.001528},{"mean":0.433667,"std":0.001528},{"mean":0.434,"std":0.001732},{"mean":0.436333,"std":0.000577},{"mean":0.438,"std":0.001732},{"mean":0.439333,"std":0.002082},{"mean":0.440333,"std":0.002082},{"mean":0.443,"std":0.004359},{"mean":0.445,"std":0.005196},{"mean":0.448,"std":0.006928},{"mean":0.453333,"std":0.009609},{"mean":0.455667,"std":0.009452},{"mean":0.460333,"std":0.01097},{"mean":0.472,"std":0.012767},{"mean":0.502667,"std":0.025166},{"mean":0.576667,"std":0.050954},{"mean":0.773,"std":0.051391}],"llm_call_rate":[{"mean":0.958667,"std":0.0029},{"mean":0.917636,"std":0.005235},{"mean":0.877758,"std":0.00861},{"mean":0.836727,"std":0.006801},{"mean":0.794,"std":0.006533},{"mean":0.751212,"std":0.008728},{"mean":0.707455,"std":0.002234},{"mean":0.661636,"std":0.002364},{"mean":0.621455,"std":0.003154},{"mean":0.575576,"std":0.002945},{"mean":0.531212,"std":0.007749},{"mean":0.485758,"std":0.007191},{"mean":0.44303,"std":0.006531},{"mean":0.400121,"std":0.002309},{"mean":0.358182,"std":0.005695},{"mean":0.312545,"std":0.003938},{"mean":0.258,"std":0.003581},{"mean":0.190788,"std":0.00105},{"mean":0.109273,"std":0.003887},{"mean":0.004242,"std":0.004622}]}}}}},"bert/k100":{"model":"bert","k":100,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["bert-base-uncased-full-seed42","bert-base-uncased-full-seed43","bert-base-uncased-full-seed44"],"logits_sha256":["673a4c11200dd61ddd6310f174ab9fb27580cf5b512c7ba3b5db19088958bab2","b6963da29f5886b915d2c5626391721fbd18c7f54a68b7720bf2d10d6cc2c58f","0324b20d4736ca9ad913b97269100b398475ab06a2bd130f653af55de239c277"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.001212,"std":0.00105},{"mean":0.001212,"std":0.0},{"mean":0.001818,"std":0.000909},{"mean":0.001697,"std":0.00042},{"mean":0.00202,"std":0.0007},{"mean":0.002597,"std":0.0},{"mean":0.002879,"std":0.000694},{"mean":0.003636,"std":0.000404},{"mean":0.003636,"std":0.00063},{"mean":0.004077,"std":0.000688},{"mean":0.004747,"std":0.000175},{"mean":0.005221,"std":0.000427},{"mean":0.006494,"std":0.00026},{"mean":0.008566,"std":0.00037},{"mean":0.013485,"std":0.001292},{"mean":0.022888,"std":0.00098},{"mean":0.03596,"std":0.001726},{"mean":0.057097,"std":0.003094},{"mean":0.087939,"std":0.003274}],"operating":{"tau":[{"mean":0.991713,"std":0.000278},{"mean":0.991243,"std":0.000297},{"mean":0.990968,"std":0.000295},{"mean":0.990639,"std":0.000262},{"mean":0.990299,"std":0.000214},{"mean":0.989943,"std":0.000103},{"mean":0.989596,"std":5e-05},{"mean":0.989251,"std":7.1e-05},{"mean":0.988881,"std":0.000105},{"mean":0.988433,"std":0.000133},{"mean":0.987978,"std":0.000207},{"mean":0.987462,"std":0.000268},{"mean":0.98681,"std":0.00024},{"mean":0.985947,"std":8.5e-05},{"mean":0.984868,"std":0.000365},{"mean":0.982944,"std":0.000549},{"mean":0.979362,"std":0.000782},{"mean":0.966343,"std":0.000176},{"mean":0.859793,"std":0.006106},{"mean":0.102443,"std":0.027654}],"coverage":[{"mean":0.064303,"std":0.001766},{"mean":0.108848,"std":0.00183},{"mean":0.148182,"std":0.005282},{"mean":0.19503,"std":0.008893},{"mean":0.239515,"std":0.003888},{"mean":0.280667,"std":0.002345},{"mean":0.324485,"std":0.004543},{"mean":0.366303,"std":0.007624},{"mean":0.407091,"std":0.010102},{"mean":0.45,"std":0.005747},{"mean":0.489333,"std":0.005498},{"mean":0.530242,"std":0.009189},{"mean":0.573576,"std":0.007897},{"mean":0.618303,"std":0.004672},{"mean":0.658364,"std":0.003757},{"mean":0.708303,"std":0.001961},{"mean":0.757394,"std":0.004547},{"mean":0.814121,"std":0.00348},{"mean":0.882424,"std":0.002442},{"mean":0.999273,"std":0.000656}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.001115,"std":0.000966},{"mean":0.001228,"std":4.4e-05},{"mean":0.001865,"std":0.000954},{"mean":0.001776,"std":0.00047},{"mean":0.001726,"std":0.000365},{"mean":0.002245,"std":0.000578},{"mean":0.002483,"std":5.1e-05},{"mean":0.002834,"std":0.000704},{"mean":0.003639,"std":0.000429},{"mean":0.003596,"std":0.000604},{"mean":0.003893,"std":0.001023},{"mean":0.004226,"std":0.000144},{"mean":0.004705,"std":3.5e-05},{"mean":0.005341,"std":0.000347},{"mean":0.007101,"std":0.000515},{"mean":0.0092,"std":0.000686},{"mean":0.015408,"std":0.000924},{"mean":0.030836,"std":0.001797},{"mean":0.087337,"std":0.003167}],"accuracy_8":[{"mean":0.826182,"std":0.001012},{"mean":0.832485,"std":0.002442},{"mean":0.837394,"std":0.00447},{"mean":0.842303,"std":0.006385},{"mean":0.84697,"std":0.006225},{"mean":0.852061,"std":0.00611},{"mean":0.856545,"std":0.004727},{"mean":0.861576,"std":0.00498},{"mean":0.866909,"std":0.004545},{"mean":0.871636,"std":0.003731},{"mean":0.877333,"std":0.005261},{"mean":0.882242,"std":0.006383},{"mean":0.887818,"std":0.005235},{"mean":0.89297,"std":0.003093},{"mean":0.897091,"std":0.003026},{"mean":0.904121,"std":0.001525},{"mean":0.911576,"std":0.000933},{"mean":0.916727,"std":0.002025},{"mean":0.922182,"std":0.002378},{"mean":0.912485,"std":0.003203}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001667,"std":0.001155},{"mean":0.002,"std":0.001},{"mean":0.002333,"std":0.000577},{"mean":0.003667,"std":0.001155},{"mean":0.004667,"std":0.000577},{"mean":0.005667,"std":0.001155},{"mean":0.008,"std":0.001},{"mean":0.008333,"std":0.001155},{"mean":0.009667,"std":0.002517},{"mean":0.011333,"std":0.001155},{"mean":0.013333,"std":0.001155},{"mean":0.016667,"std":0.001155},{"mean":0.024,"std":0.001},{"mean":0.032333,"std":0.001528},{"mean":0.061333,"std":0.005033},{"mean":0.129667,"std":0.005132},{"mean":0.427333,"std":0.011719}],"oos_misroute_rate":[{"mean":0.410333,"std":0.001155},{"mean":0.409333,"std":0.000577},{"mean":0.407333,"std":0.001528},{"mean":0.406667,"std":0.002082},{"mean":0.404667,"std":0.001155},{"mean":0.403,"std":0.001732},{"mean":0.402333,"std":0.000577},{"mean":0.402,"std":0.0},{"mean":0.402333,"std":0.001528},{"mean":0.401,"std":0.001},{"mean":0.400667,"std":0.001528},{"mean":0.398667,"std":0.003215},{"mean":0.399,"std":0.002},{"mean":0.398,"std":0.0},{"mean":0.395667,"std":0.001528},{"mean":0.394333,"std":0.003786},{"mean":0.390667,"std":0.004619},{"mean":0.385,"std":0.009539},{"mean":0.372333,"std":0.008386},{"mean":0.428667,"std":0.012342}],"llm_call_rate":[{"mean":0.935697,"std":0.001766},{"mean":0.891152,"std":0.00183},{"mean":0.851818,"std":0.005282},{"mean":0.80497,"std":0.008893},{"mean":0.760485,"std":0.003888},{"mean":0.719333,"std":0.002345},{"mean":0.675515,"std":0.004543},{"mean":0.633697,"std":0.007624},{"mean":0.592909,"std":0.010102},{"mean":0.55,"std":0.005747},{"mean":0.510667,"std":0.005498},{"mean":0.469758,"std":0.009189},{"mean":0.426424,"std":0.007897},{"mean":0.381697,"std":0.004672},{"mean":0.341636,"std":0.003757},{"mean":0.291697,"std":0.001961},{"mean":0.242606,"std":0.004547},{"mean":0.185879,"std":0.00348},{"mean":0.117576,"std":0.002442},{"mean":0.000727,"std":0.000656}]}},"entropy":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.00105},{"mean":0.001212,"std":0.0},{"mean":0.001818,"std":0.000909},{"mean":0.001697,"std":0.00042},{"mean":0.00202,"std":0.0007},{"mean":0.002424,"std":0.0003},{"mean":0.002576,"std":0.000525},{"mean":0.003636,"std":0.0007},{"mean":0.003636,"std":0.00063},{"mean":0.003857,"std":0.001063},{"mean":0.004545,"std":0.0},{"mean":0.005128,"std":0.000323},{"mean":0.006061,"std":0.0003},{"mean":0.008162,"std":0.000779},{"mean":0.013561,"std":0.001025},{"mean":0.021961,"std":0.001253},{"mean":0.034411,"std":0.002064},{"mean":0.057033,"std":0.003445},{"mean":0.087939,"std":0.003274}],"operating":{"tau":[{"mean":-0.083951,"std":0.002177},{"mean":-0.087711,"std":0.002352},{"mean":-0.09017,"std":0.00244},{"mean":-0.092934,"std":0.002108},{"mean":-0.095593,"std":0.00181},{"mean":-0.098665,"std":0.000825},{"mean":-0.101509,"std":0.000233},{"mean":-0.104162,"std":0.000376},{"mean":-0.107057,"std":0.00072},{"mean":-0.110225,"std":0.001131},{"mean":-0.113855,"std":0.001433},{"mean":-0.118303,"std":0.002337},{"mean":-0.123079,"std":0.001908},{"mean":-0.12919,"std":0.000538},{"mean":-0.137225,"std":0.001907},{"mean":-0.150781,"std":0.005177},{"mean":-0.176963,"std":0.00277},{"mean":-0.263588,"std":0.006007},{"mean":-0.758304,"std":0.031232},{"mean":-4.073649,"std":0.186905}],"coverage":[{"mean":0.065333,"std":0.002027},{"mean":0.107515,"std":0.001718},{"mean":0.148121,"std":0.004455},{"mean":0.194,"std":0.007716},{"mean":0.240727,"std":0.006502},{"mean":0.283818,"std":0.001753},{"mean":0.326848,"std":0.004964},{"mean":0.367333,"std":0.007219},{"mean":0.408182,"std":0.008649},{"mean":0.449879,"std":0.009275},{"mean":0.488909,"std":0.006},{"mean":0.533939,"std":0.009886},{"mean":0.575212,"std":0.008863},{"mean":0.616242,"std":0.007187},{"mean":0.658121,"std":0.005567},{"mean":0.704364,"std":0.002212},{"mean":0.757939,"std":0.003679},{"mean":0.813697,"std":0.005459},{"mean":0.878788,"std":0.003364},{"mean":0.999273,"std":0.000945}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.000554,"std":0.000959},{"mean":0.001228,"std":3.7e-05},{"mean":0.001876,"std":0.000961},{"mean":0.00177,"std":0.000481},{"mean":0.001496,"std":0.00038},{"mean":0.002225,"std":3.4e-05},{"mean":0.002475,"std":4.8e-05},{"mean":0.002527,"std":0.000531},{"mean":0.003641,"std":0.000443},{"mean":0.003724,"std":0.00068},{"mean":0.003743,"std":0.000996},{"mean":0.004213,"std":0.000125},{"mean":0.004723,"std":0.00035},{"mean":0.005158,"std":0.000202},{"mean":0.006625,"std":0.000522},{"mean":0.008395,"std":0.000856},{"mean":0.014967,"std":0.001081},{"mean":0.028961,"std":0.002419},{"mean":0.087398,"std":0.002936}],"accuracy_8":[{"mean":0.826182,"std":0.000833},{"mean":0.832,"std":0.001924},{"mean":0.837636,"std":0.004703},{"mean":0.842303,"std":0.006531},{"mean":0.847636,"std":0.007098},{"mean":0.852545,"std":0.005953},{"mean":0.857212,"std":0.005504},{"mean":0.861394,"std":0.00548},{"mean":0.866667,"std":0.005459},{"mean":0.871455,"std":0.004367},{"mean":0.87703,"std":0.005965},{"mean":0.882485,"std":0.007368},{"mean":0.887455,"std":0.005883},{"mean":0.892121,"std":0.002956},{"mean":0.89697,"std":0.003364},{"mean":0.903273,"std":0.001488},{"mean":0.911818,"std":0.001012},{"mean":0.916364,"std":0.001924},{"mean":0.921697,"std":0.002273},{"mean":0.912242,"std":0.003259}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001667,"std":0.001155},{"mean":0.002,"std":0.001},{"mean":0.002,"std":0.001},{"mean":0.003667,"std":0.000577},{"mean":0.004667,"std":0.000577},{"mean":0.005333,"std":0.000577},{"mean":0.007667,"std":0.001155},{"mean":0.008667,"std":0.001528},{"mean":0.009333,"std":0.002517},{"mean":0.011,"std":0.001},{"mean":0.013333,"std":0.001528},{"mean":0.015667,"std":0.001155},{"mean":0.022667,"std":0.001155},{"mean":0.029667,"std":0.001155},{"mean":0.059333,"std":0.005686},{"mean":0.120667,"std":0.008505},{"mean":0.427333,"std":0.009866}],"oos_misroute_rate":[{"mean":0.409333,"std":0.000577},{"mean":0.409,"std":0.0},{"mean":0.407,"std":0.002},{"mean":0.406333,"std":0.001155},{"mean":0.404333,"std":0.001528},{"mean":0.403,"std":0.001732},{"mean":0.402667,"std":0.000577},{"mean":0.402,"std":0.0},{"mean":0.402667,"std":0.001155},{"mean":0.401333,"std":0.002082},{"mean":0.4,"std":0.002646},{"mean":0.398667,"std":0.002517},{"mean":0.398667,"std":0.002517},{"mean":0.397667,"std":0.000577},{"mean":0.397,"std":0.0},{"mean":0.395667,"std":0.003215},{"mean":0.391333,"std":0.003215},{"mean":0.387333,"std":0.007506},{"mean":0.375,"std":0.00781},{"mean":0.429667,"std":0.012342}],"llm_call_rate":[{"mean":0.934667,"std":0.002027},{"mean":0.892485,"std":0.001718},{"mean":0.851879,"std":0.004455},{"mean":0.806,"std":0.007716},{"mean":0.759273,"std":0.006502},{"mean":0.716182,"std":0.001753},{"mean":0.673152,"std":0.004964},{"mean":0.632667,"std":0.007219},{"mean":0.591818,"std":0.008649},{"mean":0.550121,"std":0.009275},{"mean":0.511091,"std":0.006},{"mean":0.466061,"std":0.009886},{"mean":0.424788,"std":0.008863},{"mean":0.383758,"std":0.007187},{"mean":0.341879,"std":0.005567},{"mean":0.295636,"std":0.002212},{"mean":0.242061,"std":0.003679},{"mean":0.186303,"std":0.005459},{"mean":0.121212,"std":0.003364},{"mean":0.000727,"std":0.000945}]}},"margin":{"risk_coverage":[{"mean":0.002424,"std":0.002099},{"mean":0.003636,"std":0.003636},{"mean":0.003636,"std":0.002424},{"mean":0.004545,"std":0.001818},{"mean":0.005333,"std":0.00183},{"mean":0.004646,"std":0.001852},{"mean":0.004502,"std":0.001587},{"mean":0.005303,"std":0.001461},{"mean":0.006465,"std":0.002099},{"mean":0.006788,"std":0.001866},{"mean":0.006832,"std":0.001491},{"mean":0.007677,"std":0.0007},{"mean":0.008485,"std":0.000427},{"mean":0.01013,"std":0.000519},{"mean":0.012848,"std":0.0},{"mean":0.016667,"std":0.000473},{"mean":0.0241,"std":0.000123},{"mean":0.03899,"std":0.001927},{"mean":0.060223,"std":0.003004},{"mean":0.087939,"std":0.003274}],"operating":{"tau":[{"mean":7.601737,"std":0.086342},{"mean":7.422502,"std":0.070606},{"mean":7.264958,"std":0.05812},{"mean":7.139894,"std":0.058318},{"mean":7.047226,"std":0.054644},{"mean":6.957284,"std":0.058695},{"mean":6.859562,"std":0.062263},{"mean":6.768629,"std":0.059734},{"mean":6.675156,"std":0.067308},{"mean":6.589373,"std":0.063236},{"mean":6.493468,"std":0.061872},{"mean":6.390061,"std":0.060123},{"mean":6.27528,"std":0.049447},{"mean":6.131477,"std":0.035807},{"mean":5.947507,"std":0.037359},{"mean":5.705798,"std":0.057075},{"mean":5.334054,"std":0.061738},{"mean":4.594975,"std":0.03476},{"mean":2.728291,"std":0.075694},{"mean":0.00821,"std":0.011253}],"coverage":[{"mean":0.052364,"std":0.001585},{"mean":0.093333,"std":0.004034},{"mean":0.143091,"std":0.003273},{"mean":0.192667,"std":0.008608},{"mean":0.232727,"std":0.009609},{"mean":0.273152,"std":0.008406},{"mean":0.317455,"std":0.00481},{"mean":0.361697,"std":0.003374},{"mean":0.407879,"std":0.003109},{"mean":0.448667,"std":0.008199},{"mean":0.494242,"std":0.005085},{"mean":0.538727,"std":0.003217},{"mean":0.582121,"std":0.000555},{"mean":0.624606,"std":0.001944},{"mean":0.674061,"std":0.001365},{"mean":0.719091,"std":0.003464},{"mean":0.767091,"std":0.003367},{"mean":0.821939,"std":0.003234},{"mean":0.890303,"std":0.004067},{"mean":0.999455,"std":0.000481}],"selective_risk":[{"mean":0.003529,"std":0.003521},{"mean":0.003313,"std":0.003011},{"mean":0.003833,"std":0.002617},{"mean":0.004443,"std":0.001611},{"mean":0.005264,"std":0.002613},{"mean":0.004917,"std":0.00184},{"mean":0.004788,"std":0.00152},{"mean":0.004517,"std":0.001776},{"mean":0.005357,"std":0.001578},{"mean":0.006482,"std":0.002126},{"mean":0.006736,"std":0.001801},{"mean":0.006856,"std":0.00166},{"mean":0.007392,"std":0.000907},{"mean":0.008151,"std":0.000518},{"mean":0.009081,"std":0.000947},{"mean":0.011292,"std":0.000473},{"mean":0.013905,"std":0.000277},{"mean":0.01917,"std":0.000659},{"mean":0.035193,"std":0.000267},{"mean":0.087623,"std":0.003243}],"accuracy_8":[{"mean":0.825455,"std":0.001106},{"mean":0.828545,"std":0.000962},{"mean":0.834242,"std":0.002131},{"mean":0.839879,"std":0.002421},{"mean":0.845636,"std":0.00354},{"mean":0.850606,"std":0.003203},{"mean":0.858303,"std":0.003077},{"mean":0.865273,"std":0.001273},{"mean":0.871697,"std":0.002169},{"mean":0.876182,"std":0.003318},{"mean":0.880545,"std":0.002751},{"mean":0.885576,"std":0.001568},{"mean":0.891333,"std":0.001589},{"mean":0.895636,"std":0.000962},{"mean":0.901091,"std":0.001373},{"mean":0.905576,"std":0.001944},{"mean":0.912424,"std":0.003789},{"mean":0.918848,"std":0.002939},{"mean":0.922727,"std":0.001753},{"mean":0.912182,"std":0.003318}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.001},{"mean":0.001667,"std":0.001528},{"mean":0.002667,"std":0.001528},{"mean":0.004333,"std":0.001155},{"mean":0.005667,"std":0.002517},{"mean":0.006333,"std":0.002082},{"mean":0.007333,"std":0.002082},{"mean":0.007667,"std":0.002517},{"mean":0.010667,"std":0.003512},{"mean":0.013667,"std":0.004509},{"mean":0.016,"std":0.004583},{"mean":0.017667,"std":0.005033},{"mean":0.020333,"std":0.004041},{"mean":0.024667,"std":0.002517},{"mean":0.029333,"std":0.003786},{"mean":0.039,"std":0.001},{"mean":0.051,"std":0.002},{"mean":0.075667,"std":0.004041},{"mean":0.15,"std":0.00781},{"mean":0.428667,"std":0.012097}],"oos_misroute_rate":[{"mean":0.42,"std":0.003464},{"mean":0.414333,"std":0.004163},{"mean":0.409333,"std":0.002517},{"mean":0.406667,"std":0.001528},{"mean":0.404667,"std":0.000577},{"mean":0.403667,"std":0.001528},{"mean":0.402667,"std":0.001528},{"mean":0.399667,"std":0.002517},{"mean":0.397667,"std":0.000577},{"mean":0.397667,"std":0.000577},{"mean":0.395667,"std":0.002517},{"mean":0.393333,"std":0.005508},{"mean":0.390667,"std":0.005132},{"mean":0.389333,"std":0.004041},{"mean":0.39,"std":0.004583},{"mean":0.387667,"std":0.006028},{"mean":0.380333,"std":0.011372},{"mean":0.373667,"std":0.01159},{"mean":0.368667,"std":0.004163},{"mean":0.43,"std":0.012767}],"llm_call_rate":[{"mean":0.947636,"std":0.001585},{"mean":0.906667,"std":0.004034},{"mean":0.856909,"std":0.003273},{"mean":0.807333,"std":0.008608},{"mean":0.767273,"std":0.009609},{"mean":0.726848,"std":0.008406},{"mean":0.682545,"std":0.00481},{"mean":0.638303,"std":0.003374},{"mean":0.592121,"std":0.003109},{"mean":0.551333,"std":0.008199},{"mean":0.505758,"std":0.005085},{"mean":0.461273,"std":0.003217},{"mean":0.417879,"std":0.000555},{"mean":0.375394,"std":0.001944},{"mean":0.325939,"std":0.001365},{"mean":0.280909,"std":0.003464},{"mean":0.232909,"std":0.003367},{"mean":0.178061,"std":0.003234},{"mean":0.109697,"std":0.004067},{"mean":0.000545,"std":0.000481}]}},"msp_t":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.001212,"std":0.00105},{"mean":0.001212,"std":0.0},{"mean":0.001818,"std":0.000909},{"mean":0.001697,"std":0.00042},{"mean":0.00202,"std":0.0007},{"mean":0.002597,"std":0.0},{"mean":0.002879,"std":0.000694},{"mean":0.003636,"std":0.000404},{"mean":0.003636,"std":0.00063},{"mean":0.003967,"std":0.000875},{"mean":0.004747,"std":0.000175},{"mean":0.005221,"std":0.000427},{"mean":0.006494,"std":0.00026},{"mean":0.008485,"std":0.000485},{"mean":0.013485,"std":0.001292},{"mean":0.022745,"std":0.000891},{"mean":0.035892,"std":0.001822},{"mean":0.057097,"std":0.003094},{"mean":0.087939,"std":0.003274}],"operating":{"tau":[{"mean":0.99131,"std":0.000609},{"mean":0.990819,"std":0.000685},{"mean":0.990532,"std":0.000706},{"mean":0.990195,"std":0.000689},{"mean":0.989841,"std":0.000661},{"mean":0.989474,"std":0.000583},{"mean":0.989114,"std":0.000536},{"mean":0.98876,"std":0.000567},{"mean":0.988373,"std":0.000622},{"mean":0.987915,"std":0.000662},{"mean":0.987441,"std":0.000752},{"mean":0.986898,"std":0.000844},{"mean":0.986224,"std":0.000825},{"mean":0.985345,"std":0.000598},{"mean":0.984235,"std":0.000486},{"mean":0.98227,"std":0.000664},{"mean":0.978533,"std":0.001086},{"mean":0.965147,"std":0.001372},{"mean":0.856874,"std":0.005127},{"mean":0.101626,"std":0.02803}],"coverage":[{"mean":0.064182,"std":0.001967},{"mean":0.108667,"std":0.00191},{"mean":0.148061,"std":0.005},{"mean":0.194606,"std":0.007986},{"mean":0.24,"std":0.004287},{"mean":0.280424,"std":0.002302},{"mean":0.324727,"std":0.003933},{"mean":0.366,"std":0.006938},{"mean":0.407152,"std":0.009933},{"mean":0.449636,"std":0.005715},{"mean":0.489394,"std":0.005575},{"mean":0.530364,"std":0.009294},{"mean":0.573697,"std":0.008001},{"mean":0.618121,"std":0.004384},{"mean":0.658242,"std":0.003909},{"mean":0.707455,"std":0.001585},{"mean":0.757697,"std":0.005053},{"mean":0.814182,"std":0.003455},{"mean":0.882485,"std":0.002508},{"mean":0.999273,"std":0.000656}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.001116,"std":0.000967},{"mean":0.001229,"std":4.2e-05},{"mean":0.001869,"std":0.000956},{"mean":0.001773,"std":0.000472},{"mean":0.001513,"std":0.000377},{"mean":0.002243,"std":0.000576},{"mean":0.002484,"std":4.7e-05},{"mean":0.002834,"std":0.000705},{"mean":0.003642,"std":0.000433},{"mean":0.003596,"std":0.000605},{"mean":0.003892,"std":0.001023},{"mean":0.004225,"std":0.000141},{"mean":0.004707,"std":3.3e-05},{"mean":0.00525,"std":0.000507},{"mean":0.007024,"std":0.000631},{"mean":0.009276,"std":0.000717},{"mean":0.015407,"std":0.000923},{"mean":0.030833,"std":0.001798},{"mean":0.087337,"std":0.003167}],"accuracy_8":[{"mean":0.826182,"std":0.001012},{"mean":0.832364,"std":0.002378},{"mean":0.837394,"std":0.00447},{"mean":0.842242,"std":0.006299},{"mean":0.84703,"std":0.00631},{"mean":0.852,"std":0.006131},{"mean":0.856545,"std":0.004727},{"mean":0.861697,"std":0.005144},{"mean":0.86697,"std":0.004637},{"mean":0.871697,"std":0.003824},{"mean":0.877273,"std":0.005216},{"mean":0.882242,"std":0.006383},{"mean":0.887758,"std":0.005173},{"mean":0.892848,"std":0.002917},{"mean":0.89703,"std":0.002962},{"mean":0.904,"std":0.00142},{"mean":0.911576,"std":0.000933},{"mean":0.916727,"std":0.002025},{"mean":0.922182,"std":0.002378},{"mean":0.912485,"std":0.003203}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001667,"std":0.001155},{"mean":0.002,"std":0.001},{"mean":0.002,"std":0.001},{"mean":0.003667,"std":0.001155},{"mean":0.004667,"std":0.000577},{"mean":0.005667,"std":0.001155},{"mean":0.008,"std":0.001},{"mean":0.008333,"std":0.001155},{"mean":0.009667,"std":0.002517},{"mean":0.011333,"std":0.001155},{"mean":0.013333,"std":0.001155},{"mean":0.016333,"std":0.001528},{"mean":0.023667,"std":0.001155},{"mean":0.032667,"std":0.001155},{"mean":0.061333,"std":0.005033},{"mean":0.129667,"std":0.005132},{"mean":0.427333,"std":0.011719}],"oos_misroute_rate":[{"mean":0.410333,"std":0.001155},{"mean":0.409333,"std":0.000577},{"mean":0.407333,"std":0.001528},{"mean":0.406667,"std":0.002082},{"mean":0.404667,"std":0.001155},{"mean":0.403,"std":0.001732},{"mean":0.402333,"std":0.000577},{"mean":0.402,"std":0.0},{"mean":0.402333,"std":0.001528},{"mean":0.401,"std":0.001},{"mean":0.400667,"std":0.001528},{"mean":0.398667,"std":0.003215},{"mean":0.399,"std":0.002},{"mean":0.398,"std":0.0},{"mean":0.395667,"std":0.001528},{"mean":0.394333,"std":0.003786},{"mean":0.391,"std":0.005196},{"mean":0.385,"std":0.009539},{"mean":0.372333,"std":0.008386},{"mean":0.428667,"std":0.012342}],"llm_call_rate":[{"mean":0.935818,"std":0.001967},{"mean":0.891333,"std":0.00191},{"mean":0.851939,"std":0.005},{"mean":0.805394,"std":0.007986},{"mean":0.76,"std":0.004287},{"mean":0.719576,"std":0.002302},{"mean":0.675273,"std":0.003933},{"mean":0.634,"std":0.006938},{"mean":0.592848,"std":0.009933},{"mean":0.550364,"std":0.005715},{"mean":0.510606,"std":0.005575},{"mean":0.469636,"std":0.009294},{"mean":0.426303,"std":0.008001},{"mean":0.381879,"std":0.004384},{"mean":0.341758,"std":0.003909},{"mean":0.292545,"std":0.001585},{"mean":0.242303,"std":0.005053},{"mean":0.185818,"std":0.003455},{"mean":0.117515,"std":0.002508},{"mean":0.000727,"std":0.000656}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.004848,"std":0.002099},{"mean":0.005455,"std":0.001818},{"mean":0.00404,"std":0.0007},{"mean":0.00303,"std":0.000525},{"mean":0.003152,"std":0.001111},{"mean":0.002828,"std":0.001262},{"mean":0.002944,"std":0.000794},{"mean":0.002879,"std":0.000946},{"mean":0.003098,"std":0.000467},{"mean":0.003152,"std":0.000757},{"mean":0.003747,"std":0.000763},{"mean":0.00404,"std":0.000463},{"mean":0.004662,"std":0.000427},{"mean":0.005541,"std":0.000794},{"mean":0.008646,"std":0.001093},{"mean":0.0125,"std":0.000601},{"mean":0.020392,"std":0.001011},{"mean":0.034815,"std":0.000649},{"mean":0.060159,"std":0.003752},{"mean":0.091879,"std":0.003354}],"operating":{"tau":[{"mean":0.995125,"std":0.000181},{"mean":0.994647,"std":0.000111},{"mean":0.994271,"std":7.5e-05},{"mean":0.994004,"std":7.6e-05},{"mean":0.993778,"std":6.5e-05},{"mean":0.993547,"std":7.7e-05},{"mean":0.993296,"std":5.4e-05},{"mean":0.993036,"std":2.6e-05},{"mean":0.992777,"std":3.8e-05},{"mean":0.992508,"std":3.3e-05},{"mean":0.992239,"std":3.8e-05},{"mean":0.99197,"std":3.7e-05},{"mean":0.991627,"std":3.2e-05},{"mean":0.991219,"std":8.9e-05},{"mean":0.99066,"std":0.000157},{"mean":0.989722,"std":0.000269},{"mean":0.987529,"std":0.000762},{"mean":0.983115,"std":0.000502},{"mean":0.95741,"std":0.001992},{"mean":0.216662,"std":0.002193}],"coverage":[{"mean":0.047212,"std":0.002345},{"mean":0.09103,"std":0.001211},{"mean":0.136242,"std":0.007089},{"mean":0.182182,"std":0.007948},{"mean":0.223091,"std":0.006052},{"mean":0.265455,"std":0.003577},{"mean":0.311697,"std":0.003833},{"mean":0.353333,"std":0.000378},{"mean":0.396485,"std":0.00444},{"mean":0.438788,"std":0.004308},{"mean":0.478606,"std":0.006632},{"mean":0.520303,"std":0.006015},{"mean":0.561758,"std":0.008625},{"mean":0.603939,"std":0.006647},{"mean":0.647333,"std":0.007846},{"mean":0.693212,"std":0.001389},{"mean":0.746667,"std":0.003234},{"mean":0.801515,"std":0.001458},{"mean":0.86303,"std":0.003259},{"mean":0.999697,"std":0.00021}],"selective_risk":[{"mean":0.005074,"std":0.001928},{"mean":0.005987,"std":0.001989},{"mean":0.004463,"std":0.000858},{"mean":0.003324,"std":0.000528},{"mean":0.002993,"std":0.000505},{"mean":0.002961,"std":0.001009},{"mean":0.002913,"std":0.000988},{"mean":0.002916,"std":0.000789},{"mean":0.002912,"std":0.000983},{"mean":0.00318,"std":0.000513},{"mean":0.003172,"std":0.000627},{"mean":0.00315,"std":0.000645},{"mean":0.00389,"std":0.000698},{"mean":0.004315,"std":0.000618},{"mean":0.004581,"std":0.001093},{"mean":0.00542,"std":0.000792},{"mean":0.008357,"std":0.001164},{"mean":0.012702,"std":0.000766},{"mean":0.023032,"std":0.001184},{"mean":0.091664,"std":0.003564}],"accuracy_8":[{"mean":0.821515,"std":0.000378},{"mean":0.822485,"std":0.000278},{"mean":0.825152,"std":0.00134},{"mean":0.829273,"std":0.000962},{"mean":0.832485,"std":0.002273},{"mean":0.835697,"std":0.003028},{"mean":0.84097,"std":0.003384},{"mean":0.845091,"std":0.003091},{"mean":0.849091,"std":0.003988},{"mean":0.854,"std":0.004229},{"mean":0.859212,"std":0.004818},{"mean":0.864485,"std":0.004261},{"mean":0.869333,"std":0.004672},{"mean":0.876727,"std":0.00448},{"mean":0.884303,"std":0.006153},{"mean":0.893515,"std":0.003909},{"mean":0.903636,"std":0.002378},{"mean":0.913091,"std":0.000364},{"mean":0.919333,"std":0.00183},{"mean":0.908182,"std":0.003406}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.000577},{"mean":0.003,"std":0.001},{"mean":0.003333,"std":0.000577},{"mean":0.003333,"std":0.000577},{"mean":0.003667,"std":0.000577},{"mean":0.004333,"std":0.001528},{"mean":0.004667,"std":0.002082},{"mean":0.005,"std":0.002},{"mean":0.005333,"std":0.002082},{"mean":0.006667,"std":0.000577},{"mean":0.007333,"std":0.001155},{"mean":0.008,"std":0.001},{"mean":0.011,"std":0.001},{"mean":0.012667,"std":0.002082},{"mean":0.014333,"std":0.003512},{"mean":0.018,"std":0.002646},{"mean":0.03,"std":0.003464},{"mean":0.05,"std":0.003},{"mean":0.097667,"std":0.004509},{"mean":0.451333,"std":0.015503}],"oos_misroute_rate":[{"mean":0.428333,"std":0.002309},{"mean":0.425667,"std":0.001155},{"mean":0.422667,"std":0.000577},{"mean":0.422333,"std":0.000577},{"mean":0.422,"std":0.0},{"mean":0.421,"std":0.001},{"mean":0.418667,"std":0.002082},{"mean":0.417333,"std":0.002517},{"mean":0.416333,"std":0.003512},{"mean":0.415,"std":0.002646},{"mean":0.414333,"std":0.001528},{"mean":0.413,"std":0.002},{"mean":0.412,"std":0.001732},{"mean":0.409333,"std":0.000577},{"mean":0.407667,"std":0.001155},{"mean":0.405333,"std":0.001528},{"mean":0.405333,"std":0.001528},{"mean":0.399667,"std":0.004163},{"mean":0.386,"std":0.008185},{"mean":0.452333,"std":0.014503}],"llm_call_rate":[{"mean":0.952788,"std":0.002345},{"mean":0.90897,"std":0.001211},{"mean":0.863758,"std":0.007089},{"mean":0.817818,"std":0.007948},{"mean":0.776909,"std":0.006052},{"mean":0.734545,"std":0.003577},{"mean":0.688303,"std":0.003833},{"mean":0.646667,"std":0.000378},{"mean":0.603515,"std":0.00444},{"mean":0.561212,"std":0.004308},{"mean":0.521394,"std":0.006632},{"mean":0.479697,"std":0.006015},{"mean":0.438242,"std":0.008625},{"mean":0.396061,"std":0.006647},{"mean":0.352667,"std":0.007846},{"mean":0.306788,"std":0.001389},{"mean":0.253333,"std":0.003234},{"mean":0.198485,"std":0.001458},{"mean":0.13697,"std":0.003259},{"mean":0.000303,"std":0.00021}]}},"entropy":{"risk_coverage":[{"mean":0.004848,"std":0.002099},{"mean":0.005455,"std":0.001818},{"mean":0.00404,"std":0.0007},{"mean":0.00303,"std":0.000525},{"mean":0.003152,"std":0.001111},{"mean":0.002828,"std":0.001262},{"mean":0.002771,"std":0.000794},{"mean":0.002879,"std":0.000946},{"mean":0.003098,"std":0.000233},{"mean":0.003152,"std":0.000757},{"mean":0.003636,"std":0.000875},{"mean":0.003939,"std":0.000606},{"mean":0.004662,"std":0.000427},{"mean":0.005455,"std":0.000779},{"mean":0.008242,"std":0.000874},{"mean":0.012576,"std":0.000798},{"mean":0.020178,"std":0.001077},{"mean":0.034411,"std":0.000816},{"mean":0.059841,"std":0.003829},{"mean":0.091879,"std":0.003354}],"operating":{"tau":[{"mean":-0.039087,"std":0.001004},{"mean":-0.04237,"std":0.000684},{"mean":-0.044748,"std":0.000479},{"mean":-0.046577,"std":0.000533},{"mean":-0.048221,"std":0.000469},{"mean":-0.049749,"std":0.000509},{"mean":-0.051428,"std":0.000354},{"mean":-0.053166,"std":0.000239},{"mean":-0.054832,"std":0.000397},{"mean":-0.05658,"std":0.00018},{"mean":-0.058284,"std":9.8e-05},{"mean":-0.060138,"std":4.4e-05},{"mean":-0.062193,"std":0.000218},{"mean":-0.064871,"std":0.000497},{"mean":-0.068225,"std":0.000752},{"mean":-0.073992,"std":0.00198},{"mean":-0.087019,"std":0.004626},{"mean":-0.111681,"std":0.001633},{"mean":-0.237522,"std":0.011341},{"mean":-1.938575,"std":0.040906}],"coverage":[{"mean":0.047697,"std":0.00183},{"mean":0.090364,"std":0.002025},{"mean":0.132909,"std":0.007664},{"mean":0.181333,"std":0.006632},{"mean":0.223152,"std":0.005195},{"mean":0.264727,"std":0.002065},{"mean":0.310545,"std":0.001273},{"mean":0.355333,"std":0.001001},{"mean":0.396667,"std":0.004187},{"mean":0.437818,"std":0.004919},{"mean":0.479758,"std":0.006592},{"mean":0.52,"std":0.007948},{"mean":0.56297,"std":0.003888},{"mean":0.604788,"std":0.004756},{"mean":0.644909,"std":0.004724},{"mean":0.693636,"std":0.001616},{"mean":0.745697,"std":0.00444},{"mean":0.799273,"std":0.002049},{"mean":0.861818,"std":0.003338},{"mean":0.999455,"std":0.000364}],"selective_risk":[{"mean":0.005037,"std":0.001984},{"mean":0.006058,"std":0.002123},{"mean":0.00458,"std":0.000912},{"mean":0.003336,"std":0.000498},{"mean":0.002988,"std":0.000473},{"mean":0.002971,"std":0.001025},{"mean":0.002729,"std":0.001206},{"mean":0.00273,"std":0.000789},{"mean":0.002755,"std":0.000814},{"mean":0.003047,"std":0.000272},{"mean":0.003035,"std":0.000416},{"mean":0.003272,"std":0.000863},{"mean":0.003987,"std":0.000775},{"mean":0.004109,"std":0.000469},{"mean":0.004509,"std":0.000725},{"mean":0.005329,"std":0.00083},{"mean":0.008042,"std":0.001074},{"mean":0.012587,"std":0.000797},{"mean":0.022853,"std":0.001552},{"mean":0.091444,"std":0.003307}],"accuracy_8":[{"mean":0.821515,"std":0.000378},{"mean":0.822364,"std":0.000315},{"mean":0.824667,"std":0.000378},{"mean":0.828909,"std":0.001553},{"mean":0.832788,"std":0.002854},{"mean":0.836242,"std":0.003413},{"mean":0.84103,"std":0.002649},{"mean":0.845939,"std":0.0029},{"mean":0.850485,"std":0.003968},{"mean":0.855273,"std":0.004094},{"mean":0.86,"std":0.005566},{"mean":0.864909,"std":0.004892},{"mean":0.869576,"std":0.003815},{"mean":0.876545,"std":0.004703},{"mean":0.883818,"std":0.005491},{"mean":0.894788,"std":0.004273},{"mean":0.903879,"std":0.002338},{"mean":0.912727,"std":0.000481},{"mean":0.918909,"std":0.002278},{"mean":0.908364,"std":0.003196}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.000577},{"mean":0.003,"std":0.001},{"mean":0.003333,"std":0.000577},{"mean":0.003333,"std":0.000577},{"mean":0.003667,"std":0.000577},{"mean":0.004333,"std":0.001528},{"mean":0.004667,"std":0.002082},{"mean":0.004667,"std":0.002082},{"mean":0.005,"std":0.002},{"mean":0.006333,"std":0.000577},{"mean":0.007,"std":0.001},{"mean":0.008333,"std":0.001528},{"mean":0.011,"std":0.001},{"mean":0.012,"std":0.001},{"mean":0.013667,"std":0.002517},{"mean":0.017667,"std":0.002887},{"mean":0.028333,"std":0.003215},{"mean":0.049333,"std":0.002309},{"mean":0.096667,"std":0.006429},{"mean":0.45,"std":0.013528}],"oos_misroute_rate":[{"mean":0.428333,"std":0.002309},{"mean":0.425667,"std":0.001155},{"mean":0.423667,"std":0.000577},{"mean":0.422333,"std":0.000577},{"mean":0.422,"std":0.0},{"mean":0.421667,"std":0.000577},{"mean":0.42,"std":0.001732},{"mean":0.417667,"std":0.002082},{"mean":0.417,"std":0.002646},{"mean":0.416,"std":0.003},{"mean":0.414333,"std":0.002517},{"mean":0.413333,"std":0.002517},{"mean":0.412333,"std":0.001528},{"mean":0.410667,"std":0.000577},{"mean":0.408667,"std":0.000577},{"mean":0.405333,"std":0.001528},{"mean":0.405,"std":0.001732},{"mean":0.401333,"std":0.004163},{"mean":0.387667,"std":0.009018},{"mean":0.451333,"std":0.013013}],"llm_call_rate":[{"mean":0.952303,"std":0.00183},{"mean":0.909636,"std":0.002025},{"mean":0.867091,"std":0.007664},{"mean":0.818667,"std":0.006632},{"mean":0.776848,"std":0.005195},{"mean":0.735273,"std":0.002065},{"mean":0.689455,"std":0.001273},{"mean":0.644667,"std":0.001001},{"mean":0.603333,"std":0.004187},{"mean":0.562182,"std":0.004919},{"mean":0.520242,"std":0.006592},{"mean":0.48,"std":0.007948},{"mean":0.43703,"std":0.003888},{"mean":0.395212,"std":0.004756},{"mean":0.355091,"std":0.004724},{"mean":0.306364,"std":0.001616},{"mean":0.254303,"std":0.00444},{"mean":0.200727,"std":0.002049},{"mean":0.138182,"std":0.003338},{"mean":0.000545,"std":0.000364}]}},"margin":{"risk_coverage":[{"mean":0.004848,"std":0.005555},{"mean":0.004848,"std":0.00105},{"mean":0.005657,"std":0.0007},{"mean":0.005152,"std":0.00105},{"mean":0.004606,"std":0.00042},{"mean":0.003838,"std":0.00035},{"mean":0.004156,"std":0.001374},{"mean":0.004394,"std":0.001389},{"mean":0.004714,"std":0.001299},{"mean":0.005576,"std":0.00021},{"mean":0.006612,"std":0.000331},{"mean":0.007374,"std":0.000974},{"mean":0.009138,"std":0.001165},{"mean":0.009697,"std":0.001307},{"mean":0.010828,"std":0.00056},{"mean":0.013864,"std":0.001041},{"mean":0.021604,"std":0.001403},{"mean":0.035286,"std":0.001516},{"mean":0.060797,"std":0.003551},{"mean":0.091879,"std":0.003354}],"operating":{"tau":[{"mean":6.576683,"std":0.108119},{"mean":6.433602,"std":0.095153},{"mean":6.345365,"std":0.085621},{"mean":6.275736,"std":0.077644},{"mean":6.217519,"std":0.071327},{"mean":6.168242,"std":0.057621},{"mean":6.12318,"std":0.048858},{"mean":6.075631,"std":0.042247},{"mean":6.024375,"std":0.037462},{"mean":5.974289,"std":0.0416},{"mean":5.914974,"std":0.043235},{"mean":5.850953,"std":0.055496},{"mean":5.790001,"std":0.071523},{"mean":5.709865,"std":0.086418},{"mean":5.630685,"std":0.098026},{"mean":5.517358,"std":0.102515},{"mean":5.355128,"std":0.089559},{"mean":4.998255,"std":0.050723},{"mean":4.016576,"std":0.032995},{"mean":0.027924,"std":0.023718}],"coverage":[{"mean":0.046909,"std":0.007818},{"mean":0.093091,"std":0.007197},{"mean":0.135152,"std":0.00711},{"mean":0.180667,"std":0.003193},{"mean":0.222788,"std":0.008652},{"mean":0.266667,"std":0.006924},{"mean":0.309273,"std":0.011109},{"mean":0.352788,"std":0.006468},{"mean":0.394667,"std":0.005144},{"mean":0.436606,"std":0.011912},{"mean":0.481636,"std":0.006074},{"mean":0.529636,"std":0.00946},{"mean":0.566606,"std":0.012273},{"mean":0.607515,"std":0.008531},{"mean":0.652182,"std":0.00987},{"mean":0.702242,"std":0.0078},{"mean":0.751455,"std":0.003757},{"mean":0.803576,"std":0.002003},{"mean":0.866424,"std":0.004707},{"mean":0.998667,"std":0.001001}],"selective_risk":[{"mean":0.005759,"std":0.007289},{"mean":0.004573,"std":0.002304},{"mean":0.00631,"std":0.001032},{"mean":0.005715,"std":0.001256},{"mean":0.004916,"std":0.000975},{"mean":0.004325,"std":0.000489},{"mean":0.003733,"std":0.000464},{"mean":0.004132,"std":0.001412},{"mean":0.004151,"std":0.001413},{"mean":0.004584,"std":0.001455},{"mean":0.005409,"std":0.00054},{"mean":0.00652,"std":0.000629},{"mean":0.006734,"std":0.00043},{"mean":0.007579,"std":0.000883},{"mean":0.009373,"std":0.001401},{"mean":0.009658,"std":0.001207},{"mean":0.010807,"std":0.000548},{"mean":0.014255,"std":0.000488},{"mean":0.024616,"std":0.002119},{"mean":0.090972,"std":0.00269}],"accuracy_8":[{"mean":0.82303,"std":0.002302},{"mean":0.824606,"std":0.002027},{"mean":0.828242,"std":0.003279},{"mean":0.832545,"std":0.003636},{"mean":0.835758,"std":0.004672},{"mean":0.838606,"std":0.00469},{"mean":0.842606,"std":0.005339},{"mean":0.847212,"std":0.004914},{"mean":0.85103,"std":0.004388},{"mean":0.855697,"std":0.004777},{"mean":0.860364,"std":0.003483},{"mean":0.866545,"std":0.003232},{"mean":0.871939,"std":0.002989},{"mean":0.877576,"std":0.003274},{"mean":0.883818,"std":0.004299},{"mean":0.891636,"std":0.00365},{"mean":0.900667,"std":0.002548},{"mean":0.910485,"std":0.001458},{"mean":0.919394,"std":0.002222},{"mean":0.908364,"std":0.003217}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.001528},{"mean":0.002333,"std":0.001155},{"mean":0.004667,"std":0.000577},{"mean":0.005,"std":0.001},{"mean":0.005333,"std":0.000577},{"mean":0.005667,"std":0.000577},{"mean":0.005667,"std":0.000577},{"mean":0.007,"std":0.002646},{"mean":0.007667,"std":0.002082},{"mean":0.009667,"std":0.002309},{"mean":0.013,"std":0.001},{"mean":0.017333,"std":0.001528},{"mean":0.019333,"std":0.001528},{"mean":0.023667,"std":0.002517},{"mean":0.031667,"std":0.005508},{"mean":0.034667,"std":0.005033},{"mean":0.041333,"std":0.001528},{"mean":0.057667,"std":0.001155},{"mean":0.105333,"std":0.009074},{"mean":0.447,"std":0.010817}],"oos_misroute_rate":[{"mean":0.425,"std":0.006557},{"mean":0.423333,"std":0.007095},{"mean":0.420667,"std":0.005132},{"mean":0.418,"std":0.004},{"mean":0.417333,"std":0.003512},{"mean":0.415333,"std":0.002517},{"mean":0.413667,"std":0.002082},{"mean":0.412667,"std":0.003055},{"mean":0.411667,"std":0.002887},{"mean":0.409333,"std":0.002517},{"mean":0.410333,"std":0.002309},{"mean":0.410667,"std":0.002082},{"mean":0.409,"std":0.003464},{"mean":0.409,"std":0.002},{"mean":0.410667,"std":0.002082},{"mean":0.407,"std":0.005568},{"mean":0.402333,"std":0.006028},{"mean":0.397333,"std":0.003055},{"mean":0.385333,"std":0.009504},{"mean":0.451333,"std":0.014295}],"llm_call_rate":[{"mean":0.953091,"std":0.007818},{"mean":0.906909,"std":0.007197},{"mean":0.864848,"std":0.00711},{"mean":0.819333,"std":0.003193},{"mean":0.777212,"std":0.008652},{"mean":0.733333,"std":0.006924},{"mean":0.690727,"std":0.011109},{"mean":0.647212,"std":0.006468},{"mean":0.605333,"std":0.005144},{"mean":0.563394,"std":0.011912},{"mean":0.518364,"std":0.006074},{"mean":0.470364,"std":0.00946},{"mean":0.433394,"std":0.012273},{"mean":0.392485,"std":0.008531},{"mean":0.347818,"std":0.00987},{"mean":0.297758,"std":0.0078},{"mean":0.248545,"std":0.003757},{"mean":0.196424,"std":0.002003},{"mean":0.133576,"std":0.004707},{"mean":0.001333,"std":0.001001}]}},"msp_t":{"risk_coverage":[{"mean":0.004848,"std":0.002099},{"mean":0.005455,"std":0.001818},{"mean":0.00404,"std":0.0007},{"mean":0.00303,"std":0.000525},{"mean":0.003152,"std":0.001111},{"mean":0.002828,"std":0.001262},{"mean":0.002944,"std":0.000794},{"mean":0.002879,"std":0.000946},{"mean":0.003098,"std":0.000467},{"mean":0.003152,"std":0.000757},{"mean":0.003747,"std":0.000763},{"mean":0.003939,"std":0.000606},{"mean":0.004662,"std":0.000427},{"mean":0.005628,"std":0.000835},{"mean":0.008566,"std":0.001093},{"mean":0.0125,"std":0.000601},{"mean":0.020321,"std":0.00098},{"mean":0.034815,"std":0.000649},{"mean":0.060159,"std":0.003752},{"mean":0.091879,"std":0.003354}],"operating":{"tau":[{"mean":0.994878,"std":0.000413},{"mean":0.994379,"std":0.000375},{"mean":0.993986,"std":0.000338},{"mean":0.993711,"std":0.000319},{"mean":0.993473,"std":0.000315},{"mean":0.993233,"std":0.000327},{"mean":0.992973,"std":0.000337},{"mean":0.992701,"std":0.000353},{"mean":0.992425,"std":0.000398},{"mean":0.992145,"std":0.000395},{"mean":0.991868,"std":0.000392},{"mean":0.991593,"std":0.000355},{"mean":0.991232,"std":0.000409},{"mean":0.990812,"std":0.000478},{"mean":0.990234,"std":0.000533},{"mean":0.989268,"std":0.000488},{"mean":0.987001,"std":0.000807},{"mean":0.982452,"std":0.0008},{"mean":0.955983,"std":0.001107},{"mean":0.215666,"std":0.003216}],"coverage":[{"mean":0.047273,"std":0.002446},{"mean":0.090909,"std":0.001192},{"mean":0.136727,"std":0.00756},{"mean":0.182182,"std":0.007948},{"mean":0.223091,"std":0.006117},{"mean":0.26497,"std":0.003315},{"mean":0.31097,"std":0.003305},{"mean":0.353515,"std":0.000278},{"mean":0.396667,"std":0.004818},{"mean":0.439273,"std":0.004779},{"mean":0.478909,"std":0.006732},{"mean":0.519636,"std":0.005548},{"mean":0.561818,"std":0.008487},{"mean":0.603576,"std":0.006129},{"mean":0.646909,"std":0.007114},{"mean":0.693212,"std":0.001277},{"mean":0.746727,"std":0.003154},{"mean":0.801394,"std":0.00147},{"mean":0.86303,"std":0.003098},{"mean":0.999697,"std":0.00021}],"selective_risk":[{"mean":0.005065,"std":0.001913},{"mean":0.005994,"std":0.001985},{"mean":0.00445,"std":0.000872},{"mean":0.003324,"std":0.000528},{"mean":0.002993,"std":0.000502},{"mean":0.002967,"std":0.001013},{"mean":0.002724,"std":0.001196},{"mean":0.002915,"std":0.000788},{"mean":0.002911,"std":0.000984},{"mean":0.003177,"std":0.000515},{"mean":0.00317,"std":0.000627},{"mean":0.003153,"std":0.000642},{"mean":0.003889,"std":0.000697},{"mean":0.004218,"std":0.000528},{"mean":0.004492,"std":0.000981},{"mean":0.00542,"std":0.000793},{"mean":0.008438,"std":0.001113},{"mean":0.012704,"std":0.000767},{"mean":0.023032,"std":0.001185},{"mean":0.091664,"std":0.003564}],"accuracy_8":[{"mean":0.821515,"std":0.000378},{"mean":0.822485,"std":0.000278},{"mean":0.825152,"std":0.00134},{"mean":0.829273,"std":0.000962},{"mean":0.832424,"std":0.002338},{"mean":0.835697,"std":0.003028},{"mean":0.840848,"std":0.0033},{"mean":0.845212,"std":0.003182},{"mean":0.849212,"std":0.004123},{"mean":0.854242,"std":0.004565},{"mean":0.859273,"std":0.004821},{"mean":0.864364,"std":0.004158},{"mean":0.869333,"std":0.004672},{"mean":0.876606,"std":0.004384},{"mean":0.884303,"std":0.006153},{"mean":0.893576,"std":0.003888},{"mean":0.903636,"std":0.002234},{"mean":0.91303,"std":0.000278},{"mean":0.919333,"std":0.00183},{"mean":0.908182,"std":0.003406}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.000577},{"mean":0.003,"std":0.001},{"mean":0.003333,"std":0.000577},{"mean":0.003333,"std":0.000577},{"mean":0.003667,"std":0.000577},{"mean":0.004333,"std":0.001528},{"mean":0.004667,"std":0.002082},{"mean":0.005,"std":0.002},{"mean":0.005333,"std":0.002082},{"mean":0.006667,"std":0.000577},{"mean":0.007333,"std":0.001155},{"mean":0.008,"std":0.001},{"mean":0.011,"std":0.001},{"mean":0.012333,"std":0.001528},{"mean":0.014,"std":0.003},{"mean":0.018,"std":0.002646},{"mean":0.03,"std":0.003464},{"mean":0.05,"std":0.003},{"mean":0.097667,"std":0.004509},{"mean":0.451333,"std":0.015503}],"oos_misroute_rate":[{"mean":0.428333,"std":0.002309},{"mean":0.425667,"std":0.001155},{"mean":0.422667,"std":0.000577},{"mean":0.422333,"std":0.000577},{"mean":0.422,"std":0.0},{"mean":0.421,"std":0.001},{"mean":0.418667,"std":0.002082},{"mean":0.417333,"std":0.002517},{"mean":0.416333,"std":0.003512},{"mean":0.415,"std":0.002646},{"mean":0.414333,"std":0.001528},{"mean":0.413,"std":0.002},{"mean":0.412,"std":0.001732},{"mean":0.409667,"std":0.000577},{"mean":0.407333,"std":0.001528},{"mean":0.405333,"std":0.001528},{"mean":0.405667,"std":0.001155},{"mean":0.399667,"std":0.004163},{"mean":0.386,"std":0.008185},{"mean":0.452333,"std":0.014503}],"llm_call_rate":[{"mean":0.952727,"std":0.002446},{"mean":0.909091,"std":0.001192},{"mean":0.863273,"std":0.00756},{"mean":0.817818,"std":0.007948},{"mean":0.776909,"std":0.006117},{"mean":0.73503,"std":0.003315},{"mean":0.68903,"std":0.003305},{"mean":0.646485,"std":0.000278},{"mean":0.603333,"std":0.004818},{"mean":0.560727,"std":0.004779},{"mean":0.521091,"std":0.006732},{"mean":0.480364,"std":0.005548},{"mean":0.438182,"std":0.008487},{"mean":0.396424,"std":0.006129},{"mean":0.353091,"std":0.007114},{"mean":0.306788,"std":0.001277},{"mean":0.253273,"std":0.003154},{"mean":0.198606,"std":0.00147},{"mean":0.13697,"std":0.003098},{"mean":0.000303,"std":0.00021}]}}}}},"bert/k25":{"model":"bert","k":25,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["bert-base-uncased-k25-seed42","bert-base-uncased-k25-seed43","bert-base-uncased-k25-seed44"],"logits_sha256":["5397a2237935d24210a1b8e47ed837f1807db8beb0a4c18f31d24d0681bef1cc","cf40024e86a42cfe34703b4b04e58696d93a03f3408a44f820960b11d554d626","c4de18e8e6d5e96011ab20178ec669bd2462c6aae44cd2fd40ae8ba3ce077f30"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.007273,"std":0.003636},{"mean":0.007273,"std":0.001818},{"mean":0.006465,"std":0.002523},{"mean":0.007576,"std":0.003442},{"mean":0.008,"std":0.00317},{"mean":0.008889,"std":0.003943},{"mean":0.009351,"std":0.004438},{"mean":0.01,"std":0.003608},{"mean":0.012121,"std":0.003156},{"mean":0.012364,"std":0.002976},{"mean":0.013664,"std":0.001336},{"mean":0.015354,"std":0.001723},{"mean":0.018555,"std":0.003818},{"mean":0.02303,"std":0.003639},{"mean":0.029172,"std":0.003031},{"mean":0.037803,"std":0.004278},{"mean":0.053405,"std":0.005072},{"mean":0.072862,"std":0.006453},{"mean":0.100989,"std":0.008057},{"mean":0.134848,"std":0.00646}],"operating":{"tau":[{"mean":0.45325,"std":0.013769},{"mean":0.412565,"std":0.016476},{"mean":0.382732,"std":0.00981},{"mean":0.357598,"std":0.002201},{"mean":0.336669,"std":0.002805},{"mean":0.320916,"std":0.002965},{"mean":0.30476,"std":0.002208},{"mean":0.289705,"std":0.002214},{"mean":0.275309,"std":0.002993},{"mean":0.260218,"std":0.00372},{"mean":0.24728,"std":0.003494},{"mean":0.233206,"std":0.003252},{"mean":0.219084,"std":0.002329},{"mean":0.204877,"std":0.002114},{"mean":0.189295,"std":0.001876},{"mean":0.172973,"std":0.00263},{"mean":0.154363,"std":0.003411},{"mean":0.130025,"std":6e-05},{"mean":0.091434,"std":0.004539},{"mean":0.023213,"std":0.001684}],"coverage":[{"mean":0.044061,"std":0.004462},{"mean":0.085394,"std":0.004425},{"mean":0.129879,"std":0.007938},{"mean":0.176606,"std":0.00691},{"mean":0.225455,"std":0.010013},{"mean":0.263939,"std":0.008656},{"mean":0.306364,"std":0.009301},{"mean":0.347394,"std":0.007561},{"mean":0.389455,"std":0.009248},{"mean":0.433091,"std":0.009012},{"mean":0.474364,"std":0.009585},{"mean":0.51903,"std":0.009504},{"mean":0.567758,"std":0.014185},{"mean":0.608485,"std":0.0148},{"mean":0.652061,"std":0.009344},{"mean":0.697939,"std":0.011534},{"mean":0.746848,"std":0.013609},{"mean":0.801636,"std":0.005548},{"mean":0.876485,"std":0.00626},{"mean":0.999576,"std":0.000278}],"selective_risk":[{"mean":0.008028,"std":0.003338},{"mean":0.006327,"std":0.001803},{"mean":0.00655,"std":0.002128},{"mean":0.007462,"std":0.003692},{"mean":0.007238,"std":0.002731},{"mean":0.007809,"std":0.00319},{"mean":0.00857,"std":0.003816},{"mean":0.009434,"std":0.00428},{"mean":0.00981,"std":0.003882},{"mean":0.011633,"std":0.003147},{"mean":0.012149,"std":0.003163},{"mean":0.013317,"std":0.002014},{"mean":0.013989,"std":0.001552},{"mean":0.015545,"std":0.001649},{"mean":0.018685,"std":0.003665},{"mean":0.022556,"std":0.003786},{"mean":0.028558,"std":0.002988},{"mean":0.0381,"std":0.004423},{"mean":0.062576,"std":0.006023},{"mean":0.13448,"std":0.006697}],"accuracy_8":[{"mean":0.821939,"std":0.000915},{"mean":0.823697,"std":0.000458},{"mean":0.826242,"std":0.00134},{"mean":0.829939,"std":0.001821},{"mean":0.836061,"std":0.002825},{"mean":0.839879,"std":0.002482},{"mean":0.844727,"std":0.004923},{"mean":0.849879,"std":0.004866},{"mean":0.854485,"std":0.004707},{"mean":0.859939,"std":0.00548},{"mean":0.865515,"std":0.005443},{"mean":0.871152,"std":0.006833},{"mean":0.877939,"std":0.006099},{"mean":0.882667,"std":0.006886},{"mean":0.887333,"std":0.007532},{"mean":0.891576,"std":0.007182},{"mean":0.89497,"std":0.004937},{"mean":0.898545,"std":0.003713},{"mean":0.895455,"std":0.004892},{"mean":0.865333,"std":0.006546}],"high_conf_oos_misroute_rate":[{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.001155},{"mean":0.004333,"std":0.002082},{"mean":0.005667,"std":0.003215},{"mean":0.007,"std":0.002646},{"mean":0.009,"std":0.003464},{"mean":0.011667,"std":0.004509},{"mean":0.015,"std":0.006},{"mean":0.017,"std":0.006245},{"mean":0.022,"std":0.006083},{"mean":0.025,"std":0.007937},{"mean":0.030667,"std":0.005508},{"mean":0.035,"std":0.006557},{"mean":0.042,"std":0.008544},{"mean":0.050333,"std":0.010066},{"mean":0.063333,"std":0.009452},{"mean":0.085,"std":0.007},{"mean":0.121667,"std":0.008622},{"mean":0.230667,"std":0.032501},{"mean":0.617667,"std":0.033005}],"oos_misroute_rate":[{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.43,"std":0.003606},{"mean":0.43,"std":0.003},{"mean":0.431,"std":0.003464},{"mean":0.43,"std":0.004},{"mean":0.429333,"std":0.00611},{"mean":0.429333,"std":0.007506},{"mean":0.428,"std":0.009},{"mean":0.427667,"std":0.010017},{"mean":0.426667,"std":0.010599},{"mean":0.428,"std":0.009539},{"mean":0.429,"std":0.009539},{"mean":0.430667,"std":0.011676},{"mean":0.435333,"std":0.016166},{"mean":0.440333,"std":0.015535},{"mean":0.467667,"std":0.030006},{"mean":0.619,"std":0.032741}],"llm_call_rate":[{"mean":0.955939,"std":0.004462},{"mean":0.914606,"std":0.004425},{"mean":0.870121,"std":0.007938},{"mean":0.823394,"std":0.00691},{"mean":0.774545,"std":0.010013},{"mean":0.736061,"std":0.008656},{"mean":0.693636,"std":0.009301},{"mean":0.652606,"std":0.007561},{"mean":0.610545,"std":0.009248},{"mean":0.566909,"std":0.009012},{"mean":0.525636,"std":0.009585},{"mean":0.48097,"std":0.009504},{"mean":0.432242,"std":0.014185},{"mean":0.391515,"std":0.0148},{"mean":0.347939,"std":0.009344},{"mean":0.302061,"std":0.011534},{"mean":0.253152,"std":0.013609},{"mean":0.198364,"std":0.005548},{"mean":0.123515,"std":0.00626},{"mean":0.000424,"std":0.000278}]}},"entropy":{"risk_coverage":[{"mean":0.004848,"std":0.004199},{"mean":0.006061,"std":0.002777},{"mean":0.006465,"std":0.00305},{"mean":0.006364,"std":0.002405},{"mean":0.006303,"std":0.002338},{"mean":0.007273,"std":0.001818},{"mean":0.009177,"std":0.002563},{"mean":0.011667,"std":0.00486},{"mean":0.014276,"std":0.006481},{"mean":0.014667,"std":0.006406},{"mean":0.015868,"std":0.006612},{"mean":0.017374,"std":0.005784},{"mean":0.01986,"std":0.005657},{"mean":0.022684,"std":0.005645},{"mean":0.029333,"std":0.007138},{"mean":0.039773,"std":0.006302},{"mean":0.054688,"std":0.008584},{"mean":0.076431,"std":0.008517},{"mean":0.10303,"std":0.009808},{"mean":0.134848,"std":0.00646}],"operating":{"tau":[{"mean":-3.246799,"std":0.083831},{"mean":-3.416315,"std":0.041246},{"mean":-3.507724,"std":0.0389},{"mean":-3.583479,"std":0.024265},{"mean":-3.639137,"std":0.021717},{"mean":-3.695489,"std":0.020737},{"mean":-3.739709,"std":0.021145},{"mean":-3.78239,"std":0.021497},{"mean":-3.824232,"std":0.019022},{"mean":-3.865188,"std":0.021677},{"mean":-3.907364,"std":0.025163},{"mean":-3.944938,"std":0.02144},{"mean":-3.986287,"std":0.017276},{"mean":-4.029792,"std":0.01883},{"mean":-4.075041,"std":0.022532},{"mean":-4.131575,"std":0.020694},{"mean":-4.208458,"std":0.03236},{"mean":-4.310855,"std":0.025365},{"mean":-4.452983,"std":0.009914},{"mean":-4.860973,"std":0.045221}],"coverage":[{"mean":0.04497,"std":0.004937},{"mean":0.086242,"std":0.002854},{"mean":0.124424,"std":0.002956},{"mean":0.170364,"std":0.004545},{"mean":0.214182,"std":0.005654},{"mean":0.263273,"std":0.003406},{"mean":0.303091,"std":0.00778},{"mean":0.342788,"std":0.010652},{"mean":0.384424,"std":0.013292},{"mean":0.427636,"std":0.013062},{"mean":0.473515,"std":0.015761},{"mean":0.514242,"std":0.013679},{"mean":0.557758,"std":0.013976},{"mean":0.603212,"std":0.011719},{"mean":0.646545,"std":0.011551},{"mean":0.693576,"std":0.012563},{"mean":0.743879,"std":0.007184},{"mean":0.796424,"std":0.009029},{"mean":0.868182,"std":0.006322},{"mean":0.999273,"std":0.00126}],"selective_risk":[{"mean":0.00507,"std":0.004391},{"mean":0.006963,"std":0.002958},{"mean":0.006321,"std":0.002148},{"mean":0.005667,"std":0.002562},{"mean":0.006268,"std":0.002571},{"mean":0.006232,"std":0.001896},{"mean":0.007423,"std":0.001977},{"mean":0.008838,"std":0.002164},{"mean":0.010701,"std":0.003891},{"mean":0.013286,"std":0.005736},{"mean":0.014311,"std":0.006557},{"mean":0.015219,"std":0.006533},{"mean":0.015748,"std":0.006578},{"mean":0.017487,"std":0.005887},{"mean":0.02003,"std":0.00562},{"mean":0.02263,"std":0.005313},{"mean":0.02827,"std":0.005736},{"mean":0.038651,"std":0.006385},{"mean":0.061573,"std":0.008037},{"mean":0.134335,"std":0.007227}],"accuracy_8":[{"mean":0.822,"std":0.001012},{"mean":0.823273,"std":0.000315},{"mean":0.825455,"std":0.001898},{"mean":0.828061,"std":0.003022},{"mean":0.832121,"std":0.003841},{"mean":0.838545,"std":0.005068},{"mean":0.844606,"std":0.008615},{"mean":0.850364,"std":0.009162},{"mean":0.856303,"std":0.008501},{"mean":0.86097,"std":0.008376},{"mean":0.865879,"std":0.009302},{"mean":0.871212,"std":0.008912},{"mean":0.876788,"std":0.007285},{"mean":0.881455,"std":0.004902},{"mean":0.884303,"std":0.003274},{"mean":0.888727,"std":0.001898},{"mean":0.892303,"std":0.002027},{"mean":0.896364,"std":0.003464},{"mean":0.894,"std":0.005741},{"mean":0.865455,"std":0.006909}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.001},{"mean":0.003,"std":0.001732},{"mean":0.004,"std":0.002},{"mean":0.005,"std":0.003},{"mean":0.006333,"std":0.002887},{"mean":0.007667,"std":0.003215},{"mean":0.009333,"std":0.003512},{"mean":0.010333,"std":0.002517},{"mean":0.013333,"std":0.004509},{"mean":0.016667,"std":0.004041},{"mean":0.019667,"std":0.005508},{"mean":0.023,"std":0.005292},{"mean":0.026,"std":0.006928},{"mean":0.032667,"std":0.008327},{"mean":0.043,"std":0.01044},{"mean":0.055,"std":0.010392},{"mean":0.077,"std":0.014799},{"mean":0.118,"std":0.010392},{"mean":0.221,"std":0.014731},{"mean":0.616667,"std":0.037314}],"oos_misroute_rate":[{"mean":0.432333,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.431667,"std":0.001528},{"mean":0.430667,"std":0.002517},{"mean":0.430667,"std":0.003215},{"mean":0.43,"std":0.003606},{"mean":0.429667,"std":0.003512},{"mean":0.430333,"std":0.003055},{"mean":0.43,"std":0.006245},{"mean":0.429667,"std":0.00611},{"mean":0.428333,"std":0.007024},{"mean":0.427667,"std":0.007767},{"mean":0.433,"std":0.007},{"mean":0.431333,"std":0.009074},{"mean":0.437,"std":0.011358},{"mean":0.441,"std":0.014},{"mean":0.467667,"std":0.020502},{"mean":0.618,"std":0.035086}],"llm_call_rate":[{"mean":0.95503,"std":0.004937},{"mean":0.913758,"std":0.002854},{"mean":0.875576,"std":0.002956},{"mean":0.829636,"std":0.004545},{"mean":0.785818,"std":0.005654},{"mean":0.736727,"std":0.003406},{"mean":0.696909,"std":0.00778},{"mean":0.657212,"std":0.010652},{"mean":0.615576,"std":0.013292},{"mean":0.572364,"std":0.013062},{"mean":0.526485,"std":0.015761},{"mean":0.485758,"std":0.013679},{"mean":0.442242,"std":0.013976},{"mean":0.396788,"std":0.011719},{"mean":0.353455,"std":0.011551},{"mean":0.306424,"std":0.012563},{"mean":0.256121,"std":0.007184},{"mean":0.203576,"std":0.009029},{"mean":0.131818,"std":0.006322},{"mean":0.000727,"std":0.00126}]}},"margin":{"risk_coverage":[{"mean":0.009697,"std":0.004199},{"mean":0.008485,"std":0.00105},{"mean":0.012121,"std":0.001212},{"mean":0.012727,"std":0.000909},{"mean":0.012848,"std":0.001111},{"mean":0.014747,"std":0.00035},{"mean":0.015411,"std":0.001081},{"mean":0.016667,"std":0.001721},{"mean":0.018316,"std":0.002073},{"mean":0.021697,"std":0.00183},{"mean":0.026116,"std":0.002582},{"mean":0.03202,"std":0.00311},{"mean":0.041212,"std":0.004522},{"mean":0.048745,"std":0.005597},{"mean":0.058828,"std":0.006533},{"mean":0.070303,"std":0.007249},{"mean":0.085134,"std":0.009453},{"mean":0.100404,"std":0.007701},{"mean":0.116683,"std":0.007274},{"mean":0.134848,"std":0.00646}],"operating":{"tau":[{"mean":8.383183,"std":0.293929},{"mean":7.586686,"std":0.185514},{"mean":6.928291,"std":0.09767},{"mean":6.392955,"std":0.110666},{"mean":5.986679,"std":0.153153},{"mean":5.580039,"std":0.214063},{"mean":5.207407,"std":0.242481},{"mean":4.821614,"std":0.307345},{"mean":4.450969,"std":0.335333},{"mean":4.102854,"std":0.35379},{"mean":3.723521,"std":0.321712},{"mean":3.315741,"std":0.245029},{"mean":2.889288,"std":0.218158},{"mean":2.436023,"std":0.217477},{"mean":2.071558,"std":0.22564},{"mean":1.641715,"std":0.20065},{"mean":1.232863,"std":0.163392},{"mean":0.80623,"std":0.100056},{"mean":0.392075,"std":0.032145},{"mean":0.001853,"std":0.001543}],"coverage":[{"mean":0.043879,"std":0.001183},{"mean":0.088424,"std":0.00495},{"mean":0.135212,"std":0.001821},{"mean":0.182667,"std":0.010025},{"mean":0.226788,"std":0.008788},{"mean":0.272424,"std":0.007287},{"mean":0.31897,"std":0.004365},{"mean":0.362182,"std":0.005091},{"mean":0.402848,"std":0.010537},{"mean":0.443636,"std":0.014092},{"mean":0.485697,"std":0.015407},{"mean":0.534061,"std":0.010098},{"mean":0.582909,"std":0.008545},{"mean":0.638485,"std":0.007888},{"mean":0.685576,"std":0.006225},{"mean":0.743879,"std":0.005842},{"mean":0.804061,"std":0.008447},{"mean":0.867212,"std":0.004679},{"mean":0.933091,"std":0.002691},{"mean":0.999758,"std":0.000278}],"selective_risk":[{"mean":0.009596,"std":0.006063},{"mean":0.00826,"std":0.002277},{"mean":0.009845,"std":0.002732},{"mean":0.012253,"std":0.0018},{"mean":0.013346,"std":0.000779},{"mean":0.013552,"std":0.001376},{"mean":0.014827,"std":0.000767},{"mean":0.016059,"std":0.001415},{"mean":0.016848,"std":0.002161},{"mean":0.018283,"std":0.00221},{"mean":0.020085,"std":0.001346},{"mean":0.024401,"std":0.002181},{"mean":0.030332,"std":0.003942},{"mean":0.038409,"std":0.005856},{"mean":0.046653,"std":0.006484},{"mean":0.057338,"std":0.007085},{"mean":0.071988,"std":0.010438},{"mean":0.090764,"std":0.009611},{"mean":0.110235,"std":0.00607},{"mean":0.134821,"std":0.006395}],"accuracy_8":[{"mean":0.823273,"std":0.000727},{"mean":0.827212,"std":0.002729},{"mean":0.829333,"std":0.003274},{"mean":0.832424,"std":0.00147},{"mean":0.836848,"std":0.000639},{"mean":0.842485,"std":0.001481},{"mean":0.847758,"std":0.002373},{"mean":0.853939,"std":0.001458},{"mean":0.858424,"std":0.001994},{"mean":0.863333,"std":0.003661},{"mean":0.868303,"std":0.004866},{"mean":0.873333,"std":0.005157},{"mean":0.878182,"std":0.005671},{"mean":0.883333,"std":0.005111},{"mean":0.885758,"std":0.004261},{"mean":0.88897,"std":0.004947},{"mean":0.887091,"std":0.008238},{"mean":0.882848,"std":0.009117},{"mean":0.876606,"std":0.005437},{"mean":0.865152,"std":0.00646}],"high_conf_oos_misroute_rate":[{"mean":0.002,"std":0.001},{"mean":0.003667,"std":0.000577},{"mean":0.006667,"std":0.001528},{"mean":0.010667,"std":0.002309},{"mean":0.014,"std":0.001732},{"mean":0.017,"std":0.003},{"mean":0.021,"std":0.001},{"mean":0.026,"std":0.002},{"mean":0.030667,"std":0.003786},{"mean":0.036,"std":0.004},{"mean":0.044667,"std":0.002082},{"mean":0.059333,"std":0.00611},{"mean":0.08,"std":0.01253},{"mean":0.111333,"std":0.019502},{"mean":0.146,"std":0.023302},{"mean":0.196,"std":0.021071},{"mean":0.264,"std":0.028355},{"mean":0.357333,"std":0.023094},{"mean":0.470667,"std":0.014572},{"mean":0.62,"std":0.031765}],"oos_misroute_rate":[{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001732},{"mean":0.433333,"std":0.003786},{"mean":0.432667,"std":0.005859},{"mean":0.431333,"std":0.00611},{"mean":0.430333,"std":0.006807},{"mean":0.428333,"std":0.009292},{"mean":0.425333,"std":0.009292},{"mean":0.423333,"std":0.008737},{"mean":0.421,"std":0.013528},{"mean":0.418667,"std":0.015503},{"mean":0.420333,"std":0.018502},{"mean":0.425333,"std":0.018771},{"mean":0.432,"std":0.021166},{"mean":0.439667,"std":0.023861},{"mean":0.452667,"std":0.015308},{"mean":0.475667,"std":0.023459},{"mean":0.509333,"std":0.024705},{"mean":0.555667,"std":0.021197},{"mean":0.62,"std":0.031765}],"llm_call_rate":[{"mean":0.956121,"std":0.001183},{"mean":0.911576,"std":0.00495},{"mean":0.864788,"std":0.001821},{"mean":0.817333,"std":0.010025},{"mean":0.773212,"std":0.008788},{"mean":0.727576,"std":0.007287},{"mean":0.68103,"std":0.004365},{"mean":0.637818,"std":0.005091},{"mean":0.597152,"std":0.010537},{"mean":0.556364,"std":0.014092},{"mean":0.514303,"std":0.015407},{"mean":0.465939,"std":0.010098},{"mean":0.417091,"std":0.008545},{"mean":0.361515,"std":0.007888},{"mean":0.314424,"std":0.006225},{"mean":0.256121,"std":0.005842},{"mean":0.195939,"std":0.008447},{"mean":0.132788,"std":0.004679},{"mean":0.066909,"std":0.002691},{"mean":0.000242,"std":0.000278}]}},"msp_t":{"risk_coverage":[{"mean":0.007273,"std":0.003636},{"mean":0.007273,"std":0.003636},{"mean":0.007273,"std":0.001212},{"mean":0.009394,"std":0.002624},{"mean":0.011394,"std":0.00183},{"mean":0.013333,"std":0.002099},{"mean":0.01368,"std":0.002861},{"mean":0.014242,"std":0.001837},{"mean":0.015488,"std":0.002867},{"mean":0.017091,"std":0.003232},{"mean":0.020386,"std":0.002672},{"mean":0.024545,"std":0.004166},{"mean":0.029837,"std":0.002857},{"mean":0.034372,"std":0.002602},{"mean":0.042747,"std":0.005224},{"mean":0.05197,"std":0.005706},{"mean":0.064314,"std":0.005451},{"mean":0.078855,"std":0.004047},{"mean":0.102775,"std":0.005976},{"mean":0.134848,"std":0.00646}],"operating":{"tau":[{"mean":0.99886,"std":0.000235},{"mean":0.997779,"std":0.000257},{"mean":0.996425,"std":0.000311},{"mean":0.994593,"std":0.000689},{"mean":0.991918,"std":0.001734},{"mean":0.988881,"std":0.002814},{"mean":0.984604,"std":0.004359},{"mean":0.979366,"std":0.006638},{"mean":0.972301,"std":0.009304},{"mean":0.963158,"std":0.011356},{"mean":0.950347,"std":0.013202},{"mean":0.932321,"std":0.017678},{"mean":0.907959,"std":0.019112},{"mean":0.872002,"std":0.023215},{"mean":0.824475,"std":0.028082},{"mean":0.76478,"std":0.031934},{"mean":0.687157,"std":0.033985},{"mean":0.58791,"std":0.029494},{"mean":0.474281,"std":0.019737},{"mean":0.0858,"std":0.007969}],"coverage":[{"mean":0.042242,"std":0.002338},{"mean":0.088848,"std":0.00021},{"mean":0.134,"std":0.005852},{"mean":0.18,"std":0.007818},{"mean":0.226485,"std":0.007654},{"mean":0.269939,"std":0.013619},{"mean":0.314788,"std":0.007834},{"mean":0.355515,"std":0.015537},{"mean":0.396242,"std":0.017487},{"mean":0.436485,"std":0.014907},{"mean":0.481333,"std":0.010091},{"mean":0.528,"std":0.010709},{"mean":0.574303,"std":0.008735},{"mean":0.621152,"std":0.008265},{"mean":0.671879,"std":0.004937},{"mean":0.723697,"std":0.004384},{"mean":0.777879,"std":0.007129},{"mean":0.83497,"std":0.008095},{"mean":0.898545,"std":0.004409},{"mean":0.999515,"std":0.000458}],"selective_risk":[{"mean":0.008652,"std":0.004549},{"mean":0.0075,"std":0.003112},{"mean":0.007693,"std":0.001548},{"mean":0.009081,"std":0.002892},{"mean":0.010693,"std":0.003006},{"mean":0.011659,"std":0.00174},{"mean":0.013109,"std":0.00233},{"mean":0.013812,"std":0.002883},{"mean":0.013585,"std":0.0026},{"mean":0.014989,"std":0.002762},{"mean":0.016754,"std":0.002555},{"mean":0.018005,"std":0.002998},{"mean":0.022358,"std":0.00349},{"mean":0.025953,"std":0.003754},{"mean":0.031755,"std":0.003434},{"mean":0.037438,"std":0.002473},{"mean":0.047737,"std":0.006278},{"mean":0.061337,"std":0.004423},{"mean":0.078644,"std":0.003693},{"mean":0.134428,"std":0.006618}],"accuracy_8":[{"mean":0.822182,"std":0.001012},{"mean":0.82497,"std":0.001096},{"mean":0.827939,"std":0.002723},{"mean":0.832182,"std":0.002144},{"mean":0.836182,"std":0.000545},{"mean":0.840848,"std":0.000378},{"mean":0.845394,"std":0.001034},{"mean":0.852303,"std":0.002338},{"mean":0.857879,"std":0.002911},{"mean":0.862061,"std":0.002482},{"mean":0.867515,"std":0.003413},{"mean":0.873152,"std":0.004777},{"mean":0.878364,"std":0.005695},{"mean":0.883758,"std":0.004914},{"mean":0.888364,"std":0.004424},{"mean":0.89297,"std":0.003884},{"mean":0.894303,"std":0.005169},{"mean":0.893576,"std":0.005987},{"mean":0.891697,"std":0.004304},{"mean":0.865455,"std":0.006364}],"high_conf_oos_misroute_rate":[{"mean":0.001667,"std":0.000577},{"mean":0.003333,"std":0.001528},{"mean":0.005333,"std":0.001528},{"mean":0.007,"std":0.002646},{"mean":0.011,"std":0.002646},{"mean":0.014667,"std":0.001528},{"mean":0.018333,"std":0.002517},{"mean":0.021333,"std":0.004509},{"mean":0.023667,"std":0.005033},{"mean":0.03,"std":0.007211},{"mean":0.037,"std":0.007211},{"mean":0.043,"std":0.009849},{"mean":0.059333,"std":0.012503},{"mean":0.072667,"std":0.011504},{"mean":0.096333,"std":0.012897},{"mean":0.121333,"std":0.009815},{"mean":0.162,"std":0.020664},{"mean":0.225667,"std":0.004933},{"mean":0.313667,"std":0.005859},{"mean":0.617333,"std":0.034152}],"oos_misroute_rate":[{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.433333,"std":0.001528},{"mean":0.432,"std":0.002646},{"mean":0.432,"std":0.002646},{"mean":0.430333,"std":0.005859},{"mean":0.43,"std":0.007211},{"mean":0.428,"std":0.008888},{"mean":0.426,"std":0.010149},{"mean":0.423333,"std":0.010693},{"mean":0.425,"std":0.010583},{"mean":0.424,"std":0.011136},{"mean":0.426333,"std":0.015503},{"mean":0.426667,"std":0.016042},{"mean":0.431333,"std":0.020404},{"mean":0.435667,"std":0.021197},{"mean":0.445333,"std":0.027319},{"mean":0.464,"std":0.020664},{"mean":0.484667,"std":0.017616},{"mean":0.618333,"std":0.032517}],"llm_call_rate":[{"mean":0.957758,"std":0.002338},{"mean":0.911152,"std":0.00021},{"mean":0.866,"std":0.005852},{"mean":0.82,"std":0.007818},{"mean":0.773515,"std":0.007654},{"mean":0.730061,"std":0.013619},{"mean":0.685212,"std":0.007834},{"mean":0.644485,"std":0.015537},{"mean":0.603758,"std":0.017487},{"mean":0.563515,"std":0.014907},{"mean":0.518667,"std":0.010091},{"mean":0.472,"std":0.010709},{"mean":0.425697,"std":0.008735},{"mean":0.378848,"std":0.008265},{"mean":0.328121,"std":0.004937},{"mean":0.276303,"std":0.004384},{"mean":0.222121,"std":0.007129},{"mean":0.16503,"std":0.008095},{"mean":0.101455,"std":0.004409},{"mean":0.000485,"std":0.000458}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.030303,"std":0.011109},{"mean":0.024848,"std":0.010654},{"mean":0.023838,"std":0.008071},{"mean":0.025758,"std":0.002922},{"mean":0.023758,"std":0.001514},{"mean":0.023434,"std":0.002523},{"mean":0.023377,"std":0.000519},{"mean":0.022424,"std":0.000946},{"mean":0.022088,"std":0.001822},{"mean":0.023152,"std":0.002366},{"mean":0.025234,"std":0.002125},{"mean":0.027273,"std":0.003193},{"mean":0.031235,"std":0.002802},{"mean":0.035758,"std":0.003785},{"mean":0.045253,"std":0.00553},{"mean":0.059545,"std":0.006302},{"mean":0.081141,"std":0.009524},{"mean":0.103098,"std":0.009044},{"mean":0.124657,"std":0.010063},{"mean":0.140545,"std":0.007252}],"operating":{"tau":[{"mean":0.655722,"std":0.010977},{"mean":0.631398,"std":0.013017},{"mean":0.612117,"std":0.017304},{"mean":0.593304,"std":0.014858},{"mean":0.573302,"std":0.011721},{"mean":0.557488,"std":0.008969},{"mean":0.54351,"std":0.007137},{"mean":0.532218,"std":0.007119},{"mean":0.520255,"std":0.007903},{"mean":0.509096,"std":0.009281},{"mean":0.497383,"std":0.010373},{"mean":0.484133,"std":0.008996},{"mean":0.470847,"std":0.010322},{"mean":0.455963,"std":0.012048},{"mean":0.437712,"std":0.013334},{"mean":0.418806,"std":0.01386},{"mean":0.395,"std":0.014591},{"mean":0.355577,"std":0.014366},{"mean":0.301142,"std":0.011011},{"mean":0.165618,"std":0.015822}],"coverage":[{"mean":0.039818,"std":0.001616},{"mean":0.082242,"std":0.006091},{"mean":0.125636,"std":0.005953},{"mean":0.168242,"std":0.00942},{"mean":0.207091,"std":0.009183},{"mean":0.248,"std":0.005366},{"mean":0.291818,"std":0.005091},{"mean":0.333879,"std":0.003968},{"mean":0.380788,"std":0.004987},{"mean":0.425152,"std":0.00147},{"mean":0.467091,"std":0.00142},{"mean":0.509455,"std":0.0025},{"mean":0.555273,"std":0.000364},{"mean":0.602061,"std":0.006365},{"mean":0.646242,"std":0.005699},{"mean":0.690788,"std":0.003588},{"mean":0.736485,"std":0.001892},{"mean":0.789152,"std":0.005675},{"mean":0.864303,"std":0.004927},{"mean":0.998909,"std":0.000962}],"selective_risk":[{"mean":0.028703,"std":0.01806},{"mean":0.025655,"std":0.010629},{"mean":0.026733,"std":0.010104},{"mean":0.022086,"std":0.007269},{"mean":0.025198,"std":0.002701},{"mean":0.0242,"std":0.001347},{"mean":0.023444,"std":0.002444},{"mean":0.023231,"std":0.000584},{"mean":0.023404,"std":0.001044},{"mean":0.021813,"std":0.00135},{"mean":0.022706,"std":0.001827},{"mean":0.023554,"std":0.001218},{"mean":0.025432,"std":0.002492},{"mean":0.027455,"std":0.003793},{"mean":0.030645,"std":0.003828},{"mean":0.034903,"std":0.004738},{"mean":0.04229,"std":0.005274},{"mean":0.055416,"std":0.008049},{"mean":0.087117,"std":0.011896},{"mean":0.140519,"std":0.007429}],"accuracy_8":[{"mean":0.820848,"std":0.000105},{"mean":0.821091,"std":0.000182},{"mean":0.821939,"std":0.000555},{"mean":0.825091,"std":0.001845},{"mean":0.827758,"std":0.00502},{"mean":0.831212,"std":0.006864},{"mean":0.834606,"std":0.005948},{"mean":0.83703,"std":0.00537},{"mean":0.839697,"std":0.005819},{"mean":0.841939,"std":0.005976},{"mean":0.845697,"std":0.006874},{"mean":0.849515,"std":0.006833},{"mean":0.853818,"std":0.006793},{"mean":0.861212,"std":0.006531},{"mean":0.868303,"std":0.005249},{"mean":0.874364,"std":0.005403},{"mean":0.87897,"std":0.005534},{"mean":0.884485,"std":0.003},{"mean":0.879818,"std":0.007919},{"mean":0.859455,"std":0.007388}],"high_conf_oos_misroute_rate":[{"mean":0.006333,"std":0.004163},{"mean":0.011333,"std":0.004726},{"mean":0.018333,"std":0.007371},{"mean":0.020333,"std":0.007371},{"mean":0.027667,"std":0.002887},{"mean":0.031,"std":0.002},{"mean":0.034333,"std":0.003215},{"mean":0.039333,"std":0.001155},{"mean":0.045333,"std":0.002517},{"mean":0.046667,"std":0.004726},{"mean":0.053,"std":0.006},{"mean":0.060333,"std":0.004933},{"mean":0.071,"std":0.007},{"mean":0.083,"std":0.011136},{"mean":0.098,"std":0.016093},{"mean":0.116667,"std":0.020008},{"mean":0.146333,"std":0.025106},{"mean":0.204,"std":0.03629},{"mean":0.339667,"std":0.040278},{"mean":0.655,"std":0.036428}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.433333,"std":0.001155},{"mean":0.433333,"std":0.001155},{"mean":0.434,"std":0.001732},{"mean":0.435,"std":0.001732},{"mean":0.436,"std":0.001},{"mean":0.436333,"std":0.001155},{"mean":0.437,"std":0.001},{"mean":0.438333,"std":0.002082},{"mean":0.442667,"std":0.001528},{"mean":0.445667,"std":0.002517},{"mean":0.450333,"std":0.003786},{"mean":0.455667,"std":0.006506},{"mean":0.468667,"std":0.01193},{"mean":0.494333,"std":0.019088},{"mean":0.538667,"std":0.025325},{"mean":0.655667,"std":0.037005}],"llm_call_rate":[{"mean":0.960182,"std":0.001616},{"mean":0.917758,"std":0.006091},{"mean":0.874364,"std":0.005953},{"mean":0.831758,"std":0.00942},{"mean":0.792909,"std":0.009183},{"mean":0.752,"std":0.005366},{"mean":0.708182,"std":0.005091},{"mean":0.666121,"std":0.003968},{"mean":0.619212,"std":0.004987},{"mean":0.574848,"std":0.00147},{"mean":0.532909,"std":0.00142},{"mean":0.490545,"std":0.0025},{"mean":0.444727,"std":0.000364},{"mean":0.397939,"std":0.006365},{"mean":0.353758,"std":0.005699},{"mean":0.309212,"std":0.003588},{"mean":0.263515,"std":0.001892},{"mean":0.210848,"std":0.005675},{"mean":0.135697,"std":0.004927},{"mean":0.001091,"std":0.000962}]}},"entropy":{"risk_coverage":[{"mean":0.027879,"std":0.01277},{"mean":0.025455,"std":0.006298},{"mean":0.022222,"std":0.006892},{"mean":0.023333,"std":0.001892},{"mean":0.021333,"std":0.002753},{"mean":0.021414,"std":0.002799},{"mean":0.021645,"std":0.002163},{"mean":0.021061,"std":0.001837},{"mean":0.02101,"std":0.001212},{"mean":0.021697,"std":0.000555},{"mean":0.024353,"std":0.002652},{"mean":0.027273,"std":0.003415},{"mean":0.030583,"std":0.003929},{"mean":0.037229,"std":0.005657},{"mean":0.046626,"std":0.00586},{"mean":0.062803,"std":0.007732},{"mean":0.083422,"std":0.006428},{"mean":0.109024,"std":0.008981},{"mean":0.134418,"std":0.008731},{"mean":0.140545,"std":0.007252}],"operating":{"tau":[{"mean":-1.232961,"std":0.026818},{"mean":-1.291019,"std":0.032932},{"mean":-1.331361,"std":0.036836},{"mean":-1.365878,"std":0.035214},{"mean":-1.40438,"std":0.028531},{"mean":-1.434641,"std":0.022869},{"mean":-1.461005,"std":0.020815},{"mean":-1.483542,"std":0.019393},{"mean":-1.504598,"std":0.01839},{"mean":-1.527368,"std":0.018747},{"mean":-1.545548,"std":0.01564},{"mean":-1.564658,"std":0.013726},{"mean":-1.58538,"std":0.009443},{"mean":-1.607695,"std":0.010097},{"mean":-1.63186,"std":0.012687},{"mean":-1.660878,"std":0.01724},{"mean":-1.69557,"std":0.015602},{"mean":-1.750998,"std":0.018345},{"mean":-1.82398,"std":0.014742},{"mean":-2.038229,"std":0.033223}],"coverage":[{"mean":0.039515,"std":0.001183},{"mean":0.084182,"std":0.003933},{"mean":0.125818,"std":0.004367},{"mean":0.165879,"std":0.008293},{"mean":0.207091,"std":0.009609},{"mean":0.246242,"std":0.004067},{"mean":0.289697,"std":0.006843},{"mean":0.332424,"std":0.004265},{"mean":0.377212,"std":0.003413},{"mean":0.425939,"std":0.007473},{"mean":0.47103,"std":0.002099},{"mean":0.511273,"std":0.003059},{"mean":0.553818,"std":0.003938},{"mean":0.597939,"std":0.007147},{"mean":0.641455,"std":0.004021},{"mean":0.68903,"std":0.002735},{"mean":0.734182,"std":0.003288},{"mean":0.790303,"std":0.003745},{"mean":0.855394,"std":0.002759},{"mean":0.997697,"std":0.002003}],"selective_risk":[{"mean":0.024652,"std":0.011766},{"mean":0.025082,"std":0.008659},{"mean":0.02354,"std":0.006576},{"mean":0.021414,"std":0.005755},{"mean":0.022592,"std":0.001975},{"mean":0.021683,"std":0.003023},{"mean":0.021557,"std":0.003493},{"mean":0.021515,"std":0.002081},{"mean":0.021374,"std":0.001792},{"mean":0.020477,"std":0.001199},{"mean":0.021103,"std":0.001165},{"mean":0.021691,"std":0.000884},{"mean":0.024409,"std":0.002408},{"mean":0.027248,"std":0.003549},{"mean":0.029838,"std":0.004413},{"mean":0.035434,"std":0.005236},{"mean":0.043329,"std":0.005978},{"mean":0.059556,"std":0.009394},{"mean":0.086,"std":0.00808},{"mean":0.140876,"std":0.007442}],"accuracy_8":[{"mean":0.820848,"std":0.000105},{"mean":0.82103,"std":0.000278},{"mean":0.822909,"std":0.002545},{"mean":0.824545,"std":0.002886},{"mean":0.82697,"std":0.004265},{"mean":0.830545,"std":0.004468},{"mean":0.834485,"std":0.004707},{"mean":0.839333,"std":0.005825},{"mean":0.844242,"std":0.005364},{"mean":0.850364,"std":0.004166},{"mean":0.855818,"std":0.005491},{"mean":0.86,"std":0.004703},{"mean":0.863697,"std":0.005043},{"mean":0.867939,"std":0.004091},{"mean":0.872727,"std":0.005601},{"mean":0.876303,"std":0.005468},{"mean":0.881636,"std":0.003783},{"mean":0.883273,"std":0.005068},{"mean":0.879879,"std":0.007122},{"mean":0.859333,"std":0.007323}],"high_conf_oos_misroute_rate":[{"mean":0.005333,"std":0.002517},{"mean":0.011667,"std":0.004163},{"mean":0.016,"std":0.004583},{"mean":0.019333,"std":0.005686},{"mean":0.024667,"std":0.000577},{"mean":0.027667,"std":0.003512},{"mean":0.031333,"std":0.004041},{"mean":0.035,"std":0.002},{"mean":0.039333,"std":0.000577},{"mean":0.043,"std":0.002646},{"mean":0.048667,"std":0.005686},{"mean":0.053,"std":0.005292},{"mean":0.060667,"std":0.006351},{"mean":0.071,"std":0.004},{"mean":0.082333,"std":0.003512},{"mean":0.102333,"std":0.007572},{"mean":0.131667,"std":0.011015},{"mean":0.201667,"std":0.020429},{"mean":0.327333,"std":0.027755},{"mean":0.655667,"std":0.036964}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.433333,"std":0.001155},{"mean":0.433333,"std":0.001155},{"mean":0.434667,"std":0.001155},{"mean":0.435667,"std":0.002887},{"mean":0.436667,"std":0.002517},{"mean":0.437667,"std":0.002082},{"mean":0.44,"std":0.002},{"mean":0.441333,"std":0.001528},{"mean":0.444667,"std":0.000577},{"mean":0.450667,"std":0.004619},{"mean":0.457,"std":0.008185},{"mean":0.485333,"std":0.012858},{"mean":0.535333,"std":0.021825},{"mean":0.656333,"std":0.037541}],"llm_call_rate":[{"mean":0.960485,"std":0.001183},{"mean":0.915818,"std":0.003933},{"mean":0.874182,"std":0.004367},{"mean":0.834121,"std":0.008293},{"mean":0.792909,"std":0.009609},{"mean":0.753758,"std":0.004067},{"mean":0.710303,"std":0.006843},{"mean":0.667576,"std":0.004265},{"mean":0.622788,"std":0.003413},{"mean":0.574061,"std":0.007473},{"mean":0.52897,"std":0.002099},{"mean":0.488727,"std":0.003059},{"mean":0.446182,"std":0.003938},{"mean":0.402061,"std":0.007147},{"mean":0.358545,"std":0.004021},{"mean":0.31097,"std":0.002735},{"mean":0.265818,"std":0.003288},{"mean":0.209697,"std":0.003745},{"mean":0.144606,"std":0.002759},{"mean":0.002303,"std":0.002003}]}},"margin":{"risk_coverage":[{"mean":0.008485,"std":0.00757},{"mean":0.008485,"std":0.002099},{"mean":0.010505,"std":0.004589},{"mean":0.01,"std":0.003149},{"mean":0.010909,"std":0.001455},{"mean":0.010909,"std":0.00105},{"mean":0.011429,"std":0.0009},{"mean":0.012576,"std":0.001461},{"mean":0.012929,"std":0.00225},{"mean":0.013091,"std":0.001666},{"mean":0.014986,"std":0.002322},{"mean":0.017273,"std":0.002121},{"mean":0.021818,"std":0.003226},{"mean":0.026494,"std":0.005304},{"mean":0.034343,"std":0.006737},{"mean":0.043712,"std":0.006588},{"mean":0.060606,"std":0.005945},{"mean":0.080539,"std":0.007476},{"mean":0.10941,"std":0.007167},{"mean":0.140545,"std":0.007252}],"operating":{"tau":[{"mean":8.5597,"std":0.186437},{"mean":8.135005,"std":0.166252},{"mean":7.765273,"std":0.170781},{"mean":7.469703,"std":0.181539},{"mean":7.196549,"std":0.184651},{"mean":6.930802,"std":0.174771},{"mean":6.662013,"std":0.202289},{"mean":6.459439,"std":0.191998},{"mean":6.261722,"std":0.167635},{"mean":6.07555,"std":0.16142},{"mean":5.846778,"std":0.173857},{"mean":5.623096,"std":0.180961},{"mean":5.324787,"std":0.18766},{"mean":4.961348,"std":0.251182},{"mean":4.583177,"std":0.294283},{"mean":4.169749,"std":0.329034},{"mean":3.616355,"std":0.329278},{"mean":2.780267,"std":0.128751},{"mean":1.573064,"std":0.18541},{"mean":0.015799,"std":0.015514}],"coverage":[{"mean":0.041758,"std":0.002482},{"mean":0.084606,"std":0.004679},{"mean":0.128788,"std":0.004615},{"mean":0.171212,"std":0.007214},{"mean":0.216788,"std":0.007168},{"mean":0.256303,"std":0.010991},{"mean":0.299939,"std":0.003993},{"mean":0.341333,"std":0.003714},{"mean":0.385758,"std":0.005258},{"mean":0.426909,"std":0.009349},{"mean":0.469939,"std":0.005808},{"mean":0.508424,"std":0.005144},{"mean":0.554485,"std":0.005634},{"mean":0.605939,"std":0.010075},{"mean":0.649152,"std":0.012589},{"mean":0.694667,"std":0.012885},{"mean":0.745273,"std":0.013723},{"mean":0.805091,"std":0.002234},{"mean":0.885394,"std":0.007466},{"mean":0.998848,"std":0.001096}],"selective_risk":[{"mean":0.010334,"std":0.009472},{"mean":0.006386,"std":0.003622},{"mean":0.009524,"std":0.004715},{"mean":0.01077,"std":0.00538},{"mean":0.011219,"std":0.001668},{"mean":0.010909,"std":0.001754},{"mean":0.011107,"std":0.001171},{"mean":0.011003,"std":0.001015},{"mean":0.012094,"std":0.001373},{"mean":0.012922,"std":0.002175},{"mean":0.01304,"std":0.002438},{"mean":0.01336,"std":0.001393},{"mean":0.015311,"std":0.001913},{"mean":0.017811,"std":0.001025},{"mean":0.021469,"std":0.002905},{"mean":0.026086,"std":0.00516},{"mean":0.033251,"std":0.00685},{"mean":0.045082,"std":0.006449},{"mean":0.074897,"std":0.010199},{"mean":0.139796,"std":0.007298}],"accuracy_8":[{"mean":0.821273,"std":0.000833},{"mean":0.823091,"std":0.001012},{"mean":0.825394,"std":0.002259},{"mean":0.828364,"std":0.002622},{"mean":0.831273,"std":0.002823},{"mean":0.834182,"std":0.004299},{"mean":0.837515,"std":0.00313},{"mean":0.840788,"std":0.002922},{"mean":0.843939,"std":0.004163},{"mean":0.847273,"std":0.005671},{"mean":0.851697,"std":0.005979},{"mean":0.856,"std":0.006254},{"mean":0.861394,"std":0.007182},{"mean":0.867333,"std":0.007113},{"mean":0.873394,"std":0.005258},{"mean":0.879333,"std":0.005756},{"mean":0.886182,"std":0.005601},{"mean":0.893576,"std":0.004087},{"mean":0.892061,"std":0.006833},{"mean":0.86,"std":0.007388}],"high_conf_oos_misroute_rate":[{"mean":0.002,"std":0.001732},{"mean":0.002667,"std":0.001528},{"mean":0.005667,"std":0.002082},{"mean":0.009,"std":0.004},{"mean":0.012333,"std":0.000577},{"mean":0.014333,"std":0.001155},{"mean":0.017,"std":0.001},{"mean":0.018667,"std":0.002082},{"mean":0.023333,"std":0.003215},{"mean":0.028,"std":0.005292},{"mean":0.031,"std":0.005568},{"mean":0.034,"std":0.003606},{"mean":0.042667,"std":0.005508},{"mean":0.054,"std":0.004359},{"mean":0.069667,"std":0.011015},{"mean":0.090667,"std":0.019858},{"mean":0.122333,"std":0.028006},{"mean":0.179,"std":0.029309},{"mean":0.32,"std":0.043267},{"mean":0.652,"std":0.03985}],"oos_misroute_rate":[{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.433,"std":0.001},{"mean":0.433333,"std":0.001155},{"mean":0.433,"std":0.002646},{"mean":0.432667,"std":0.003215},{"mean":0.433333,"std":0.002082},{"mean":0.435333,"std":0.004041},{"mean":0.435,"std":0.005292},{"mean":0.434667,"std":0.005508},{"mean":0.436667,"std":0.006351},{"mean":0.435333,"std":0.009238},{"mean":0.436,"std":0.010817},{"mean":0.437,"std":0.012124},{"mean":0.443,"std":0.015875},{"mean":0.450333,"std":0.022723},{"mean":0.497,"std":0.034871},{"mean":0.653333,"std":0.038397}],"llm_call_rate":[{"mean":0.958242,"std":0.002482},{"mean":0.915394,"std":0.004679},{"mean":0.871212,"std":0.004615},{"mean":0.828788,"std":0.007214},{"mean":0.783212,"std":0.007168},{"mean":0.743697,"std":0.010991},{"mean":0.700061,"std":0.003993},{"mean":0.658667,"std":0.003714},{"mean":0.614242,"std":0.005258},{"mean":0.573091,"std":0.009349},{"mean":0.530061,"std":0.005808},{"mean":0.491576,"std":0.005144},{"mean":0.445515,"std":0.005634},{"mean":0.394061,"std":0.010075},{"mean":0.350848,"std":0.012589},{"mean":0.305333,"std":0.012885},{"mean":0.254727,"std":0.013723},{"mean":0.194909,"std":0.002234},{"mean":0.114606,"std":0.007466},{"mean":0.001152,"std":0.001096}]}},"msp_t":{"risk_coverage":[{"mean":0.007273,"std":0.007273},{"mean":0.009091,"std":0.001818},{"mean":0.008485,"std":0.002099},{"mean":0.009394,"std":0.002099},{"mean":0.009939,"std":0.00183},{"mean":0.009899,"std":0.0007},{"mean":0.010043,"std":0.0012},{"mean":0.010455,"std":0.001364},{"mean":0.011044,"std":0.001419},{"mean":0.012121,"std":0.001169},{"mean":0.013223,"std":0.001515},{"mean":0.01596,"std":0.002063},{"mean":0.019674,"std":0.002024},{"mean":0.024502,"std":0.004471},{"mean":0.029899,"std":0.005984},{"mean":0.040227,"std":0.00606},{"mean":0.055187,"std":0.004273},{"mean":0.077912,"std":0.008187},{"mean":0.107177,"std":0.006641},{"mean":0.140545,"std":0.007252}],"operating":{"tau":[{"mean":0.999472,"std":0.000102},{"mean":0.999182,"std":0.000128},{"mean":0.998951,"std":0.000155},{"mean":0.998626,"std":0.000242},{"mean":0.998246,"std":0.000272},{"mean":0.997773,"std":0.00036},{"mean":0.99719,"std":0.00049},{"mean":0.996559,"std":0.00052},{"mean":0.995833,"std":0.000621},{"mean":0.994973,"std":0.000899},{"mean":0.993877,"std":0.001113},{"mean":0.992524,"std":0.00136},{"mean":0.9906,"std":0.001837},{"mean":0.987154,"std":0.002918},{"mean":0.98149,"std":0.005201},{"mean":0.972046,"std":0.007942},{"mean":0.957439,"std":0.01178},{"mean":0.914989,"std":0.011583},{"mean":0.76243,"std":0.043426},{"mean":0.228515,"std":0.028899}],"coverage":[{"mean":0.042121,"std":0.005118},{"mean":0.086,"std":0.003988},{"mean":0.125758,"std":0.004866},{"mean":0.170545,"std":0.005671},{"mean":0.211879,"std":0.006546},{"mean":0.255152,"std":0.012165},{"mean":0.298545,"std":0.00993},{"mean":0.339879,"std":0.007273},{"mean":0.383636,"std":0.007593},{"mean":0.427576,"std":0.004203},{"mean":0.473273,"std":0.005464},{"mean":0.513455,"std":0.004542},{"mean":0.553636,"std":0.009455},{"mean":0.599212,"std":0.010686},{"mean":0.643939,"std":0.013299},{"mean":0.689758,"std":0.012639},{"mean":0.738182,"std":0.011073},{"mean":0.797576,"std":0.001866},{"mean":0.876909,"std":0.009793},{"mean":0.999212,"std":0.000688}],"selective_risk":[{"mean":0.007534,"std":0.007428},{"mean":0.009807,"std":0.001942},{"mean":0.007223,"std":0.001345},{"mean":0.008597,"std":0.003158},{"mean":0.009472,"std":0.001773},{"mean":0.010492,"std":0.001333},{"mean":0.009962,"std":0.000908},{"mean":0.010152,"std":0.001246},{"mean":0.010727,"std":0.001233},{"mean":0.010621,"std":0.001868},{"mean":0.01191,"std":0.001674},{"mean":0.012285,"std":0.001767},{"mean":0.01359,"std":0.001452},{"mean":0.015683,"std":0.002157},{"mean":0.01919,"std":0.002041},{"mean":0.023468,"std":0.003839},{"mean":0.028838,"std":0.005518},{"mean":0.039809,"std":0.006242},{"mean":0.066753,"std":0.0112},{"mean":0.140111,"std":0.007054}],"accuracy_8":[{"mean":0.821818,"std":0.000962},{"mean":0.823636,"std":0.000909},{"mean":0.825455,"std":0.001898},{"mean":0.82697,"std":0.002735},{"mean":0.829697,"std":0.002819},{"mean":0.833394,"std":0.004055},{"mean":0.836606,"std":0.004091},{"mean":0.84103,"std":0.005007},{"mean":0.844242,"std":0.006004},{"mean":0.848364,"std":0.006472},{"mean":0.852485,"std":0.006708},{"mean":0.856667,"std":0.006943},{"mean":0.861394,"std":0.008019},{"mean":0.867939,"std":0.008926},{"mean":0.874364,"std":0.006309},{"mean":0.881818,"std":0.005354},{"mean":0.887758,"std":0.004215},{"mean":0.89497,"std":0.003379},{"mean":0.895455,"std":0.005953},{"mean":0.859818,"std":0.007163}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.001155},{"mean":0.004333,"std":0.000577},{"mean":0.004667,"std":0.000577},{"mean":0.007333,"std":0.002517},{"mean":0.01,"std":0.001},{"mean":0.013333,"std":0.000577},{"mean":0.015,"std":0.001},{"mean":0.017,"std":0.002646},{"mean":0.020333,"std":0.003215},{"mean":0.022667,"std":0.004726},{"mean":0.028333,"std":0.004933},{"mean":0.031333,"std":0.004933},{"mean":0.037333,"std":0.003055},{"mean":0.046667,"std":0.005508},{"mean":0.060333,"std":0.008021},{"mean":0.080333,"std":0.016803},{"mean":0.105667,"std":0.022189},{"mean":0.155333,"std":0.027592},{"mean":0.277667,"std":0.045081},{"mean":0.652667,"std":0.037846}],"oos_misroute_rate":[{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.433,"std":0.001},{"mean":0.433,"std":0.001},{"mean":0.433333,"std":0.001528},{"mean":0.433667,"std":0.001528},{"mean":0.433,"std":0.002646},{"mean":0.433667,"std":0.003215},{"mean":0.434333,"std":0.004041},{"mean":0.435667,"std":0.005508},{"mean":0.435333,"std":0.005859},{"mean":0.435333,"std":0.007095},{"mean":0.435667,"std":0.007371},{"mean":0.436,"std":0.010536},{"mean":0.439667,"std":0.013051},{"mean":0.449667,"std":0.021221},{"mean":0.479333,"std":0.036679},{"mean":0.653667,"std":0.03782}],"llm_call_rate":[{"mean":0.957879,"std":0.005118},{"mean":0.914,"std":0.003988},{"mean":0.874242,"std":0.004866},{"mean":0.829455,"std":0.005671},{"mean":0.788121,"std":0.006546},{"mean":0.744848,"std":0.012165},{"mean":0.701455,"std":0.00993},{"mean":0.660121,"std":0.007273},{"mean":0.616364,"std":0.007593},{"mean":0.572424,"std":0.004203},{"mean":0.526727,"std":0.005464},{"mean":0.486545,"std":0.004542},{"mean":0.446364,"std":0.009455},{"mean":0.400788,"std":0.010686},{"mean":0.356061,"std":0.013299},{"mean":0.310242,"std":0.012639},{"mean":0.261818,"std":0.011073},{"mean":0.202424,"std":0.001866},{"mean":0.123091,"std":0.009793},{"mean":0.000788,"std":0.000688}]}}}}},"bert/k5":{"model":"bert","k":5,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["bert-base-uncased-k5-seed42","bert-base-uncased-k5-seed43","bert-base-uncased-k5-seed44"],"logits_sha256":["12e3d723ecbdeda888a687bd3b539ef5b8bc07515bd72f28293b0dfc4314f94d","78dd86d8b8a700aa8986c7bf23fef4a99720a8f226df3e6d0d21d896cae3aa8a","175a274248600f6f85681b2e8a824dd9bf17f281782044110b1c3facb10f1da5"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.003636,"std":0.003636},{"mean":0.006667,"std":0.003785},{"mean":0.008889,"std":0.0007},{"mean":0.01,"std":0.000909},{"mean":0.012121,"std":0.002222},{"mean":0.014545,"std":0.001212},{"mean":0.016623,"std":0.001374},{"mean":0.020303,"std":0.002957},{"mean":0.026532,"std":0.005888},{"mean":0.034303,"std":0.009901},{"mean":0.040661,"std":0.011254},{"mean":0.047475,"std":0.012444},{"mean":0.057995,"std":0.013299},{"mean":0.069004,"std":0.01468},{"mean":0.084687,"std":0.01735},{"mean":0.101288,"std":0.018182},{"mean":0.124135,"std":0.02058},{"mean":0.150842,"std":0.020327},{"mean":0.179841,"std":0.020432},{"mean":0.211879,"std":0.019711}],"operating":{"tau":[{"mean":0.180304,"std":0.015792},{"mean":0.155671,"std":0.012933},{"mean":0.139291,"std":0.010988},{"mean":0.127491,"std":0.010838},{"mean":0.117093,"std":0.009352},{"mean":0.108608,"std":0.00931},{"mean":0.100823,"std":0.008606},{"mean":0.093844,"std":0.007975},{"mean":0.087288,"std":0.007366},{"mean":0.081034,"std":0.006707},{"mean":0.075238,"std":0.006448},{"mean":0.070279,"std":0.005862},{"mean":0.064923,"std":0.005199},{"mean":0.060275,"std":0.004791},{"mean":0.05536,"std":0.003635},{"mean":0.050336,"std":0.003256},{"mean":0.044326,"std":0.001662},{"mean":0.038587,"std":0.001036},{"mean":0.031761,"std":0.00202},{"mean":0.016707,"std":0.001371}],"coverage":[{"mean":0.044242,"std":0.002548},{"mean":0.088242,"std":0.004572},{"mean":0.133879,"std":0.002345},{"mean":0.175152,"std":0.002222},{"mean":0.219515,"std":0.003815},{"mean":0.260121,"std":0.002401},{"mean":0.305091,"std":0.006801},{"mean":0.350424,"std":0.00674},{"mean":0.394182,"std":0.006298},{"mean":0.440727,"std":0.002993},{"mean":0.48297,"std":0.006118},{"mean":0.526,"std":0.00778},{"mean":0.574,"std":0.004717},{"mean":0.615333,"std":0.008982},{"mean":0.659515,"std":0.008874},{"mean":0.710121,"std":0.012312},{"mean":0.769636,"std":0.012055},{"mean":0.82697,"std":0.008908},{"mean":0.900061,"std":0.011123},{"mean":0.999394,"std":0.000735}],"selective_risk":[{"mean":0.00396,"std":0.003893},{"mean":0.006947,"std":0.003348},{"mean":0.009062,"std":0.0009},{"mean":0.009332,"std":0.0017},{"mean":0.010499,"std":0.000651},{"mean":0.013059,"std":0.002529},{"mean":0.014709,"std":0.000874},{"mean":0.016762,"std":0.001935},{"mean":0.019824,"std":0.003101},{"mean":0.025007,"std":0.006138},{"mean":0.031698,"std":0.009373},{"mean":0.037077,"std":0.010868},{"mean":0.043646,"std":0.01245},{"mean":0.050438,"std":0.013426},{"mean":0.060237,"std":0.012092},{"mean":0.071969,"std":0.014737},{"mean":0.091004,"std":0.012921},{"mean":0.113766,"std":0.015588},{"mean":0.150467,"std":0.024964},{"mean":0.211394,"std":0.020127}],"accuracy_8":[{"mean":0.823333,"std":0.000915},{"mean":0.826667,"std":0.002956},{"mean":0.830788,"std":0.005007},{"mean":0.834606,"std":0.006367},{"mean":0.838364,"std":0.006727},{"mean":0.842182,"std":0.007119},{"mean":0.846485,"std":0.007458},{"mean":0.851273,"std":0.006636},{"mean":0.855515,"std":0.004859},{"mean":0.859879,"std":0.004576},{"mean":0.862,"std":0.005197},{"mean":0.864909,"std":0.006771},{"mean":0.866303,"std":0.007546},{"mean":0.867333,"std":0.007863},{"mean":0.866545,"std":0.007388},{"mean":0.865333,"std":0.007997},{"mean":0.860909,"std":0.008092},{"mean":0.851758,"std":0.010423},{"mean":0.832424,"std":0.017078},{"mean":0.788485,"std":0.019929}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.003667,"std":0.002517},{"mean":0.005,"std":0.002},{"mean":0.007,"std":0.004},{"mean":0.009333,"std":0.005132},{"mean":0.011667,"std":0.006028},{"mean":0.014333,"std":0.006351},{"mean":0.019333,"std":0.007234},{"mean":0.025333,"std":0.006658},{"mean":0.035,"std":0.009849},{"mean":0.046,"std":0.014},{"mean":0.058667,"std":0.017898},{"mean":0.075,"std":0.020952},{"mean":0.101667,"std":0.018502},{"mean":0.136667,"std":0.022368},{"mean":0.198667,"std":0.023288},{"mean":0.281667,"std":0.03421},{"mean":0.443667,"std":0.084453},{"mean":0.769333,"std":0.074299}],"oos_misroute_rate":[{"mean":0.431333,"std":0.001155},{"mean":0.431,"std":0.001732},{"mean":0.430333,"std":0.002887},{"mean":0.430333,"std":0.002887},{"mean":0.43,"std":0.002646},{"mean":0.429333,"std":0.002887},{"mean":0.429,"std":0.002646},{"mean":0.429333,"std":0.003512},{"mean":0.429,"std":0.002},{"mean":0.428,"std":0.001},{"mean":0.429,"std":0.003},{"mean":0.431667,"std":0.005033},{"mean":0.433333,"std":0.007024},{"mean":0.438667,"std":0.009713},{"mean":0.448,"std":0.009644},{"mean":0.456333,"std":0.008963},{"mean":0.473667,"std":0.014978},{"mean":0.504,"std":0.02358},{"mean":0.579667,"std":0.057012},{"mean":0.770667,"std":0.072858}],"llm_call_rate":[{"mean":0.955758,"std":0.002548},{"mean":0.911758,"std":0.004572},{"mean":0.866121,"std":0.002345},{"mean":0.824848,"std":0.002222},{"mean":0.780485,"std":0.003815},{"mean":0.739879,"std":0.002401},{"mean":0.694909,"std":0.006801},{"mean":0.649576,"std":0.00674},{"mean":0.605818,"std":0.006298},{"mean":0.559273,"std":0.002993},{"mean":0.51703,"std":0.006118},{"mean":0.474,"std":0.00778},{"mean":0.426,"std":0.004717},{"mean":0.384667,"std":0.008982},{"mean":0.340485,"std":0.008874},{"mean":0.289879,"std":0.012312},{"mean":0.230364,"std":0.012055},{"mean":0.17303,"std":0.008908},{"mean":0.099939,"std":0.011123},{"mean":0.000606,"std":0.000735}]}},"entropy":{"risk_coverage":[{"mean":0.004848,"std":0.005555},{"mean":0.009697,"std":0.00757},{"mean":0.014545,"std":0.006061},{"mean":0.021212,"std":0.007732},{"mean":0.025455,"std":0.010182},{"mean":0.030101,"std":0.008578},{"mean":0.035152,"std":0.007952},{"mean":0.04,"std":0.011309},{"mean":0.045791,"std":0.01067},{"mean":0.053455,"std":0.010842},{"mean":0.060937,"std":0.011343},{"mean":0.067778,"std":0.010978},{"mean":0.078695,"std":0.012448},{"mean":0.091688,"std":0.016883},{"mean":0.105293,"std":0.01697},{"mean":0.121818,"std":0.017894},{"mean":0.140036,"std":0.019099},{"mean":0.162626,"std":0.02117},{"mean":0.186411,"std":0.02046},{"mean":0.211879,"std":0.019711}],"operating":{"tau":[{"mean":-4.362326,"std":0.042981},{"mean":-4.424036,"std":0.031863},{"mean":-4.463788,"std":0.030297},{"mean":-4.4964,"std":0.029925},{"mean":-4.520769,"std":0.030657},{"mean":-4.542669,"std":0.029096},{"mean":-4.562002,"std":0.025386},{"mean":-4.580434,"std":0.02377},{"mean":-4.598311,"std":0.021206},{"mean":-4.617134,"std":0.020529},{"mean":-4.635032,"std":0.019977},{"mean":-4.65208,"std":0.019963},{"mean":-4.671136,"std":0.018647},{"mean":-4.690468,"std":0.017829},{"mean":-4.711456,"std":0.017727},{"mean":-4.732438,"std":0.015601},{"mean":-4.75623,"std":0.014539},{"mean":-4.785395,"std":0.013094},{"mean":-4.825911,"std":0.012471},{"mean":-4.943473,"std":0.002816}],"coverage":[{"mean":0.045455,"std":0.001106},{"mean":0.08897,"std":0.00191},{"mean":0.130848,"std":0.003135},{"mean":0.174121,"std":0.008967},{"mean":0.217091,"std":0.008706},{"mean":0.261758,"std":0.010208},{"mean":0.298485,"std":0.009189},{"mean":0.342,"std":0.009278},{"mean":0.384545,"std":0.00769},{"mean":0.431394,"std":0.008185},{"mean":0.475394,"std":0.007757},{"mean":0.51703,"std":0.007791},{"mean":0.561212,"std":0.009841},{"mean":0.610909,"std":0.007677},{"mean":0.662,"std":0.009354},{"mean":0.714061,"std":0.011093},{"mean":0.768,"std":0.005852},{"mean":0.83103,"std":0.007061},{"mean":0.90303,"std":0.009344},{"mean":0.999818,"std":0.000315}],"selective_risk":[{"mean":0.005415,"std":0.006211},{"mean":0.007582,"std":0.007958},{"mean":0.014003,"std":0.006753},{"mean":0.017829,"std":0.004435},{"mean":0.023542,"std":0.011485},{"mean":0.027525,"std":0.011451},{"mean":0.030262,"std":0.00953},{"mean":0.033865,"std":0.009108},{"mean":0.038048,"std":0.011948},{"mean":0.042754,"std":0.012094},{"mean":0.050784,"std":0.012306},{"mean":0.055381,"std":0.011935},{"mean":0.062676,"std":0.011996},{"mean":0.070653,"std":0.012501},{"mean":0.082458,"std":0.016049},{"mean":0.094771,"std":0.018241},{"mean":0.111283,"std":0.019279},{"mean":0.133289,"std":0.020906},{"mean":0.16406,"std":0.025777},{"mean":0.2118,"std":0.019578}],"accuracy_8":[{"mean":0.824606,"std":0.002107},{"mean":0.827455,"std":0.004229},{"mean":0.829939,"std":0.004365},{"mean":0.833818,"std":0.003912},{"mean":0.836364,"std":0.003933},{"mean":0.839333,"std":0.003989},{"mean":0.841636,"std":0.004502},{"mean":0.844909,"std":0.005184},{"mean":0.847576,"std":0.003611},{"mean":0.849636,"std":0.002886},{"mean":0.850727,"std":0.002405},{"mean":0.852303,"std":0.002469},{"mean":0.85497,"std":0.004131},{"mean":0.856061,"std":0.006053},{"mean":0.85497,"std":0.00814},{"mean":0.853333,"std":0.009782},{"mean":0.848303,"std":0.01122},{"mean":0.840121,"std":0.013324},{"mean":0.824121,"std":0.019742},{"mean":0.788242,"std":0.019507}],"high_conf_oos_misroute_rate":[{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.000577},{"mean":0.004,"std":0.0},{"mean":0.007667,"std":0.002517},{"mean":0.011333,"std":0.004933},{"mean":0.017,"std":0.008718},{"mean":0.02,"std":0.00866},{"mean":0.025667,"std":0.009815},{"mean":0.029,"std":0.010392},{"mean":0.037333,"std":0.012702},{"mean":0.050333,"std":0.014434},{"mean":0.060333,"std":0.014468},{"mean":0.08,"std":0.014933},{"mean":0.104667,"std":0.016563},{"mean":0.142667,"std":0.026502},{"mean":0.187333,"std":0.039311},{"mean":0.247333,"std":0.051287},{"mean":0.344333,"std":0.069328},{"mean":0.495333,"std":0.094817},{"mean":0.772,"std":0.070661}],"oos_misroute_rate":[{"mean":0.431333,"std":0.001155},{"mean":0.431333,"std":0.001155},{"mean":0.430667,"std":0.001528},{"mean":0.429667,"std":0.002082},{"mean":0.429,"std":0.002646},{"mean":0.429667,"std":0.002887},{"mean":0.429333,"std":0.003055},{"mean":0.428333,"std":0.004041},{"mean":0.428333,"std":0.004509},{"mean":0.429,"std":0.003606},{"mean":0.432667,"std":0.004726},{"mean":0.433667,"std":0.005508},{"mean":0.436333,"std":0.009238},{"mean":0.444333,"std":0.009713},{"mean":0.458333,"std":0.01097},{"mean":0.472,"std":0.017088},{"mean":0.497667,"std":0.031533},{"mean":0.54,"std":0.049},{"mean":0.607667,"std":0.0727},{"mean":0.772,"std":0.070661}],"llm_call_rate":[{"mean":0.954545,"std":0.001106},{"mean":0.91103,"std":0.00191},{"mean":0.869152,"std":0.003135},{"mean":0.825879,"std":0.008967},{"mean":0.782909,"std":0.008706},{"mean":0.738242,"std":0.010208},{"mean":0.701515,"std":0.009189},{"mean":0.658,"std":0.009278},{"mean":0.615455,"std":0.00769},{"mean":0.568606,"std":0.008185},{"mean":0.524606,"std":0.007757},{"mean":0.48297,"std":0.007791},{"mean":0.438788,"std":0.009841},{"mean":0.389091,"std":0.007677},{"mean":0.338,"std":0.009354},{"mean":0.285939,"std":0.011093},{"mean":0.232,"std":0.005852},{"mean":0.16897,"std":0.007061},{"mean":0.09697,"std":0.009344},{"mean":0.000182,"std":0.000315}]}},"margin":{"risk_coverage":[{"mean":0.003636,"std":0.006298},{"mean":0.010303,"std":0.008398},{"mean":0.011313,"std":0.003896},{"mean":0.013333,"std":0.00105},{"mean":0.018182,"std":0.004364},{"mean":0.022424,"std":0.005781},{"mean":0.025455,"std":0.005619},{"mean":0.031364,"std":0.009844},{"mean":0.040269,"std":0.010889},{"mean":0.050667,"std":0.011751},{"mean":0.0627,"std":0.014486},{"mean":0.077778,"std":0.018906},{"mean":0.092308,"std":0.020511},{"mean":0.107792,"std":0.021464},{"mean":0.126222,"std":0.021467},{"mean":0.142652,"std":0.021393},{"mean":0.162068,"std":0.021837},{"mean":0.179596,"std":0.022602},{"mean":0.195789,"std":0.020641},{"mean":0.211879,"std":0.019711}],"operating":{"tau":[{"mean":6.707974,"std":0.11538},{"mean":5.856177,"std":0.151323},{"mean":5.240098,"std":0.239159},{"mean":4.781078,"std":0.189685},{"mean":4.34317,"std":0.174228},{"mean":3.958594,"std":0.186309},{"mean":3.583256,"std":0.225382},{"mean":3.203626,"std":0.244372},{"mean":2.821085,"std":0.235721},{"mean":2.486729,"std":0.260157},{"mean":2.169192,"std":0.21814},{"mean":1.867911,"std":0.225433},{"mean":1.55642,"std":0.202718},{"mean":1.301346,"std":0.1769},{"mean":1.045066,"std":0.14132},{"mean":0.787802,"std":0.105378},{"mean":0.567855,"std":0.049951},{"mean":0.362739,"std":0.029817},{"mean":0.171799,"std":0.016152},{"mean":0.000924,"std":0.000484}],"coverage":[{"mean":0.046545,"std":0.002823},{"mean":0.092848,"std":0.003182},{"mean":0.138424,"std":0.006629},{"mean":0.180182,"std":0.006813},{"mean":0.227515,"std":0.002548},{"mean":0.274424,"std":0.00134},{"mean":0.315576,"std":0.001944},{"mean":0.362,"std":0.006322},{"mean":0.408788,"std":0.003683},{"mean":0.454182,"std":0.009557},{"mean":0.498,"std":0.006371},{"mean":0.544121,"std":0.007572},{"mean":0.600121,"std":0.008575},{"mean":0.646545,"std":0.010351},{"mean":0.699273,"std":0.008286},{"mean":0.757091,"std":0.009578},{"mean":0.813394,"std":0.005376},{"mean":0.874121,"std":0.005053},{"mean":0.937212,"std":0.002184},{"mean":0.999697,"std":0.000278}],"selective_risk":[{"mean":0.00369,"std":0.006391},{"mean":0.007789,"std":0.007054},{"mean":0.010832,"std":0.004805},{"mean":0.013093,"std":0.00249},{"mean":0.015171,"std":0.002754},{"mean":0.019442,"std":0.004302},{"mean":0.023059,"std":0.005188},{"mean":0.026322,"std":0.00668},{"mean":0.032365,"std":0.010182},{"mean":0.041087,"std":0.012726},{"mean":0.050516,"std":0.010755},{"mean":0.061619,"std":0.015415},{"mean":0.077849,"std":0.019887},{"mean":0.091535,"std":0.023568},{"mean":0.106682,"std":0.024035},{"mean":0.12831,"std":0.024665},{"mean":0.148334,"std":0.023313},{"mean":0.171231,"std":0.024102},{"mean":0.192067,"std":0.020592},{"mean":0.211702,"std":0.019595}],"accuracy_8":[{"mean":0.822909,"std":0.000182},{"mean":0.827273,"std":0.001311},{"mean":0.830364,"std":0.001012},{"mean":0.835152,"std":0.00134},{"mean":0.839576,"std":0.001944},{"mean":0.844727,"std":0.003338},{"mean":0.848545,"std":0.003455},{"mean":0.853394,"std":0.003182},{"mean":0.857273,"std":0.004166},{"mean":0.859818,"std":0.003367},{"mean":0.86103,"std":0.003403},{"mean":0.861515,"std":0.005053},{"mean":0.858545,"std":0.00882},{"mean":0.856303,"std":0.01211},{"mean":0.852,"std":0.012896},{"mean":0.84303,"std":0.015196},{"mean":0.833152,"std":0.015806},{"mean":0.819273,"std":0.018282},{"mean":0.803394,"std":0.01849},{"mean":0.788242,"std":0.019638}],"high_conf_oos_misroute_rate":[{"mean":0.000667,"std":0.001155},{"mean":0.002,"std":0.002646},{"mean":0.004667,"std":0.002517},{"mean":0.007333,"std":0.000577},{"mean":0.011,"std":0.002646},{"mean":0.016,"std":0.005196},{"mean":0.022,"std":0.007},{"mean":0.031333,"std":0.012342},{"mean":0.045,"std":0.016703},{"mean":0.066,"std":0.028792},{"mean":0.084667,"std":0.026652},{"mean":0.113667,"std":0.037528},{"mean":0.16,"std":0.050319},{"mean":0.208667,"std":0.064532},{"mean":0.261333,"std":0.065851},{"mean":0.348,"std":0.074806},{"mean":0.432667,"std":0.071452},{"mean":0.54,"std":0.073082},{"mean":0.652,"std":0.064366},{"mean":0.771667,"std":0.071696}],"oos_misroute_rate":[{"mean":0.431,"std":0.001732},{"mean":0.430333,"std":0.002887},{"mean":0.430333,"std":0.002082},{"mean":0.430667,"std":0.001155},{"mean":0.431667,"std":0.001528},{"mean":0.432667,"std":0.004163},{"mean":0.434333,"std":0.005686},{"mean":0.435,"std":0.009165},{"mean":0.436667,"std":0.01193},{"mean":0.441333,"std":0.015503},{"mean":0.443667,"std":0.014154},{"mean":0.453667,"std":0.020306},{"mean":0.474667,"std":0.033232},{"mean":0.494333,"std":0.044287},{"mean":0.516667,"std":0.047501},{"mean":0.557667,"std":0.057709},{"mean":0.595333,"std":0.056889},{"mean":0.648,"std":0.06338},{"mean":0.708333,"std":0.064361},{"mean":0.772333,"std":0.072037}],"llm_call_rate":[{"mean":0.953455,"std":0.002823},{"mean":0.907152,"std":0.003182},{"mean":0.861576,"std":0.006629},{"mean":0.819818,"std":0.006813},{"mean":0.772485,"std":0.002548},{"mean":0.725576,"std":0.00134},{"mean":0.684424,"std":0.001944},{"mean":0.638,"std":0.006322},{"mean":0.591212,"std":0.003683},{"mean":0.545818,"std":0.009557},{"mean":0.502,"std":0.006371},{"mean":0.455879,"std":0.007572},{"mean":0.399879,"std":0.008575},{"mean":0.353455,"std":0.010351},{"mean":0.300727,"std":0.008286},{"mean":0.242909,"std":0.009578},{"mean":0.186606,"std":0.005376},{"mean":0.125879,"std":0.005053},{"mean":0.062788,"std":0.002184},{"mean":0.000303,"std":0.000278}]}},"msp_t":{"risk_coverage":[{"mean":0.004848,"std":0.008398},{"mean":0.007879,"std":0.002777},{"mean":0.009293,"std":0.001852},{"mean":0.01,"std":0.002405},{"mean":0.013818,"std":0.002519},{"mean":0.01596,"std":0.003703},{"mean":0.019567,"std":0.005255},{"mean":0.024091,"std":0.005961},{"mean":0.030438,"std":0.008323},{"mean":0.037455,"std":0.008818},{"mean":0.046171,"std":0.013158},{"mean":0.058384,"std":0.013334},{"mean":0.068904,"std":0.014931},{"mean":0.081299,"std":0.016775},{"mean":0.095515,"std":0.019133},{"mean":0.110682,"std":0.019005},{"mean":0.131052,"std":0.020402},{"mean":0.153939,"std":0.020029},{"mean":0.179904,"std":0.020486},{"mean":0.211879,"std":0.019711}],"operating":{"tau":[{"mean":0.994499,"std":0.001158},{"mean":0.988131,"std":0.002368},{"mean":0.98025,"std":0.004093},{"mean":0.971149,"std":0.005443},{"mean":0.958874,"std":0.007561},{"mean":0.943327,"std":0.013097},{"mean":0.921257,"std":0.017632},{"mean":0.893574,"std":0.024159},{"mean":0.858321,"std":0.032312},{"mean":0.811471,"std":0.041629},{"mean":0.75862,"std":0.048877},{"mean":0.704814,"std":0.051599},{"mean":0.647613,"std":0.050875},{"mean":0.58586,"std":0.048424},{"mean":0.524799,"std":0.041548},{"mean":0.460723,"std":0.039394},{"mean":0.395956,"std":0.031889},{"mean":0.320024,"std":0.021534},{"mean":0.2363,"std":0.004843},{"mean":0.077313,"std":0.011096}],"coverage":[{"mean":0.046848,"std":0.00371},{"mean":0.090788,"std":0.00134},{"mean":0.137091,"std":0.00381},{"mean":0.180788,"std":0.003858},{"mean":0.223394,"std":0.002131},{"mean":0.264121,"std":0.005131},{"mean":0.310485,"std":0.001866},{"mean":0.351636,"std":0.002751},{"mean":0.397455,"std":0.001616},{"mean":0.445333,"std":0.002735},{"mean":0.492,"std":0.003933},{"mean":0.537212,"std":0.007005},{"mean":0.584,"std":0.011389},{"mean":0.630909,"std":0.008797},{"mean":0.676909,"std":0.010177},{"mean":0.728667,"std":0.009563},{"mean":0.781455,"std":0.00381},{"mean":0.842,"std":0.003912},{"mean":0.908545,"std":0.009226},{"mean":0.998727,"std":0.001192}],"selective_risk":[{"mean":0.004762,"std":0.008248},{"mean":0.008668,"std":0.002987},{"mean":0.008847,"std":0.002747},{"mean":0.00976,"std":0.002721},{"mean":0.011954,"std":0.002717},{"mean":0.014448,"std":0.002306},{"mean":0.016789,"std":0.0036},{"mean":0.019468,"std":0.005188},{"mean":0.023634,"std":0.005492},{"mean":0.029361,"std":0.008498},{"mean":0.036425,"std":0.009393},{"mean":0.04412,"std":0.011886},{"mean":0.053545,"std":0.011355},{"mean":0.064433,"std":0.01258},{"mean":0.076502,"std":0.013574},{"mean":0.089541,"std":0.015047},{"mean":0.106148,"std":0.018247},{"mean":0.127358,"std":0.020852},{"mean":0.158232,"std":0.021473},{"mean":0.21092,"std":0.020521}],"accuracy_8":[{"mean":0.823212,"std":0.000555},{"mean":0.826667,"std":0.001525},{"mean":0.830848,"std":0.003315},{"mean":0.834909,"std":0.003107},{"mean":0.838788,"std":0.002599},{"mean":0.843091,"std":0.004252},{"mean":0.848242,"std":0.003922},{"mean":0.852848,"std":0.004637},{"mean":0.856727,"std":0.003796},{"mean":0.860424,"std":0.003918},{"mean":0.863515,"std":0.005833},{"mean":0.865636,"std":0.005911},{"mean":0.866242,"std":0.00611},{"mean":0.866424,"std":0.006943},{"mean":0.864545,"std":0.008346},{"mean":0.861455,"std":0.009042},{"mean":0.854848,"std":0.010885},{"mean":0.844485,"std":0.013299},{"mean":0.826909,"std":0.01561},{"mean":0.788848,"std":0.020234}],"high_conf_oos_misroute_rate":[{"mean":0.000667,"std":0.001155},{"mean":0.002333,"std":0.002082},{"mean":0.004,"std":0.003464},{"mean":0.006,"std":0.003464},{"mean":0.007667,"std":0.003215},{"mean":0.01,"std":0.002646},{"mean":0.015333,"std":0.005508},{"mean":0.02,"std":0.006928},{"mean":0.027667,"std":0.010693},{"mean":0.038667,"std":0.016623},{"mean":0.053333,"std":0.016563},{"mean":0.07,"std":0.021656},{"mean":0.090333,"std":0.021502},{"mean":0.118,"std":0.02623},{"mean":0.155667,"std":0.02616},{"mean":0.200667,"std":0.029023},{"mean":0.266333,"std":0.041669},{"mean":0.349667,"std":0.060863},{"mean":0.483333,"std":0.068646},{"mean":0.766,"std":0.077149}],"oos_misroute_rate":[{"mean":0.431,"std":0.001732},{"mean":0.430333,"std":0.002887},{"mean":0.430333,"std":0.002887},{"mean":0.430333,"std":0.002887},{"mean":0.43,"std":0.002},{"mean":0.430333,"std":0.002517},{"mean":0.431667,"std":0.003055},{"mean":0.430667,"std":0.002517},{"mean":0.431,"std":0.004},{"mean":0.431333,"std":0.006028},{"mean":0.433667,"std":0.010066},{"mean":0.438667,"std":0.011504},{"mean":0.446333,"std":0.011372},{"mean":0.452,"std":0.01179},{"mean":0.464333,"std":0.015044},{"mean":0.480333,"std":0.022189},{"mean":0.505,"std":0.033181},{"mean":0.541333,"std":0.045391},{"mean":0.605667,"std":0.049014},{"mean":0.768667,"std":0.074574}],"llm_call_rate":[{"mean":0.953152,"std":0.00371},{"mean":0.909212,"std":0.00134},{"mean":0.862909,"std":0.00381},{"mean":0.819212,"std":0.003858},{"mean":0.776606,"std":0.002131},{"mean":0.735879,"std":0.005131},{"mean":0.689515,"std":0.001866},{"mean":0.648364,"std":0.002751},{"mean":0.602545,"std":0.001616},{"mean":0.554667,"std":0.002735},{"mean":0.508,"std":0.003933},{"mean":0.462788,"std":0.007005},{"mean":0.416,"std":0.011389},{"mean":0.369091,"std":0.008797},{"mean":0.323091,"std":0.010177},{"mean":0.271333,"std":0.009563},{"mean":0.218545,"std":0.00381},{"mean":0.158,"std":0.003912},{"mean":0.091455,"std":0.009226},{"mean":0.001273,"std":0.001192}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.05697,"std":0.00757},{"mean":0.046667,"std":0.005249},{"mean":0.054949,"std":0.005728},{"mean":0.058182,"std":0.007767},{"mean":0.062061,"std":0.009901},{"mean":0.070909,"std":0.005387},{"mean":0.080346,"std":0.002342},{"mean":0.088333,"std":0.007348},{"mean":0.0967,"std":0.010761},{"mean":0.104606,"std":0.013333},{"mean":0.112397,"std":0.013432},{"mean":0.120808,"std":0.01342},{"mean":0.130629,"std":0.012708},{"mean":0.140606,"std":0.016065},{"mean":0.151192,"std":0.013074},{"mean":0.161439,"std":0.015425},{"mean":0.172906,"std":0.013911},{"mean":0.185859,"std":0.012173},{"mean":0.197959,"std":0.013676},{"mean":0.210061,"std":0.016089}],"operating":{"tau":[{"mean":0.491247,"std":0.011682},{"mean":0.462267,"std":0.012803},{"mean":0.429119,"std":0.008537},{"mean":0.399331,"std":0.005595},{"mean":0.379624,"std":0.005266},{"mean":0.360946,"std":0.003191},{"mean":0.344592,"std":0.000569},{"mean":0.330661,"std":0.001515},{"mean":0.31814,"std":0.00245},{"mean":0.307472,"std":0.00244},{"mean":0.296936,"std":0.003207},{"mean":0.287282,"std":0.003305},{"mean":0.277241,"std":0.002366},{"mean":0.268872,"std":0.002031},{"mean":0.260514,"std":0.001926},{"mean":0.2523,"std":0.001666},{"mean":0.243535,"std":0.00128},{"mean":0.232741,"std":0.002455},{"mean":0.220164,"std":0.003189},{"mean":0.179169,"std":0.004744}],"coverage":[{"mean":0.040909,"std":0.006254},{"mean":0.083636,"std":0.002271},{"mean":0.127212,"std":0.001034},{"mean":0.174303,"std":0.00021},{"mean":0.21697,"std":0.002259},{"mean":0.261212,"std":0.005948},{"mean":0.308121,"std":0.005912},{"mean":0.355394,"std":0.002455},{"mean":0.401636,"std":0.000962},{"mean":0.446485,"std":0.004026},{"mean":0.500061,"std":0.006132},{"mean":0.545576,"std":0.009507},{"mean":0.597515,"std":0.013828},{"mean":0.648,"std":0.016443},{"mean":0.699091,"std":0.013886},{"mean":0.747879,"std":0.011494},{"mean":0.80103,"std":0.010094},{"mean":0.863576,"std":0.005468},{"mean":0.921879,"std":0.004964},{"mean":0.99897,"std":0.001481}],"selective_risk":[{"mean":0.049986,"std":0.014558},{"mean":0.046328,"std":0.006255},{"mean":0.051914,"std":0.005186},{"mean":0.054942,"std":0.006843},{"mean":0.058681,"std":0.009028},{"mean":0.063911,"std":0.008848},{"mean":0.073001,"std":0.002156},{"mean":0.08201,"std":0.00354},{"mean":0.088265,"std":0.007391},{"mean":0.09623,"std":0.011429},{"mean":0.104293,"std":0.013862},{"mean":0.112748,"std":0.013877},{"mean":0.119879,"std":0.012813},{"mean":0.131125,"std":0.013288},{"mean":0.140452,"std":0.015588},{"mean":0.15053,"std":0.011712},{"mean":0.161652,"std":0.0145},{"mean":0.176762,"std":0.011433},{"mean":0.191672,"std":0.010877},{"mean":0.209843,"std":0.016086}],"accuracy_8":[{"mean":0.820909,"std":0.000481},{"mean":0.821455,"std":0.000545},{"mean":0.821879,"std":0.000933},{"mean":0.822364,"std":0.000909},{"mean":0.824485,"std":0.000278},{"mean":0.826727,"std":0.001012},{"mean":0.829333,"std":0.001568},{"mean":0.831273,"std":0.003102},{"mean":0.832667,"std":0.003},{"mean":0.834061,"std":0.003737},{"mean":0.835697,"std":0.003888},{"mean":0.836121,"std":0.004967},{"mean":0.835697,"std":0.004866},{"mean":0.833515,"std":0.004261},{"mean":0.829394,"std":0.006225},{"mean":0.826303,"std":0.005289},{"mean":0.822061,"std":0.008915},{"mean":0.814485,"std":0.009128},{"mean":0.805091,"std":0.009557},{"mean":0.790303,"std":0.016187}],"high_conf_oos_misroute_rate":[{"mean":0.010333,"std":0.004726},{"mean":0.019,"std":0.002646},{"mean":0.033,"std":0.004},{"mean":0.046333,"std":0.004163},{"mean":0.058,"std":0.006557},{"mean":0.073333,"std":0.006506},{"mean":0.097667,"std":0.002887},{"mean":0.126,"std":0.006928},{"mean":0.147667,"std":0.013204},{"mean":0.173667,"std":0.022591},{"mean":0.209667,"std":0.030827},{"mean":0.245,"std":0.035791},{"mean":0.284333,"std":0.036937},{"mean":0.333333,"std":0.046047},{"mean":0.380667,"std":0.051013},{"mean":0.432333,"std":0.036226},{"mean":0.49,"std":0.039509},{"mean":0.573667,"std":0.034933},{"mean":0.665333,"std":0.038018},{"mean":0.808,"std":0.069735}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.433333,"std":0.002082},{"mean":0.437,"std":0.003464},{"mean":0.439,"std":0.004359},{"mean":0.443,"std":0.005196},{"mean":0.447667,"std":0.00611},{"mean":0.457333,"std":0.009074},{"mean":0.465333,"std":0.009452},{"mean":0.474667,"std":0.013868},{"mean":0.488333,"std":0.013868},{"mean":0.499333,"std":0.018556},{"mean":0.519,"std":0.020881},{"mean":0.542,"std":0.028478},{"mean":0.570667,"std":0.036005},{"mean":0.597,"std":0.029614},{"mean":0.623333,"std":0.038501},{"mean":0.669667,"std":0.039004},{"mean":0.724333,"std":0.040673},{"mean":0.808,"std":0.069735}],"llm_call_rate":[{"mean":0.959091,"std":0.006254},{"mean":0.916364,"std":0.002271},{"mean":0.872788,"std":0.001034},{"mean":0.825697,"std":0.00021},{"mean":0.78303,"std":0.002259},{"mean":0.738788,"std":0.005948},{"mean":0.691879,"std":0.005912},{"mean":0.644606,"std":0.002455},{"mean":0.598364,"std":0.000962},{"mean":0.553515,"std":0.004026},{"mean":0.499939,"std":0.006132},{"mean":0.454424,"std":0.009507},{"mean":0.402485,"std":0.013828},{"mean":0.352,"std":0.016443},{"mean":0.300909,"std":0.013886},{"mean":0.252121,"std":0.011494},{"mean":0.19897,"std":0.010094},{"mean":0.136424,"std":0.005468},{"mean":0.078121,"std":0.004964},{"mean":0.00103,"std":0.001481}]}},"entropy":{"risk_coverage":[{"mean":0.054545,"std":0.010909},{"mean":0.044848,"std":0.00105},{"mean":0.045657,"std":0.002523},{"mean":0.051212,"std":0.007624},{"mean":0.057939,"std":0.008492},{"mean":0.060606,"std":0.007348},{"mean":0.067186,"std":0.005792},{"mean":0.08,"std":0.006412},{"mean":0.089966,"std":0.007519},{"mean":0.098303,"std":0.0098},{"mean":0.105565,"std":0.009634},{"mean":0.113333,"std":0.008672},{"mean":0.12028,"std":0.009933},{"mean":0.130649,"std":0.011671},{"mean":0.139879,"std":0.011644},{"mean":0.150606,"std":0.011854},{"mean":0.165704,"std":0.010878},{"mean":0.18202,"std":0.010433},{"mean":0.200319,"std":0.009633},{"mean":0.210061,"std":0.016089}],"operating":{"tau":[{"mean":-1.586516,"std":0.039302},{"mean":-1.630735,"std":0.035912},{"mean":-1.672396,"std":0.02445},{"mean":-1.705022,"std":0.014879},{"mean":-1.732723,"std":0.004208},{"mean":-1.751307,"std":0.004454},{"mean":-1.76865,"std":0.00832},{"mean":-1.782229,"std":0.009638},{"mean":-1.795532,"std":0.010242},{"mean":-1.807798,"std":0.010867},{"mean":-1.818954,"std":0.010496},{"mean":-1.830084,"std":0.008883},{"mean":-1.840085,"std":0.007621},{"mean":-1.849884,"std":0.007433},{"mean":-1.859567,"std":0.006515},{"mean":-1.870271,"std":0.005639},{"mean":-1.883213,"std":0.004769},{"mean":-1.899159,"std":0.0031},{"mean":-1.919605,"std":0.003186},{"mean":-2.005049,"std":0.014368}],"coverage":[{"mean":0.044,"std":0.007091},{"mean":0.082545,"std":0.004118},{"mean":0.128606,"std":0.002888},{"mean":0.169576,"std":0.005157},{"mean":0.21503,"std":0.008805},{"mean":0.257091,"std":0.01069},{"mean":0.302909,"std":0.009216},{"mean":0.345333,"std":0.006711},{"mean":0.390909,"std":0.002622},{"mean":0.435333,"std":0.002508},{"mean":0.486182,"std":0.005911},{"mean":0.536606,"std":0.010598},{"mean":0.587515,"std":0.013314},{"mean":0.634303,"std":0.01416},{"mean":0.680848,"std":0.014455},{"mean":0.730242,"std":0.008406},{"mean":0.783636,"std":0.00634},{"mean":0.841273,"std":0.01024},{"mean":0.905091,"std":0.009944},{"mean":0.99703,"std":0.002027}],"selective_risk":[{"mean":0.054071,"std":0.014388},{"mean":0.046962,"std":0.001295},{"mean":0.045702,"std":0.001495},{"mean":0.047194,"std":0.003338},{"mean":0.051419,"std":0.007843},{"mean":0.058666,"std":0.007557},{"mean":0.061114,"std":0.004493},{"mean":0.066397,"std":0.004862},{"mean":0.078303,"std":0.005754},{"mean":0.088524,"std":0.007198},{"mean":0.096415,"std":0.010397},{"mean":0.102918,"std":0.010281},{"mean":0.111609,"std":0.01034},{"mean":0.117778,"std":0.008589},{"mean":0.12684,"std":0.009585},{"mean":0.1367,"std":0.011826},{"mean":0.146731,"std":0.013405},{"mean":0.162987,"std":0.014072},{"mean":0.183512,"std":0.013335},{"mean":0.210313,"std":0.016244}],"accuracy_8":[{"mean":0.82103,"std":0.000555},{"mean":0.821091,"std":0.000364},{"mean":0.823333,"std":0.001111},{"mean":0.826727,"std":0.000962},{"mean":0.827697,"std":0.002107},{"mean":0.828848,"std":0.001525},{"mean":0.830242,"std":0.00168},{"mean":0.83303,"std":0.002421},{"mean":0.834848,"std":0.004135},{"mean":0.836485,"std":0.004001},{"mean":0.837455,"std":0.002097},{"mean":0.840121,"std":0.00129},{"mean":0.841879,"std":0.003555},{"mean":0.843394,"std":0.003951},{"mean":0.842303,"std":0.006905},{"mean":0.838303,"std":0.009084},{"mean":0.834545,"std":0.011273},{"mean":0.824606,"std":0.013569},{"mean":0.812545,"std":0.013546},{"mean":0.789939,"std":0.016313}],"high_conf_oos_misroute_rate":[{"mean":0.012,"std":0.004583},{"mean":0.02,"std":0.003},{"mean":0.029,"std":0.002},{"mean":0.038,"std":0.003},{"mean":0.049667,"std":0.007638},{"mean":0.067,"std":0.007937},{"mean":0.08,"std":0.005292},{"mean":0.097667,"std":0.003786},{"mean":0.124667,"std":0.005033},{"mean":0.155,"std":0.010536},{"mean":0.178,"std":0.016643},{"mean":0.209333,"std":0.021127},{"mean":0.243333,"std":0.018717},{"mean":0.276333,"std":0.016166},{"mean":0.315667,"std":0.022053},{"mean":0.358333,"std":0.034429},{"mean":0.413,"std":0.049518},{"mean":0.501667,"std":0.060178},{"mean":0.624,"std":0.06691},{"mean":0.808667,"std":0.069745}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.433,"std":0.001},{"mean":0.435667,"std":0.001528},{"mean":0.438667,"std":0.003512},{"mean":0.442667,"std":0.003055},{"mean":0.445333,"std":0.002517},{"mean":0.450667,"std":0.001155},{"mean":0.457667,"std":0.003512},{"mean":0.466,"std":0.006245},{"mean":0.475333,"std":0.009074},{"mean":0.486333,"std":0.013051},{"mean":0.495333,"std":0.017243},{"mean":0.51,"std":0.016093},{"mean":0.529333,"std":0.02318},{"mean":0.552667,"std":0.031786},{"mean":0.580667,"std":0.041765},{"mean":0.632,"std":0.05671},{"mean":0.705333,"std":0.063217},{"mean":0.809667,"std":0.069924}],"llm_call_rate":[{"mean":0.956,"std":0.007091},{"mean":0.917455,"std":0.004118},{"mean":0.871394,"std":0.002888},{"mean":0.830424,"std":0.005157},{"mean":0.78497,"std":0.008805},{"mean":0.742909,"std":0.01069},{"mean":0.697091,"std":0.009216},{"mean":0.654667,"std":0.006711},{"mean":0.609091,"std":0.002622},{"mean":0.564667,"std":0.002508},{"mean":0.513818,"std":0.005911},{"mean":0.463394,"std":0.010598},{"mean":0.412485,"std":0.013314},{"mean":0.365697,"std":0.01416},{"mean":0.319152,"std":0.014455},{"mean":0.269758,"std":0.008406},{"mean":0.216364,"std":0.00634},{"mean":0.158727,"std":0.01024},{"mean":0.094909,"std":0.009944},{"mean":0.00297,"std":0.002027}]}},"margin":{"risk_coverage":[{"mean":0.008485,"std":0.009151},{"mean":0.007879,"std":0.002099},{"mean":0.008081,"std":0.002523},{"mean":0.009394,"std":0.001389},{"mean":0.012121,"std":0.003279},{"mean":0.014141,"std":0.003338},{"mean":0.015931,"std":0.004418},{"mean":0.018939,"std":0.004223},{"mean":0.023569,"std":0.005258},{"mean":0.027515,"std":0.005979},{"mean":0.032727,"std":0.008441},{"mean":0.039495,"std":0.010952},{"mean":0.051282,"std":0.012715},{"mean":0.06329,"std":0.01376},{"mean":0.081051,"std":0.015504},{"mean":0.101742,"std":0.017614},{"mean":0.124706,"std":0.018511},{"mean":0.152929,"std":0.017579},{"mean":0.18252,"std":0.017558},{"mean":0.210061,"std":0.016089}],"operating":{"tau":[{"mean":7.036423,"std":0.514988},{"mean":6.344652,"std":0.50502},{"mean":5.884502,"std":0.502941},{"mean":5.515292,"std":0.466376},{"mean":5.176863,"std":0.438396},{"mean":4.860761,"std":0.441128},{"mean":4.565621,"std":0.446047},{"mean":4.298292,"std":0.430974},{"mean":4.030166,"std":0.385482},{"mean":3.760214,"std":0.349505},{"mean":3.493989,"std":0.306626},{"mean":3.198926,"std":0.219995},{"mean":2.887488,"std":0.171763},{"mean":2.52706,"std":0.146961},{"mean":2.175263,"std":0.077869},{"mean":1.772895,"std":0.063482},{"mean":1.339252,"std":0.062583},{"mean":0.911301,"std":0.057374},{"mean":0.455299,"std":0.045977},{"mean":0.003628,"std":0.003245}],"coverage":[{"mean":0.042,"std":0.001091},{"mean":0.087758,"std":0.003109},{"mean":0.131576,"std":0.008728},{"mean":0.172242,"std":0.008623},{"mean":0.215152,"std":0.006912},{"mean":0.259939,"std":0.004832},{"mean":0.304,"std":0.004329},{"mean":0.345697,"std":0.002366},{"mean":0.388727,"std":0.005827},{"mean":0.432424,"std":0.00165},{"mean":0.473818,"std":0.003507},{"mean":0.518606,"std":0.00498},{"mean":0.565212,"std":0.006099},{"mean":0.614848,"std":0.002482},{"mean":0.661152,"std":0.010674},{"mean":0.718061,"std":0.005979},{"mean":0.778727,"std":0.004943},{"mean":0.839636,"std":0.004237},{"mean":0.910727,"std":0.005773},{"mean":0.999455,"std":0.000182}],"selective_risk":[{"mean":0.008735,"std":0.008659},{"mean":0.00901,"std":0.002534},{"mean":0.007458,"std":0.00201},{"mean":0.007826,"std":0.002538},{"mean":0.009894,"std":0.002295},{"mean":0.012595,"std":0.002787},{"mean":0.014381,"std":0.00279},{"mean":0.015442,"std":0.003877},{"mean":0.018587,"std":0.004119},{"mean":0.021865,"std":0.004547},{"mean":0.025938,"std":0.005738},{"mean":0.028392,"std":0.006338},{"mean":0.033923,"std":0.009033},{"mean":0.042499,"std":0.011791},{"mean":0.053182,"std":0.012178},{"mean":0.069159,"std":0.013033},{"mean":0.091191,"std":0.013967},{"mean":0.119829,"std":0.018573},{"mean":0.159309,"std":0.020354},{"mean":0.209749,"std":0.016272}],"accuracy_8":[{"mean":0.822667,"std":0.001155},{"mean":0.825576,"std":0.003714},{"mean":0.828182,"std":0.004409},{"mean":0.831091,"std":0.004766},{"mean":0.833758,"std":0.00502},{"mean":0.836848,"std":0.005214},{"mean":0.840303,"std":0.005558},{"mean":0.843212,"std":0.005398},{"mean":0.846121,"std":0.006278},{"mean":0.850303,"std":0.007543},{"mean":0.853273,"std":0.006938},{"mean":0.857515,"std":0.007017},{"mean":0.862,"std":0.006873},{"mean":0.865758,"std":0.008925},{"mean":0.867939,"std":0.007182},{"mean":0.867333,"std":0.007458},{"mean":0.86297,"std":0.008822},{"mean":0.851152,"std":0.01265},{"mean":0.827879,"std":0.016547},{"mean":0.790182,"std":0.016339}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.001155},{"mean":0.002667,"std":0.001155},{"mean":0.003333,"std":0.001528},{"mean":0.005,"std":0.003},{"mean":0.009,"std":0.003464},{"mean":0.014,"std":0.003606},{"mean":0.018,"std":0.004583},{"mean":0.021,"std":0.006245},{"mean":0.028333,"std":0.007371},{"mean":0.038,"std":0.009539},{"mean":0.049333,"std":0.012503},{"mean":0.060333,"std":0.015948},{"mean":0.078667,"std":0.022679},{"mean":0.108333,"std":0.03147},{"mean":0.144,"std":0.03593},{"mean":0.202,"std":0.046936},{"mean":0.279667,"std":0.047226},{"mean":0.397,"std":0.064374},{"mean":0.565333,"std":0.07444},{"mean":0.807,"std":0.071253}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.431,"std":0.001732},{"mean":0.431,"std":0.001732},{"mean":0.430667,"std":0.002309},{"mean":0.430667,"std":0.002309},{"mean":0.431,"std":0.001732},{"mean":0.431667,"std":0.002309},{"mean":0.432667,"std":0.002517},{"mean":0.434333,"std":0.003055},{"mean":0.435,"std":0.002646},{"mean":0.436333,"std":0.005686},{"mean":0.44,"std":0.008185},{"mean":0.444667,"std":0.01365},{"mean":0.453667,"std":0.018877},{"mean":0.465,"std":0.016703},{"mean":0.487667,"std":0.030501},{"mean":0.517333,"std":0.042771},{"mean":0.576667,"std":0.061809},{"mean":0.665,"std":0.065483},{"mean":0.808,"std":0.071435}],"llm_call_rate":[{"mean":0.958,"std":0.001091},{"mean":0.912242,"std":0.003109},{"mean":0.868424,"std":0.008728},{"mean":0.827758,"std":0.008623},{"mean":0.784848,"std":0.006912},{"mean":0.740061,"std":0.004832},{"mean":0.696,"std":0.004329},{"mean":0.654303,"std":0.002366},{"mean":0.611273,"std":0.005827},{"mean":0.567576,"std":0.00165},{"mean":0.526182,"std":0.003507},{"mean":0.481394,"std":0.00498},{"mean":0.434788,"std":0.006099},{"mean":0.385152,"std":0.002482},{"mean":0.338848,"std":0.010674},{"mean":0.281939,"std":0.005979},{"mean":0.221273,"std":0.004943},{"mean":0.160364,"std":0.004237},{"mean":0.089273,"std":0.005773},{"mean":0.000545,"std":0.000182}]}},"msp_t":{"risk_coverage":[{"mean":0.008485,"std":0.005555},{"mean":0.006061,"std":0.003785},{"mean":0.008889,"std":0.003896},{"mean":0.009091,"std":0.001818},{"mean":0.010667,"std":0.002554},{"mean":0.012525,"std":0.003943},{"mean":0.015065,"std":0.003244},{"mean":0.01803,"std":0.003412},{"mean":0.021145,"std":0.002923},{"mean":0.024606,"std":0.004308},{"mean":0.02876,"std":0.006879},{"mean":0.035253,"std":0.008594},{"mean":0.044103,"std":0.010336},{"mean":0.056797,"std":0.0123},{"mean":0.073051,"std":0.014269},{"mean":0.095,"std":0.015726},{"mean":0.117148,"std":0.017354},{"mean":0.145387,"std":0.015925},{"mean":0.177927,"std":0.016375},{"mean":0.210061,"std":0.016089}],"operating":{"tau":[{"mean":0.997603,"std":0.001099},{"mean":0.995413,"std":0.001796},{"mean":0.993239,"std":0.002548},{"mean":0.990353,"std":0.003551},{"mean":0.986327,"std":0.005043},{"mean":0.98149,"std":0.006719},{"mean":0.975918,"std":0.008636},{"mean":0.969593,"std":0.010627},{"mean":0.961416,"std":0.01205},{"mean":0.950944,"std":0.014027},{"mean":0.938612,"std":0.014754},{"mean":0.920534,"std":0.016043},{"mean":0.897202,"std":0.014739},{"mean":0.86611,"std":0.01532},{"mean":0.825028,"std":0.012583},{"mean":0.767214,"std":0.017086},{"mean":0.682347,"std":0.011581},{"mean":0.577375,"std":0.003281},{"mean":0.459385,"std":0.010535},{"mean":0.210519,"std":0.024386}],"coverage":[{"mean":0.041879,"std":0.000688},{"mean":0.088545,"std":0.004299},{"mean":0.129515,"std":0.005649},{"mean":0.17,"std":0.003691},{"mean":0.216182,"std":0.006548},{"mean":0.261091,"std":0.007468},{"mean":0.303576,"std":0.005567},{"mean":0.343576,"std":0.004226},{"mean":0.388061,"std":0.002011},{"mean":0.434121,"std":0.003442},{"mean":0.473152,"std":0.003461},{"mean":0.518667,"std":0.004399},{"mean":0.564485,"std":0.007276},{"mean":0.609212,"std":0.008044},{"mean":0.656364,"std":0.00512},{"mean":0.708424,"std":0.008705},{"mean":0.765515,"std":0.007821},{"mean":0.828061,"std":0.006986},{"mean":0.900061,"std":0.010885},{"mean":0.998424,"std":0.002107}],"selective_risk":[{"mean":0.005729,"std":0.006554},{"mean":0.006727,"std":0.004034},{"mean":0.008026,"std":0.002457},{"mean":0.008949,"std":0.002684},{"mean":0.008728,"std":0.001983},{"mean":0.01096,"std":0.002751},{"mean":0.013019,"std":0.00387},{"mean":0.015168,"std":0.003093},{"mean":0.017483,"std":0.003329},{"mean":0.020224,"std":0.003613},{"mean":0.022524,"std":0.004222},{"mean":0.02546,"std":0.005341},{"mean":0.030492,"std":0.008007},{"mean":0.036566,"std":0.007438},{"mean":0.045708,"std":0.011289},{"mean":0.059118,"std":0.011347},{"mean":0.078597,"std":0.011406},{"mean":0.107056,"std":0.015124},{"mean":0.144973,"std":0.020818},{"mean":0.209294,"std":0.016293}],"accuracy_8":[{"mean":0.823091,"std":0.001898},{"mean":0.825636,"std":0.004276},{"mean":0.828,"std":0.004589},{"mean":0.830606,"std":0.004742},{"mean":0.834,"std":0.005421},{"mean":0.837879,"std":0.006196},{"mean":0.841394,"std":0.006748},{"mean":0.843758,"std":0.006531},{"mean":0.847636,"std":0.005961},{"mean":0.852424,"std":0.00691},{"mean":0.855879,"std":0.006809},{"mean":0.859576,"std":0.006948},{"mean":0.864364,"std":0.006142},{"mean":0.867879,"std":0.007237},{"mean":0.871394,"std":0.007838},{"mean":0.872303,"std":0.008095},{"mean":0.869152,"std":0.007395},{"mean":0.859394,"std":0.01014},{"mean":0.838424,"std":0.01465},{"mean":0.790424,"std":0.016191}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.001},{"mean":0.002,"std":0.001},{"mean":0.003667,"std":0.002082},{"mean":0.006,"std":0.003464},{"mean":0.007333,"std":0.003055},{"mean":0.011667,"std":0.004163},{"mean":0.015667,"std":0.004933},{"mean":0.02,"std":0.005568},{"mean":0.026,"std":0.007},{"mean":0.034333,"std":0.008327},{"mean":0.041333,"std":0.009609},{"mean":0.051667,"std":0.01365},{"mean":0.066,"std":0.020421},{"mean":0.086333,"std":0.021221},{"mean":0.115667,"std":0.033292},{"mean":0.159333,"std":0.031943},{"mean":0.220333,"std":0.036747},{"mean":0.323333,"std":0.049692},{"mean":0.477,"std":0.081505},{"mean":0.804,"std":0.071267}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.431,"std":0.001732},{"mean":0.431,"std":0.001732},{"mean":0.430667,"std":0.002309},{"mean":0.430333,"std":0.002887},{"mean":0.430333,"std":0.002887},{"mean":0.431,"std":0.002646},{"mean":0.432,"std":0.002646},{"mean":0.433,"std":0.003},{"mean":0.433667,"std":0.003512},{"mean":0.434,"std":0.004},{"mean":0.435667,"std":0.005859},{"mean":0.438,"std":0.007},{"mean":0.442333,"std":0.01097},{"mean":0.45,"std":0.01562},{"mean":0.465,"std":0.016823},{"mean":0.486667,"std":0.021595},{"mean":0.528333,"std":0.034239},{"mean":0.600667,"std":0.058192},{"mean":0.806333,"std":0.070437}],"llm_call_rate":[{"mean":0.958121,"std":0.000688},{"mean":0.911455,"std":0.004299},{"mean":0.870485,"std":0.005649},{"mean":0.83,"std":0.003691},{"mean":0.783818,"std":0.006548},{"mean":0.738909,"std":0.007468},{"mean":0.696424,"std":0.005567},{"mean":0.656424,"std":0.004226},{"mean":0.611939,"std":0.002011},{"mean":0.565879,"std":0.003442},{"mean":0.526848,"std":0.003461},{"mean":0.481333,"std":0.004399},{"mean":0.435515,"std":0.007276},{"mean":0.390788,"std":0.008044},{"mean":0.343636,"std":0.00512},{"mean":0.291576,"std":0.008705},{"mean":0.234485,"std":0.007821},{"mean":0.171939,"std":0.006986},{"mean":0.099939,"std":0.010885},{"mean":0.001576,"std":0.002107}]}}}}},"bert/k50":{"model":"bert","k":50,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["bert-base-uncased-k50-seed42","bert-base-uncased-k50-seed43","bert-base-uncased-k50-seed44"],"logits_sha256":["42bd57f63c3f37264962334df25861a73c5279f7559302c9a467d5fadd246b7d","21bb8489d1c181fa6982ff89fb8a587505a5fb00ad53031246c0bdecb16bd949","e335e894b2eb54e9f4074b861f202c478078700f308503c82ecc34ec7fc1c6fa"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.003636,"std":0.0},{"mean":0.004242,"std":0.002099},{"mean":0.005253,"std":0.0014},{"mean":0.004848,"std":0.00105},{"mean":0.004848,"std":0.00084},{"mean":0.005859,"std":0.001852},{"mean":0.006061,"std":0.002099},{"mean":0.007121,"std":0.001389},{"mean":0.007273,"std":0.001852},{"mean":0.007636,"std":0.001311},{"mean":0.007824,"std":0.000763},{"mean":0.008384,"std":0.001064},{"mean":0.009231,"std":0.001009},{"mean":0.010909,"std":0.001558},{"mean":0.015111,"std":0.001959},{"mean":0.021667,"std":0.001479},{"mean":0.033298,"std":0.00081},{"mean":0.050572,"std":0.001184},{"mean":0.075215,"std":0.001013},{"mean":0.109212,"std":0.004034}],"operating":{"tau":[{"mean":0.935527,"std":0.002498},{"mean":0.929803,"std":0.001865},{"mean":0.924598,"std":0.001586},{"mean":0.920953,"std":0.001322},{"mean":0.917695,"std":0.00122},{"mean":0.913456,"std":0.000917},{"mean":0.910053,"std":0.000815},{"mean":0.906465,"std":0.001377},{"mean":0.902893,"std":0.001797},{"mean":0.898326,"std":0.001939},{"mean":0.893451,"std":0.002713},{"mean":0.887209,"std":0.002942},{"mean":0.8803,"std":0.00328},{"mean":0.872057,"std":0.003925},{"mean":0.861568,"std":0.005074},{"mean":0.845588,"std":0.007645},{"mean":0.808731,"std":0.018184},{"mean":0.721042,"std":0.014423},{"mean":0.491747,"std":0.012003},{"mean":0.06626,"std":0.007311}],"coverage":[{"mean":0.055152,"std":0.005876},{"mean":0.100242,"std":0.005299},{"mean":0.143697,"std":0.004714},{"mean":0.183939,"std":0.004644},{"mean":0.225939,"std":0.009144},{"mean":0.271515,"std":0.008788},{"mean":0.31297,"std":0.008952},{"mean":0.357455,"std":0.004229},{"mean":0.396848,"std":0.005339},{"mean":0.443758,"std":0.003909},{"mean":0.488424,"std":0.004203},{"mean":0.532424,"std":0.001315},{"mean":0.575091,"std":0.00426},{"mean":0.619333,"std":0.004147},{"mean":0.659152,"std":0.004155},{"mean":0.699879,"std":0.003203},{"mean":0.751091,"std":0.003186},{"mean":0.804606,"std":0.004365},{"mean":0.878545,"std":0.006464},{"mean":0.998485,"std":0.000915}],"selective_risk":[{"mean":0.003321,"std":0.000337},{"mean":0.00431,"std":0.002251},{"mean":0.005086,"std":0.001382},{"mean":0.004961,"std":0.001113},{"mean":0.004866,"std":0.001737},{"mean":0.005148,"std":0.001481},{"mean":0.005815,"std":0.001794},{"mean":0.006094,"std":0.002001},{"mean":0.007013,"std":0.001528},{"mean":0.007372,"std":0.001862},{"mean":0.007569,"std":0.001135},{"mean":0.007741,"std":0.000799},{"mean":0.008119,"std":0.00085},{"mean":0.00861,"std":0.001205},{"mean":0.009655,"std":0.001105},{"mean":0.01108,"std":0.001782},{"mean":0.015325,"std":0.002284},{"mean":0.022669,"std":0.001126},{"mean":0.042553,"std":0.0025},{"mean":0.108044,"std":0.003231}],"accuracy_8":[{"mean":0.824788,"std":0.001747},{"mean":0.827939,"std":0.002469},{"mean":0.830788,"std":0.002783},{"mean":0.833576,"std":0.003901},{"mean":0.83903,"std":0.002011},{"mean":0.845091,"std":0.000833},{"mean":0.85103,"std":0.000458},{"mean":0.857758,"std":0.001277},{"mean":0.861394,"std":0.001169},{"mean":0.867697,"std":0.001821},{"mean":0.873515,"std":0.002421},{"mean":0.878788,"std":0.003611},{"mean":0.884182,"std":0.005636},{"mean":0.890364,"std":0.004409},{"mean":0.893636,"std":0.003623},{"mean":0.898424,"std":0.002421},{"mean":0.905152,"std":0.001277},{"mean":0.909697,"std":0.001001},{"mean":0.911879,"std":0.000458},{"mean":0.891515,"std":0.003789}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.001333,"std":0.001155},{"mean":0.003,"std":0.001},{"mean":0.004,"std":0.001},{"mean":0.005,"std":0.002},{"mean":0.006667,"std":0.002082},{"mean":0.009,"std":0.003},{"mean":0.01,"std":0.003606},{"mean":0.013,"std":0.003606},{"mean":0.015,"std":0.004583},{"mean":0.017333,"std":0.003055},{"mean":0.019667,"std":0.002309},{"mean":0.022333,"std":0.002517},{"mean":0.025,"std":0.003464},{"mean":0.029333,"std":0.002517},{"mean":0.036,"std":0.005292},{"mean":0.055,"std":0.009539},{"mean":0.088,"std":0.004359},{"mean":0.176333,"std":0.01097},{"mean":0.53,"std":0.017349}],"oos_misroute_rate":[{"mean":0.427333,"std":0.002517},{"mean":0.426333,"std":0.003512},{"mean":0.425,"std":0.004583},{"mean":0.424,"std":0.006245},{"mean":0.423333,"std":0.006807},{"mean":0.423333,"std":0.006807},{"mean":0.424,"std":0.006083},{"mean":0.423333,"std":0.006658},{"mean":0.424667,"std":0.006429},{"mean":0.423667,"std":0.006028},{"mean":0.424,"std":0.004583},{"mean":0.422667,"std":0.005508},{"mean":0.422333,"std":0.005774},{"mean":0.420667,"std":0.006429},{"mean":0.420667,"std":0.007371},{"mean":0.419333,"std":0.005774},{"mean":0.417667,"std":0.005508},{"mean":0.416667,"std":0.008505},{"mean":0.422,"std":0.003},{"mean":0.533333,"std":0.020599}],"llm_call_rate":[{"mean":0.944848,"std":0.005876},{"mean":0.899758,"std":0.005299},{"mean":0.856303,"std":0.004714},{"mean":0.816061,"std":0.004644},{"mean":0.774061,"std":0.009144},{"mean":0.728485,"std":0.008788},{"mean":0.68703,"std":0.008952},{"mean":0.642545,"std":0.004229},{"mean":0.603152,"std":0.005339},{"mean":0.556242,"std":0.003909},{"mean":0.511576,"std":0.004203},{"mean":0.467576,"std":0.001315},{"mean":0.424909,"std":0.00426},{"mean":0.380667,"std":0.004147},{"mean":0.340848,"std":0.004155},{"mean":0.300121,"std":0.003203},{"mean":0.248909,"std":0.003186},{"mean":0.195394,"std":0.004365},{"mean":0.121455,"std":0.006464},{"mean":0.001515,"std":0.000915}]}},"entropy":{"risk_coverage":[{"mean":0.003636,"std":0.0},{"mean":0.00303,"std":0.002099},{"mean":0.00404,"std":0.0014},{"mean":0.004848,"std":0.00105},{"mean":0.005091,"std":0.00126},{"mean":0.005051,"std":0.001525},{"mean":0.006061,"std":0.001824},{"mean":0.006667,"std":0.001144},{"mean":0.007138,"std":0.001234},{"mean":0.007152,"std":0.001277},{"mean":0.007493,"std":0.001063},{"mean":0.007677,"std":0.000926},{"mean":0.008485,"std":0.001261},{"mean":0.010736,"std":0.001949},{"mean":0.014141,"std":0.00098},{"mean":0.019394,"std":0.00086},{"mean":0.031373,"std":0.001098},{"mean":0.049899,"std":0.000202},{"mean":0.076427,"std":0.000797},{"mean":0.109212,"std":0.004034}],"operating":{"tau":[{"mean":-0.528229,"std":0.01573},{"mean":-0.565639,"std":0.011616},{"mean":-0.599077,"std":0.013421},{"mean":-0.623411,"std":0.010088},{"mean":-0.644536,"std":0.009115},{"mean":-0.669129,"std":0.007739},{"mean":-0.689797,"std":0.008703},{"mean":-0.709283,"std":0.010798},{"mean":-0.730433,"std":0.01369},{"mean":-0.756261,"std":0.012331},{"mean":-0.784555,"std":0.015577},{"mean":-0.816254,"std":0.013981},{"mean":-0.853119,"std":0.017732},{"mean":-0.89601,"std":0.023233},{"mean":-0.949686,"std":0.032022},{"mean":-1.032666,"std":0.038763},{"mean":-1.195759,"std":0.080011},{"mean":-1.509454,"std":0.035504},{"mean":-2.298144,"std":0.012718},{"mean":-4.471986,"std":0.007829}],"coverage":[{"mean":0.055273,"std":0.004639},{"mean":0.099273,"std":0.006472},{"mean":0.144667,"std":0.005842},{"mean":0.184182,"std":0.002691},{"mean":0.226485,"std":0.005976},{"mean":0.270788,"std":0.009621},{"mean":0.312364,"std":0.005571},{"mean":0.356242,"std":0.003745},{"mean":0.400424,"std":0.005675},{"mean":0.445939,"std":0.00502},{"mean":0.492182,"std":0.006507},{"mean":0.532727,"std":0.005512},{"mean":0.576061,"std":0.006427},{"mean":0.619091,"std":0.005548},{"mean":0.656303,"std":0.006273},{"mean":0.7,"std":0.003154},{"mean":0.747818,"std":0.003549},{"mean":0.798485,"std":0.002003},{"mean":0.869091,"std":0.003988},{"mean":0.999152,"std":0.000378}],"selective_risk":[{"mean":0.003304,"std":0.000266},{"mean":0.003099,"std":0.002239},{"mean":0.003806,"std":0.001392},{"mean":0.004945,"std":0.00105},{"mean":0.005358,"std":0.000969},{"mean":0.00494,"std":0.001147},{"mean":0.00563,"std":0.002074},{"mean":0.005949,"std":0.001773},{"mean":0.006652,"std":0.001081},{"mean":0.007065,"std":0.001424},{"mean":0.007268,"std":0.001327},{"mean":0.00763,"std":0.001058},{"mean":0.007792,"std":0.000873},{"mean":0.007928,"std":0.000745},{"mean":0.008949,"std":0.001814},{"mean":0.01073,"std":0.002165},{"mean":0.013933,"std":0.002033},{"mean":0.018976,"std":0.00051},{"mean":0.037026,"std":0.00114},{"mean":0.108638,"std":0.00375}],"accuracy_8":[{"mean":0.824606,"std":0.001728},{"mean":0.827697,"std":0.002302},{"mean":0.830788,"std":0.002414},{"mean":0.834,"std":0.002926},{"mean":0.839879,"std":0.000915},{"mean":0.846364,"std":0.000945},{"mean":0.851576,"std":0.001183},{"mean":0.857152,"std":0.00084},{"mean":0.861818,"std":0.001373},{"mean":0.868485,"std":0.002508},{"mean":0.873879,"std":0.004265},{"mean":0.879515,"std":0.006129},{"mean":0.885091,"std":0.006431},{"mean":0.889818,"std":0.004983},{"mean":0.893091,"std":0.004094},{"mean":0.898424,"std":0.002783},{"mean":0.904121,"std":0.001365},{"mean":0.909091,"std":0.001012},{"mean":0.911212,"std":0.001377},{"mean":0.891152,"std":0.004087}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.001155},{"mean":0.002,"std":0.001},{"mean":0.004,"std":0.001},{"mean":0.005667,"std":0.001155},{"mean":0.006333,"std":0.001528},{"mean":0.008333,"std":0.003512},{"mean":0.01,"std":0.004},{"mean":0.012333,"std":0.002517},{"mean":0.014333,"std":0.003512},{"mean":0.016667,"std":0.003512},{"mean":0.019333,"std":0.002887},{"mean":0.021333,"std":0.003055},{"mean":0.023333,"std":0.003215},{"mean":0.026667,"std":0.004726},{"mean":0.034667,"std":0.006028},{"mean":0.049,"std":0.00755},{"mean":0.071667,"std":0.003215},{"mean":0.153333,"std":0.007024},{"mean":0.533333,"std":0.020232}],"oos_misroute_rate":[{"mean":0.427333,"std":0.002517},{"mean":0.426333,"std":0.003512},{"mean":0.424667,"std":0.005033},{"mean":0.424333,"std":0.00611},{"mean":0.423333,"std":0.007095},{"mean":0.422667,"std":0.006658},{"mean":0.423333,"std":0.005774},{"mean":0.423667,"std":0.006351},{"mean":0.423,"std":0.006928},{"mean":0.423,"std":0.005568},{"mean":0.424333,"std":0.005132},{"mean":0.423667,"std":0.004726},{"mean":0.421667,"std":0.004619},{"mean":0.420333,"std":0.005132},{"mean":0.419667,"std":0.004726},{"mean":0.42,"std":0.005568},{"mean":0.419333,"std":0.006658},{"mean":0.417333,"std":0.004509},{"mean":0.424667,"std":0.001528},{"mean":0.535,"std":0.021932}],"llm_call_rate":[{"mean":0.944727,"std":0.004639},{"mean":0.900727,"std":0.006472},{"mean":0.855333,"std":0.005842},{"mean":0.815818,"std":0.002691},{"mean":0.773515,"std":0.005976},{"mean":0.729212,"std":0.009621},{"mean":0.687636,"std":0.005571},{"mean":0.643758,"std":0.003745},{"mean":0.599576,"std":0.005675},{"mean":0.554061,"std":0.00502},{"mean":0.507818,"std":0.006507},{"mean":0.467273,"std":0.005512},{"mean":0.423939,"std":0.006427},{"mean":0.380909,"std":0.005548},{"mean":0.343697,"std":0.006273},{"mean":0.3,"std":0.003154},{"mean":0.252182,"std":0.003549},{"mean":0.201515,"std":0.002003},{"mean":0.130909,"std":0.003988},{"mean":0.000848,"std":0.000378}]}},"margin":{"risk_coverage":[{"mean":0.009697,"std":0.009151},{"mean":0.009697,"std":0.005555},{"mean":0.010101,"std":0.004589},{"mean":0.009394,"std":0.004296},{"mean":0.00897,"std":0.004006},{"mean":0.009495,"std":0.002295},{"mean":0.010043,"std":0.0012},{"mean":0.011061,"std":0.001144},{"mean":0.010909,"std":0.000404},{"mean":0.012364,"std":0.00063},{"mean":0.012452,"std":0.000688},{"mean":0.013636,"std":0.000303},{"mean":0.015385,"std":0.0},{"mean":0.017229,"std":0.001081},{"mean":0.021899,"std":0.001009},{"mean":0.029621,"std":0.003135},{"mean":0.041355,"std":0.002588},{"mean":0.057239,"std":0.001993},{"mean":0.082935,"std":0.004061},{"mean":0.109212,"std":0.004034}],"operating":{"tau":[{"mean":7.500489,"std":0.058324},{"mean":7.171784,"std":0.08796},{"mean":6.944141,"std":0.098834},{"mean":6.749937,"std":0.074757},{"mean":6.578967,"std":0.030694},{"mean":6.42439,"std":0.048893},{"mean":6.278472,"std":0.075462},{"mean":6.14621,"std":0.092565},{"mean":5.993694,"std":0.081656},{"mean":5.819706,"std":0.092374},{"mean":5.651368,"std":0.094369},{"mean":5.473079,"std":0.093475},{"mean":5.272854,"std":0.115033},{"mean":5.036627,"std":0.123958},{"mean":4.756738,"std":0.116574},{"mean":4.412374,"std":0.13506},{"mean":3.87499,"std":0.164409},{"mean":2.971917,"std":0.209965},{"mean":1.525956,"std":0.048921},{"mean":0.01461,"std":0.011639}],"coverage":[{"mean":0.046364,"std":0.002691},{"mean":0.094303,"std":0.002573},{"mean":0.139394,"std":0.003728},{"mean":0.180788,"std":0.007341},{"mean":0.225333,"std":0.003011},{"mean":0.272061,"std":0.003093},{"mean":0.317758,"std":0.004187},{"mean":0.356485,"std":0.006048},{"mean":0.397212,"std":0.001546},{"mean":0.443576,"std":0.005459},{"mean":0.487576,"std":0.006099},{"mean":0.535333,"std":0.005684},{"mean":0.581152,"std":0.00731},{"mean":0.626485,"std":0.009417},{"mean":0.669879,"std":0.003918},{"mean":0.718,"std":0.004779},{"mean":0.767273,"std":0.005677},{"mean":0.822364,"std":0.007184},{"mean":0.899455,"std":0.00513},{"mean":0.998788,"std":0.001315}],"selective_risk":[{"mean":0.009162,"std":0.009565},{"mean":0.010389,"std":0.006088},{"mean":0.010458,"std":0.004713},{"mean":0.009799,"std":0.004269},{"mean":0.009165,"std":0.004095},{"mean":0.009826,"std":0.003279},{"mean":0.009918,"std":0.002392},{"mean":0.009858,"std":0.001129},{"mean":0.01083,"std":0.001112},{"mean":0.010656,"std":0.000359},{"mean":0.011682,"std":0.000461},{"mean":0.012235,"std":0.001337},{"mean":0.013141,"std":0.000633},{"mean":0.014223,"std":0.000216},{"mean":0.016101,"std":0.000737},{"mean":0.018482,"std":0.001059},{"mean":0.024166,"std":0.001782},{"mean":0.034097,"std":0.004685},{"mean":0.056665,"std":0.001394},{"mean":0.108313,"std":0.003832}],"accuracy_8":[{"mean":0.824364,"std":0.00142},{"mean":0.828,"std":0.003026},{"mean":0.832,"std":0.003831},{"mean":0.835273,"std":0.003232},{"mean":0.840061,"std":0.003732},{"mean":0.844,"std":0.003288},{"mean":0.849091,"std":0.00365},{"mean":0.855273,"std":0.004413},{"mean":0.860485,"std":0.004034},{"mean":0.867576,"std":0.002455},{"mean":0.871818,"std":0.002234},{"mean":0.878,"std":0.003004},{"mean":0.884485,"std":0.003588},{"mean":0.889818,"std":0.00411},{"mean":0.895333,"std":0.003833},{"mean":0.901818,"std":0.001967},{"mean":0.906727,"std":0.001012},{"mean":0.909758,"std":0.001857},{"mean":0.90897,"std":0.000555},{"mean":0.891212,"std":0.003888}],"high_conf_oos_misroute_rate":[{"mean":0.002333,"std":0.002517},{"mean":0.004333,"std":0.003055},{"mean":0.007,"std":0.003606},{"mean":0.008,"std":0.004359},{"mean":0.009333,"std":0.005033},{"mean":0.012667,"std":0.004726},{"mean":0.014667,"std":0.003786},{"mean":0.016667,"std":0.002082},{"mean":0.020333,"std":0.001155},{"mean":0.022333,"std":0.000577},{"mean":0.026333,"std":0.002309},{"mean":0.030667,"std":0.003786},{"mean":0.036,"std":0.002646},{"mean":0.043,"std":0.001},{"mean":0.053333,"std":0.002082},{"mean":0.065333,"std":0.003512},{"mean":0.090667,"std":0.005686},{"mean":0.134,"std":0.020664},{"mean":0.242667,"std":0.006658},{"mean":0.531667,"std":0.020207}],"oos_misroute_rate":[{"mean":0.431333,"std":0.001155},{"mean":0.431,"std":0.001732},{"mean":0.43,"std":0.003606},{"mean":0.428,"std":0.004583},{"mean":0.426333,"std":0.004933},{"mean":0.427,"std":0.005196},{"mean":0.426333,"std":0.005774},{"mean":0.426,"std":0.007937},{"mean":0.425,"std":0.006245},{"mean":0.423,"std":0.006245},{"mean":0.423,"std":0.006083},{"mean":0.422333,"std":0.006658},{"mean":0.420333,"std":0.008083},{"mean":0.42,"std":0.006245},{"mean":0.416,"std":0.006245},{"mean":0.413667,"std":0.006658},{"mean":0.412667,"std":0.007506},{"mean":0.413333,"std":0.001155},{"mean":0.434,"std":0.005292},{"mean":0.534667,"std":0.020984}],"llm_call_rate":[{"mean":0.953636,"std":0.002691},{"mean":0.905697,"std":0.002573},{"mean":0.860606,"std":0.003728},{"mean":0.819212,"std":0.007341},{"mean":0.774667,"std":0.003011},{"mean":0.727939,"std":0.003093},{"mean":0.682242,"std":0.004187},{"mean":0.643515,"std":0.006048},{"mean":0.602788,"std":0.001546},{"mean":0.556424,"std":0.005459},{"mean":0.512424,"std":0.006099},{"mean":0.464667,"std":0.005684},{"mean":0.418848,"std":0.00731},{"mean":0.373515,"std":0.009417},{"mean":0.330121,"std":0.003918},{"mean":0.282,"std":0.004779},{"mean":0.232727,"std":0.005677},{"mean":0.177636,"std":0.007184},{"mean":0.100545,"std":0.00513},{"mean":0.001212,"std":0.001315}]}},"msp_t":{"risk_coverage":[{"mean":0.004848,"std":0.002099},{"mean":0.004242,"std":0.002099},{"mean":0.005253,"std":0.002523},{"mean":0.005758,"std":0.001892},{"mean":0.005333,"std":0.00183},{"mean":0.005657,"std":0.001525},{"mean":0.0071,"std":0.002615},{"mean":0.007424,"std":0.001892},{"mean":0.007946,"std":0.001633},{"mean":0.008121,"std":0.00147},{"mean":0.009036,"std":0.000832},{"mean":0.009293,"std":0.000974},{"mean":0.01035,"std":0.001282},{"mean":0.012468,"std":0.00119},{"mean":0.016162,"std":0.000918},{"mean":0.023864,"std":0.00149},{"mean":0.035936,"std":0.00037},{"mean":0.052862,"std":0.001113},{"mean":0.075407,"std":0.001657},{"mean":0.109212,"std":0.004034}],"operating":{"tau":[{"mean":0.993445,"std":9.7e-05},{"mean":0.992407,"std":0.000272},{"mean":0.991516,"std":0.0004},{"mean":0.990836,"std":0.00048},{"mean":0.990135,"std":0.000621},{"mean":0.98925,"std":0.000798},{"mean":0.988486,"std":0.000762},{"mean":0.98767,"std":0.000896},{"mean":0.986708,"std":0.000947},{"mean":0.985615,"std":0.001084},{"mean":0.984226,"std":0.001396},{"mean":0.982666,"std":0.001673},{"mean":0.980451,"std":0.002119},{"mean":0.977924,"std":0.002293},{"mean":0.974465,"std":0.002706},{"mean":0.96807,"std":0.003966},{"mean":0.952363,"std":0.007553},{"mean":0.90241,"std":0.008457},{"mean":0.688348,"std":0.021149},{"mean":0.107157,"std":0.010495}],"coverage":[{"mean":0.054485,"std":0.007532},{"mean":0.099152,"std":0.004433},{"mean":0.143515,"std":0.007631},{"mean":0.186424,"std":0.007909},{"mean":0.226061,"std":0.010197},{"mean":0.272424,"std":0.013263},{"mean":0.314364,"std":0.01095},{"mean":0.352606,"std":0.007205},{"mean":0.394485,"std":0.005043},{"mean":0.438424,"std":0.00147},{"mean":0.488182,"std":0.00531},{"mean":0.529091,"std":0.002271},{"mean":0.576606,"std":0.002783},{"mean":0.617697,"std":0.00165},{"mean":0.659697,"std":0.003177},{"mean":0.704848,"std":0.004853},{"mean":0.755091,"std":0.002976},{"mean":0.808424,"std":0.003714},{"mean":0.885273,"std":0.007875},{"mean":0.998242,"std":0.000639}],"selective_risk":[{"mean":0.004559,"std":0.002232},{"mean":0.004345,"std":0.002252},{"mean":0.00513,"std":0.00235},{"mean":0.005595,"std":0.002429},{"mean":0.005425,"std":0.002216},{"mean":0.006007,"std":0.001341},{"mean":0.00653,"std":0.002264},{"mean":0.007201,"std":0.002521},{"mean":0.007359,"std":0.002024},{"mean":0.0076,"std":0.00185},{"mean":0.007815,"std":0.001447},{"mean":0.008933,"std":0.000559},{"mean":0.009043,"std":0.001146},{"mean":0.010204,"std":0.001108},{"mean":0.010656,"std":0.00109},{"mean":0.012636,"std":0.001493},{"mean":0.016772,"std":0.001259},{"mean":0.025562,"std":0.000974},{"mean":0.047156,"std":0.002853},{"mean":0.107888,"std":0.003509}],"accuracy_8":[{"mean":0.824909,"std":0.001585},{"mean":0.827758,"std":0.002945},{"mean":0.830545,"std":0.002857},{"mean":0.833818,"std":0.003004},{"mean":0.837879,"std":0.002475},{"mean":0.843818,"std":0.002278},{"mean":0.849455,"std":0.001666},{"mean":0.855091,"std":0.000962},{"mean":0.862667,"std":0.000915},{"mean":0.867394,"std":0.000278},{"mean":0.87297,"std":0.000757},{"mean":0.877818,"std":0.001845},{"mean":0.884061,"std":0.003732},{"mean":0.888848,"std":0.003379},{"mean":0.894121,"std":0.002753},{"mean":0.899515,"std":0.003193},{"mean":0.905939,"std":0.001412},{"mean":0.909697,"std":0.00021},{"mean":0.911394,"std":0.000105},{"mean":0.891576,"std":0.004067}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.001333,"std":0.001155},{"mean":0.003,"std":0.001732},{"mean":0.004667,"std":0.002309},{"mean":0.005667,"std":0.002517},{"mean":0.008,"std":0.002},{"mean":0.01,"std":0.004},{"mean":0.012333,"std":0.004509},{"mean":0.013667,"std":0.004726},{"mean":0.015667,"std":0.004041},{"mean":0.018,"std":0.004},{"mean":0.022333,"std":0.001528},{"mean":0.024,"std":0.002646},{"mean":0.029333,"std":0.003215},{"mean":0.033333,"std":0.003215},{"mean":0.042,"std":0.004583},{"mean":0.060667,"std":0.005508},{"mean":0.100667,"std":0.004726},{"mean":0.197667,"std":0.012741},{"mean":0.529,"std":0.019079}],"oos_misroute_rate":[{"mean":0.428,"std":0.004},{"mean":0.426667,"std":0.003512},{"mean":0.425667,"std":0.005033},{"mean":0.424333,"std":0.005859},{"mean":0.424333,"std":0.005774},{"mean":0.424333,"std":0.005859},{"mean":0.425,"std":0.005196},{"mean":0.425667,"std":0.004726},{"mean":0.425333,"std":0.006807},{"mean":0.425333,"std":0.007572},{"mean":0.423,"std":0.007},{"mean":0.424,"std":0.00781},{"mean":0.422,"std":0.006928},{"mean":0.422,"std":0.00781},{"mean":0.420333,"std":0.006658},{"mean":0.42,"std":0.007},{"mean":0.415667,"std":0.007095},{"mean":0.416333,"std":0.008145},{"mean":0.422667,"std":0.003512},{"mean":0.532667,"std":0.021733}],"llm_call_rate":[{"mean":0.945515,"std":0.007532},{"mean":0.900848,"std":0.004433},{"mean":0.856485,"std":0.007631},{"mean":0.813576,"std":0.007909},{"mean":0.773939,"std":0.010197},{"mean":0.727576,"std":0.013263},{"mean":0.685636,"std":0.01095},{"mean":0.647394,"std":0.007205},{"mean":0.605515,"std":0.005043},{"mean":0.561576,"std":0.00147},{"mean":0.511818,"std":0.00531},{"mean":0.470909,"std":0.002271},{"mean":0.423394,"std":0.002783},{"mean":0.382303,"std":0.00165},{"mean":0.340303,"std":0.003177},{"mean":0.295152,"std":0.004853},{"mean":0.244909,"std":0.002976},{"mean":0.191576,"std":0.003714},{"mean":0.114727,"std":0.007875},{"mean":0.001758,"std":0.000639}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.004848,"std":0.004199},{"mean":0.006061,"std":0.002099},{"mean":0.006061,"std":0.003207},{"mean":0.005455,"std":0.003278},{"mean":0.004848,"std":0.002338},{"mean":0.005051,"std":0.001262},{"mean":0.005195,"std":0.0009},{"mean":0.005606,"std":0.000694},{"mean":0.005657,"std":0.000404},{"mean":0.006667,"std":0.000915},{"mean":0.007052,"std":0.001252},{"mean":0.007677,"std":0.000926},{"mean":0.008485,"std":0.000855},{"mean":0.00987,"std":0.00119},{"mean":0.012687,"std":0.0007},{"mean":0.019545,"std":0.000787},{"mean":0.032799,"std":0.001358},{"mean":0.053064,"std":0.00153},{"mean":0.083126,"std":0.001054},{"mean":0.113152,"std":0.003364}],"operating":{"tau":[{"mean":0.957725,"std":0.001352},{"mean":0.954486,"std":0.000616},{"mean":0.951662,"std":0.000666},{"mean":0.949355,"std":0.001244},{"mean":0.947379,"std":0.001557},{"mean":0.945382,"std":0.001665},{"mean":0.943191,"std":0.001792},{"mean":0.940756,"std":0.001646},{"mean":0.938431,"std":0.001559},{"mean":0.936407,"std":0.001433},{"mean":0.933999,"std":0.001633},{"mean":0.931155,"std":0.001551},{"mean":0.927491,"std":0.001154},{"mean":0.922936,"std":0.001581},{"mean":0.917524,"std":0.002324},{"mean":0.908966,"std":0.00372},{"mean":0.894899,"std":0.005896},{"mean":0.863329,"std":0.005131},{"mean":0.759403,"std":0.020471},{"mean":0.196451,"std":0.026103}],"coverage":[{"mean":0.043515,"std":0.000757},{"mean":0.087152,"std":0.001766},{"mean":0.129212,"std":0.00626},{"mean":0.175455,"std":0.004567},{"mean":0.217697,"std":0.003254},{"mean":0.257636,"std":0.005512},{"mean":0.304606,"std":0.003461},{"mean":0.353758,"std":0.004211},{"mean":0.393818,"std":0.003848},{"mean":0.432303,"std":0.003234},{"mean":0.472788,"std":0.006129},{"mean":0.513152,"std":0.0029},{"mean":0.55703,"std":0.004707},{"mean":0.605333,"std":0.013104},{"mean":0.645152,"std":0.009825},{"mean":0.692788,"std":0.00731},{"mean":0.739636,"std":0.002799},{"mean":0.79097,"std":0.003546},{"mean":0.853939,"std":0.00042},{"mean":0.998242,"std":0.001927}],"selective_risk":[{"mean":0.005626,"std":0.004872},{"mean":0.005571,"std":0.002439},{"mean":0.005196,"std":0.002207},{"mean":0.00558,"std":0.003263},{"mean":0.005042,"std":0.00307},{"mean":0.004738,"std":0.002342},{"mean":0.004982,"std":0.001278},{"mean":0.005138,"std":0.000876},{"mean":0.005692,"std":0.000673},{"mean":0.00561,"std":0.000523},{"mean":0.005768,"std":0.000363},{"mean":0.006852,"std":0.000914},{"mean":0.007395,"std":0.001338},{"mean":0.007995,"std":0.001105},{"mean":0.008446,"std":0.000898},{"mean":0.009615,"std":0.001259},{"mean":0.012127,"std":0.000489},{"mean":0.018081,"std":0.00068},{"mean":0.034067,"std":0.001483},{"mean":0.1122,"std":0.002539}],"accuracy_8":[{"mean":0.821515,"std":0.000555},{"mean":0.823212,"std":0.000735},{"mean":0.825091,"std":0.002204},{"mean":0.827273,"std":0.00365},{"mean":0.831273,"std":0.006426},{"mean":0.834909,"std":0.006945},{"mean":0.838909,"std":0.006077},{"mean":0.842303,"std":0.004746},{"mean":0.845515,"std":0.004825},{"mean":0.849758,"std":0.003203},{"mean":0.854788,"std":0.003785},{"mean":0.859212,"std":0.003413},{"mean":0.863333,"std":0.002345},{"mean":0.872182,"std":0.001106},{"mean":0.878061,"std":0.000378},{"mean":0.888242,"std":0.003569},{"mean":0.897697,"std":0.002777},{"mean":0.905091,"std":0.001616},{"mean":0.909273,"std":0.001192},{"mean":0.887455,"std":0.002834}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.001155},{"mean":0.002333,"std":0.000577},{"mean":0.003,"std":0.001},{"mean":0.004333,"std":0.002517},{"mean":0.005,"std":0.003},{"mean":0.005333,"std":0.003055},{"mean":0.007,"std":0.002},{"mean":0.008667,"std":0.002082},{"mean":0.010667,"std":0.002082},{"mean":0.011667,"std":0.001528},{"mean":0.013333,"std":0.001528},{"mean":0.017,"std":0.002646},{"mean":0.019333,"std":0.003055},{"mean":0.023,"std":0.003606},{"mean":0.026,"std":0.002646},{"mean":0.032333,"std":0.003786},{"mean":0.044,"std":0.001732},{"mean":0.070667,"std":0.002309},{"mean":0.144667,"std":0.007506},{"mean":0.555,"std":0.013856}],"oos_misroute_rate":[{"mean":0.431,"std":0.001},{"mean":0.43,"std":0.002},{"mean":0.429333,"std":0.002309},{"mean":0.429333,"std":0.002309},{"mean":0.429333,"std":0.002309},{"mean":0.429667,"std":0.002887},{"mean":0.429667,"std":0.002887},{"mean":0.429333,"std":0.003215},{"mean":0.43,"std":0.002646},{"mean":0.429333,"std":0.002517},{"mean":0.429667,"std":0.001528},{"mean":0.43,"std":0.001732},{"mean":0.430333,"std":0.002082},{"mean":0.429,"std":0.003},{"mean":0.429,"std":0.003606},{"mean":0.429667,"std":0.003786},{"mean":0.429333,"std":0.007506},{"mean":0.425667,"std":0.007234},{"mean":0.430333,"std":0.002517},{"mean":0.558,"std":0.016523}],"llm_call_rate":[{"mean":0.956485,"std":0.000757},{"mean":0.912848,"std":0.001766},{"mean":0.870788,"std":0.00626},{"mean":0.824545,"std":0.004567},{"mean":0.782303,"std":0.003254},{"mean":0.742364,"std":0.005512},{"mean":0.695394,"std":0.003461},{"mean":0.646242,"std":0.004211},{"mean":0.606182,"std":0.003848},{"mean":0.567697,"std":0.003234},{"mean":0.527212,"std":0.006129},{"mean":0.486848,"std":0.0029},{"mean":0.44297,"std":0.004707},{"mean":0.394667,"std":0.013104},{"mean":0.354848,"std":0.009825},{"mean":0.307212,"std":0.00731},{"mean":0.260364,"std":0.002799},{"mean":0.20903,"std":0.003546},{"mean":0.146061,"std":0.00042},{"mean":0.001758,"std":0.001927}]}},"entropy":{"risk_coverage":[{"mean":0.004848,"std":0.004199},{"mean":0.006061,"std":0.002099},{"mean":0.005657,"std":0.00305},{"mean":0.005455,"std":0.003278},{"mean":0.004364,"std":0.002622},{"mean":0.004848,"std":0.001603},{"mean":0.005022,"std":0.0006},{"mean":0.005303,"std":0.000694},{"mean":0.005522,"std":0.000233},{"mean":0.006061,"std":0.000757},{"mean":0.006722,"std":0.001063},{"mean":0.007677,"std":0.000926},{"mean":0.008019,"std":0.000855},{"mean":0.00961,"std":0.00119},{"mean":0.012525,"std":0.000779},{"mean":0.019545,"std":0.000819},{"mean":0.032299,"std":0.001671},{"mean":0.053805,"std":0.00162},{"mean":0.083764,"std":0.001344},{"mean":0.113152,"std":0.003364}],"operating":{"tau":[{"mean":-0.246417,"std":0.005139},{"mean":-0.261898,"std":0.002825},{"mean":-0.275039,"std":0.004438},{"mean":-0.285914,"std":0.006526},{"mean":-0.294484,"std":0.007247},{"mean":-0.303666,"std":0.007523},{"mean":-0.313166,"std":0.007558},{"mean":-0.323776,"std":0.006562},{"mean":-0.333917,"std":0.006113},{"mean":-0.343231,"std":0.006346},{"mean":-0.354499,"std":0.007843},{"mean":-0.36648,"std":0.007152},{"mean":-0.381414,"std":0.00659},{"mean":-0.398248,"std":0.007145},{"mean":-0.421107,"std":0.009786},{"mean":-0.451476,"std":0.0152},{"mean":-0.508874,"std":0.027498},{"mean":-0.620455,"std":0.016754},{"mean":-0.915379,"std":0.045445},{"mean":-1.984239,"std":0.063475}],"coverage":[{"mean":0.044303,"std":0.001944},{"mean":0.087091,"std":0.006203},{"mean":0.126121,"std":0.003737},{"mean":0.174727,"std":0.004329},{"mean":0.215212,"std":0.003203},{"mean":0.259576,"std":0.003674},{"mean":0.304485,"std":0.002699},{"mean":0.351636,"std":0.004727},{"mean":0.391152,"std":0.003737},{"mean":0.43,"std":0.001575},{"mean":0.474848,"std":0.00498},{"mean":0.514848,"std":0.002819},{"mean":0.558242,"std":0.002911},{"mean":0.600424,"std":0.014077},{"mean":0.645697,"std":0.009766},{"mean":0.690121,"std":0.006765},{"mean":0.741212,"std":0.003279},{"mean":0.790303,"std":0.002099},{"mean":0.851394,"std":0.003745},{"mean":0.996667,"std":0.002922}],"selective_risk":[{"mean":0.005614,"std":0.004862},{"mean":0.006378,"std":0.0025},{"mean":0.004806,"std":0.001668},{"mean":0.005603,"std":0.003286},{"mean":0.005096,"std":0.003108},{"mean":0.004456,"std":0.002185},{"mean":0.004983,"std":0.001274},{"mean":0.004999,"std":0.0006},{"mean":0.005116,"std":0.000836},{"mean":0.005215,"std":0.000235},{"mean":0.005868,"std":0.000527},{"mean":0.006355,"std":0.000916},{"mean":0.007055,"std":0.001304},{"mean":0.007656,"std":0.001065},{"mean":0.007781,"std":0.001182},{"mean":0.009388,"std":0.001472},{"mean":0.011283,"std":0.000382},{"mean":0.017638,"std":0.000318},{"mean":0.032959,"std":0.001268},{"mean":0.111527,"std":0.001953}],"accuracy_8":[{"mean":0.821515,"std":0.000639},{"mean":0.823515,"std":0.001238},{"mean":0.825333,"std":0.002945},{"mean":0.827515,"std":0.003922},{"mean":0.83097,"std":0.006129},{"mean":0.834909,"std":0.007406},{"mean":0.838485,"std":0.006331},{"mean":0.842424,"std":0.004746},{"mean":0.846364,"std":0.005357},{"mean":0.850303,"std":0.006113},{"mean":0.856182,"std":0.003988},{"mean":0.860788,"std":0.002783},{"mean":0.865333,"std":0.001001},{"mean":0.872182,"std":0.000656},{"mean":0.880242,"std":0.002154},{"mean":0.888909,"std":0.004442},{"mean":0.899091,"std":0.001791},{"mean":0.905455,"std":0.001488},{"mean":0.909273,"std":0.001311},{"mean":0.887576,"std":0.002753}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.001155},{"mean":0.002667,"std":0.000577},{"mean":0.003,"std":0.001},{"mean":0.004333,"std":0.002517},{"mean":0.005,"std":0.003},{"mean":0.005333,"std":0.002517},{"mean":0.007,"std":0.002},{"mean":0.008333,"std":0.001528},{"mean":0.009333,"std":0.002309},{"mean":0.010667,"std":0.001155},{"mean":0.013667,"std":0.002082},{"mean":0.015667,"std":0.002887},{"mean":0.018333,"std":0.003215},{"mean":0.021667,"std":0.003512},{"mean":0.023667,"std":0.003512},{"mean":0.031333,"std":0.004619},{"mean":0.040333,"std":0.001528},{"mean":0.068333,"std":0.001528},{"mean":0.138,"std":0.006083},{"mean":0.55,"std":0.009644}],"oos_misroute_rate":[{"mean":0.431,"std":0.001},{"mean":0.430667,"std":0.001528},{"mean":0.429667,"std":0.002082},{"mean":0.429333,"std":0.002309},{"mean":0.429333,"std":0.002309},{"mean":0.429333,"std":0.002309},{"mean":0.43,"std":0.002646},{"mean":0.429667,"std":0.002887},{"mean":0.429,"std":0.003464},{"mean":0.429667,"std":0.003055},{"mean":0.429667,"std":0.001528},{"mean":0.43,"std":0.001732},{"mean":0.43,"std":0.001732},{"mean":0.43,"std":0.001732},{"mean":0.428333,"std":0.003512},{"mean":0.429667,"std":0.003055},{"mean":0.429667,"std":0.005508},{"mean":0.426667,"std":0.007234},{"mean":0.430667,"std":0.002082},{"mean":0.557,"std":0.016093}],"llm_call_rate":[{"mean":0.955697,"std":0.001944},{"mean":0.912909,"std":0.006203},{"mean":0.873879,"std":0.003737},{"mean":0.825273,"std":0.004329},{"mean":0.784788,"std":0.003203},{"mean":0.740424,"std":0.003674},{"mean":0.695515,"std":0.002699},{"mean":0.648364,"std":0.004727},{"mean":0.608848,"std":0.003737},{"mean":0.57,"std":0.001575},{"mean":0.525152,"std":0.00498},{"mean":0.485152,"std":0.002819},{"mean":0.441758,"std":0.002911},{"mean":0.399576,"std":0.014077},{"mean":0.354303,"std":0.009766},{"mean":0.309879,"std":0.006765},{"mean":0.258788,"std":0.003279},{"mean":0.209697,"std":0.002099},{"mean":0.148606,"std":0.003745},{"mean":0.003333,"std":0.002922}]}},"margin":{"risk_coverage":[{"mean":0.006061,"std":0.00757},{"mean":0.004848,"std":0.002777},{"mean":0.006869,"std":0.0007},{"mean":0.007576,"std":0.000525},{"mean":0.008485,"std":0.001111},{"mean":0.008687,"std":0.0014},{"mean":0.008312,"std":0.001374},{"mean":0.008182,"std":0.000455},{"mean":0.007946,"std":0.000841},{"mean":0.008485,"std":0.001169},{"mean":0.008926,"std":0.001192},{"mean":0.009495,"std":0.000926},{"mean":0.010629,"std":0.00074},{"mean":0.012987,"std":0.00045},{"mean":0.016727,"std":0.001212},{"mean":0.022652,"std":0.000798},{"mean":0.034581,"std":0.002287},{"mean":0.054141,"std":0.002109},{"mean":0.079298,"std":0.000963},{"mean":0.113152,"std":0.003364}],"operating":{"tau":[{"mean":6.75174,"std":0.112736},{"mean":6.616531,"std":0.085581},{"mean":6.528343,"std":0.091023},{"mean":6.436667,"std":0.087549},{"mean":6.346472,"std":0.072656},{"mean":6.266876,"std":0.054464},{"mean":6.198656,"std":0.048707},{"mean":6.113526,"std":0.051731},{"mean":6.034352,"std":0.048159},{"mean":5.952632,"std":0.043811},{"mean":5.875117,"std":0.047456},{"mean":5.781801,"std":0.052476},{"mean":5.677515,"std":0.077383},{"mean":5.550008,"std":0.092688},{"mean":5.392732,"std":0.104762},{"mean":5.19387,"std":0.111474},{"mean":4.876971,"std":0.113286},{"mean":4.32019,"std":0.106344},{"mean":3.029941,"std":0.223732},{"mean":0.003754,"std":0.003737}],"coverage":[{"mean":0.049515,"std":0.003254},{"mean":0.096,"std":0.009278},{"mean":0.13503,"std":0.014305},{"mean":0.176727,"std":0.010581},{"mean":0.222182,"std":0.008043},{"mean":0.264485,"std":0.00537},{"mean":0.301515,"std":0.006273},{"mean":0.345939,"std":0.006587},{"mean":0.387273,"std":0.002},{"mean":0.431333,"std":0.001857},{"mean":0.471212,"std":0.006414},{"mean":0.513515,"std":0.007323},{"mean":0.55497,"std":0.006406},{"mean":0.599636,"std":0.006556},{"mean":0.649455,"std":0.009001},{"mean":0.689939,"std":0.012704},{"mean":0.740848,"std":0.007968},{"mean":0.796,"std":0.004923},{"mean":0.867758,"std":0.007508},{"mean":0.999576,"std":0.000278}],"selective_risk":[{"mean":0.004954,"std":0.005879},{"mean":0.005212,"std":0.003344},{"mean":0.007207,"std":0.000713},{"mean":0.006188,"std":0.000383},{"mean":0.008479,"std":0.001419},{"mean":0.008481,"std":0.000437},{"mean":0.008646,"std":0.001402},{"mean":0.008416,"std":0.001439},{"mean":0.008294,"std":0.000544},{"mean":0.00815,"std":0.000648},{"mean":0.008241,"std":0.001011},{"mean":0.008981,"std":0.001364},{"mean":0.008952,"std":0.001214},{"mean":0.009598,"std":0.000921},{"mean":0.010351,"std":0.000883},{"mean":0.012467,"std":0.000928},{"mean":0.015788,"std":0.000315},{"mean":0.022232,"std":0.000556},{"mean":0.041453,"std":0.006059},{"mean":0.112957,"std":0.003192}],"accuracy_8":[{"mean":0.821697,"std":0.000933},{"mean":0.824,"std":0.002526},{"mean":0.825758,"std":0.003379},{"mean":0.829152,"std":0.002548},{"mean":0.832788,"std":0.002554},{"mean":0.83697,"std":0.001111},{"mean":0.839818,"std":0.001636},{"mean":0.843818,"std":0.001734},{"mean":0.847394,"std":0.003077},{"mean":0.850606,"std":0.00371},{"mean":0.854424,"std":0.003546},{"mean":0.858606,"std":0.003692},{"mean":0.864182,"std":0.001311},{"mean":0.869697,"std":0.000915},{"mean":0.875697,"std":0.001183},{"mean":0.882485,"std":0.001718},{"mean":0.89297,"std":0.00082},{"mean":0.902909,"std":0.003107},{"mean":0.907879,"std":0.001211},{"mean":0.88703,"std":0.003213}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.001},{"mean":0.002,"std":0.001},{"mean":0.004667,"std":0.001155},{"mean":0.005333,"std":0.000577},{"mean":0.009,"std":0.001732},{"mean":0.011,"std":0.001},{"mean":0.013,"std":0.002646},{"mean":0.014333,"std":0.002309},{"mean":0.015667,"std":0.001528},{"mean":0.017,"std":0.001732},{"mean":0.019,"std":0.002},{"mean":0.022,"std":0.003},{"mean":0.024,"std":0.003464},{"mean":0.028,"std":0.002646},{"mean":0.033,"std":0.002646},{"mean":0.042667,"std":0.004041},{"mean":0.058667,"std":0.002887},{"mean":0.09,"std":0.002},{"mean":0.180667,"std":0.02669},{"mean":0.559667,"std":0.018339}],"oos_misroute_rate":[{"mean":0.431667,"std":0.001528},{"mean":0.431,"std":0.002},{"mean":0.431333,"std":0.001528},{"mean":0.43,"std":0.003},{"mean":0.430667,"std":0.002517},{"mean":0.430667,"std":0.003215},{"mean":0.43,"std":0.003606},{"mean":0.430333,"std":0.003055},{"mean":0.430333,"std":0.003786},{"mean":0.43,"std":0.004359},{"mean":0.43,"std":0.003},{"mean":0.43,"std":0.002},{"mean":0.428667,"std":0.003055},{"mean":0.428333,"std":0.006658},{"mean":0.427,"std":0.00781},{"mean":0.429333,"std":0.009292},{"mean":0.426667,"std":0.007234},{"mean":0.427,"std":0.007},{"mean":0.436,"std":0.003606},{"mean":0.56,"std":0.01852}],"llm_call_rate":[{"mean":0.950485,"std":0.003254},{"mean":0.904,"std":0.009278},{"mean":0.86497,"std":0.014305},{"mean":0.823273,"std":0.010581},{"mean":0.777818,"std":0.008043},{"mean":0.735515,"std":0.00537},{"mean":0.698485,"std":0.006273},{"mean":0.654061,"std":0.006587},{"mean":0.612727,"std":0.002},{"mean":0.568667,"std":0.001857},{"mean":0.528788,"std":0.006414},{"mean":0.486485,"std":0.007323},{"mean":0.44503,"std":0.006406},{"mean":0.400364,"std":0.006556},{"mean":0.350545,"std":0.009001},{"mean":0.310061,"std":0.012704},{"mean":0.259152,"std":0.007968},{"mean":0.204,"std":0.004923},{"mean":0.132242,"std":0.007508},{"mean":0.000424,"std":0.000278}]}},"msp_t":{"risk_coverage":[{"mean":0.004848,"std":0.004199},{"mean":0.003636,"std":0.003149},{"mean":0.006061,"std":0.003207},{"mean":0.005152,"std":0.002922},{"mean":0.004848,"std":0.00168},{"mean":0.005051,"std":0.00175},{"mean":0.005022,"std":0.002099},{"mean":0.005303,"std":0.00105},{"mean":0.005387,"std":0.000841},{"mean":0.006545,"std":0.00063},{"mean":0.007493,"std":0.001161},{"mean":0.007475,"std":0.000974},{"mean":0.008578,"std":0.000899},{"mean":0.009957,"std":0.000397},{"mean":0.013576,"std":0.000485},{"mean":0.020455,"std":0.000787},{"mean":0.03287,"std":0.001869},{"mean":0.051448,"std":0.001516},{"mean":0.078596,"std":0.001169},{"mean":0.113152,"std":0.003364}],"operating":{"tau":[{"mean":0.99606,"std":0.000455},{"mean":0.995632,"std":0.000378},{"mean":0.995291,"std":0.000318},{"mean":0.994942,"std":0.00023},{"mean":0.994626,"std":0.000202},{"mean":0.994335,"std":0.000208},{"mean":0.99396,"std":0.000185},{"mean":0.993583,"std":0.000198},{"mean":0.993205,"std":0.000221},{"mean":0.992843,"std":0.000242},{"mean":0.99236,"std":0.000258},{"mean":0.991792,"std":0.000347},{"mean":0.991055,"std":0.00055},{"mean":0.990149,"std":0.000813},{"mean":0.988896,"std":0.000889},{"mean":0.986757,"std":0.001094},{"mean":0.982529,"std":0.001555},{"mean":0.972482,"std":0.001519},{"mean":0.918182,"std":0.007835},{"mean":0.245849,"std":0.031899}],"coverage":[{"mean":0.045879,"std":0.002783},{"mean":0.089576,"std":0.00701},{"mean":0.132848,"std":0.005261},{"mean":0.180909,"std":0.00387},{"mean":0.216909,"std":0.003511},{"mean":0.257576,"std":0.006988},{"mean":0.306485,"std":0.005842},{"mean":0.349758,"std":0.003993},{"mean":0.390182,"std":0.001734},{"mean":0.42903,"std":0.003546},{"mean":0.472303,"std":0.004693},{"mean":0.513939,"std":0.00021},{"mean":0.558424,"std":0.006587},{"mean":0.602545,"std":0.008236},{"mean":0.644848,"std":0.008934},{"mean":0.692364,"std":0.007759},{"mean":0.741394,"std":0.007422},{"mean":0.792848,"std":0.001892},{"mean":0.860545,"std":0.003549},{"mean":0.998061,"std":0.001961}],"selective_risk":[{"mean":0.005362,"std":0.004667},{"mean":0.002818,"std":0.002448},{"mean":0.006007,"std":0.003012},{"mean":0.0054,"std":0.002976},{"mean":0.005054,"std":0.002283},{"mean":0.004735,"std":0.001726},{"mean":0.004966,"std":0.001786},{"mean":0.005038,"std":0.001716},{"mean":0.005285,"std":0.001365},{"mean":0.005514,"std":0.000893},{"mean":0.005899,"std":0.001205},{"mean":0.006722,"std":0.000938},{"mean":0.007594,"std":0.00096},{"mean":0.007536,"std":0.000959},{"mean":0.008544,"std":0.000964},{"mean":0.00971,"std":0.000844},{"mean":0.012912,"std":0.000745},{"mean":0.018728,"std":9e-05},{"mean":0.035772,"std":0.002783},{"mean":0.11149,"std":0.002587}],"accuracy_8":[{"mean":0.821515,"std":0.000555},{"mean":0.824061,"std":0.001525},{"mean":0.825091,"std":0.001924},{"mean":0.827939,"std":0.003305},{"mean":0.831818,"std":0.004556},{"mean":0.835636,"std":0.006182},{"mean":0.839333,"std":0.006172},{"mean":0.842727,"std":0.005645},{"mean":0.845758,"std":0.006015},{"mean":0.849515,"std":0.004171},{"mean":0.854424,"std":0.003714},{"mean":0.859515,"std":0.003093},{"mean":0.864727,"std":0.001273},{"mean":0.870424,"std":0.001389},{"mean":0.877879,"std":0.000859},{"mean":0.886909,"std":0.000315},{"mean":0.898061,"std":0.002939},{"mean":0.905515,"std":0.00147},{"mean":0.909515,"std":0.00105},{"mean":0.887879,"std":0.002548}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.001155},{"mean":0.001333,"std":0.001155},{"mean":0.003333,"std":0.001528},{"mean":0.004333,"std":0.002082},{"mean":0.004667,"std":0.002517},{"mean":0.005333,"std":0.002082},{"mean":0.007,"std":0.002646},{"mean":0.008333,"std":0.003055},{"mean":0.01,"std":0.002646},{"mean":0.011667,"std":0.002082},{"mean":0.014,"std":0.003464},{"mean":0.016667,"std":0.003055},{"mean":0.019667,"std":0.002517},{"mean":0.021333,"std":0.002887},{"mean":0.026333,"std":0.003055},{"mean":0.032667,"std":0.002309},{"mean":0.046667,"std":0.002517},{"mean":0.074,"std":0.001},{"mean":0.153333,"std":0.012055},{"mean":0.552,"std":0.013748}],"oos_misroute_rate":[{"mean":0.431,"std":0.001},{"mean":0.43,"std":0.002},{"mean":0.429333,"std":0.002309},{"mean":0.429333,"std":0.002309},{"mean":0.429333,"std":0.002309},{"mean":0.429667,"std":0.002887},{"mean":0.429333,"std":0.003215},{"mean":0.429,"std":0.003606},{"mean":0.429333,"std":0.003512},{"mean":0.429667,"std":0.003512},{"mean":0.429667,"std":0.003512},{"mean":0.428667,"std":0.003215},{"mean":0.428,"std":0.003},{"mean":0.428667,"std":0.003055},{"mean":0.429333,"std":0.003512},{"mean":0.429667,"std":0.005508},{"mean":0.428,"std":0.007937},{"mean":0.425667,"std":0.008083},{"mean":0.427667,"std":0.006658},{"mean":0.556,"std":0.014799}],"llm_call_rate":[{"mean":0.954121,"std":0.002783},{"mean":0.910424,"std":0.00701},{"mean":0.867152,"std":0.005261},{"mean":0.819091,"std":0.00387},{"mean":0.783091,"std":0.003511},{"mean":0.742424,"std":0.006988},{"mean":0.693515,"std":0.005842},{"mean":0.650242,"std":0.003993},{"mean":0.609818,"std":0.001734},{"mean":0.57097,"std":0.003546},{"mean":0.527697,"std":0.004693},{"mean":0.486061,"std":0.00021},{"mean":0.441576,"std":0.006587},{"mean":0.397455,"std":0.008236},{"mean":0.355152,"std":0.008934},{"mean":0.307636,"std":0.007759},{"mean":0.258606,"std":0.007422},{"mean":0.207152,"std":0.001892},{"mean":0.139455,"std":0.003549},{"mean":0.001939,"std":0.001961}]}}}}},"majority/k1":{"model":"majority","k":1,"oos_train":null,"kind":"majority","seeds":[42,43,44],"run_names":["majority-k1-seed42","majority-k1-seed43","majority-k1-seed44"],"logits_sha256":["ad9e1099ed7d88354021da1cda7ae310d1313af9d7d4654eafd03fabb8318e34","1b6de003a49322dea12ae7a224e9e6934b084fbbf6014bf7cb6721950b9d80e6","52a40757315fd9be80c2fd0cf5329f0b4c446ff25cf9a23ff714beab956155c9"],"result":{"argmax":{},"summed":{}}},"majority/k10":{"model":"majority","k":10,"oos_train":null,"kind":"majority","seeds":[42,43,44],"run_names":["majority-k10-seed42","majority-k10-seed43","majority-k10-seed44"],"logits_sha256":["73c39d0e3e96319a1a3e339c4f7beeb57fa9c05ffbdda933ad50d8da0b47ce42","10fd16db95cd9c12358fdb5a5d5d17c0ca0f6248c6f2c7506e03c8d30da5d650","68305d812ce5fa3dc38d311014e4231cb19cb2a5fc25548011c3d448accfd491"],"result":{"argmax":{},"summed":{}}},"majority/k100":{"model":"majority","k":100,"oos_train":null,"kind":"majority","seeds":[42,43,44],"run_names":["majority-k100-seed42","majority-k100-seed43","majority-k100-seed44"],"logits_sha256":["45de4fe9a649263e6470be6fdbe46e8a4fceb5a97a0afe36c21062b7d6447fb4","d57231c421661de1827d36c2b11d1a9cba0d5107f57447d639b25a70bcc46bb6","7422da060f87bd4325a17ef3f38948acf314349861f27c352038cbad2834c72b"],"result":{"argmax":{},"summed":{}}},"majority/k25":{"model":"majority","k":25,"oos_train":null,"kind":"majority","seeds":[42,43,44],"run_names":["majority-k25-seed42","majority-k25-seed43","majority-k25-seed44"],"logits_sha256":["0847cf359ec51cb09bfec7ae8dfea24c09254963a5ea73a82875b4d1780c593e","1d6145a48b53b954eba6a1d453b77f5eafc5f9cada582ff9012f7d39f1201ad5","be5db7a3d7fda2f50097788b959e62a40acd27100a64ad4350f1527883183e1f"],"result":{"argmax":{},"summed":{}}},"majority/k5":{"model":"majority","k":5,"oos_train":null,"kind":"majority","seeds":[42,43,44],"run_names":["majority-k5-seed42","majority-k5-seed43","majority-k5-seed44"],"logits_sha256":["9e6bb6f163ed274f4c53aedd58604b2c1259d3c083384ea7fb2d49455f9db466","3314ad9a70b585c6f4fccb9eb0bd56a5df209206e20b552c464727e10bbaba98","1da11415daed0f054fcb2477402f323bdcc4c0908abfebacfc116bc6f1ec5837"],"result":{"argmax":{},"summed":{}}},"majority/k50":{"model":"majority","k":50,"oos_train":null,"kind":"majority","seeds":[42,43,44],"run_names":["majority-k50-seed42","majority-k50-seed43","majority-k50-seed44"],"logits_sha256":["7c4872ed6ac1c993a1850633e26c44fac500db0efdc4923ea6a1ad1486a5ef97","ddd5754bd4acf617e268c6c68c49b3ab257bef78fb1e41e56cdb2d5508c3638e","15276ebf3a368cb46ccb4beb03a529067369266cbfcfe1d5e6294e022d374e7c"],"result":{"argmax":{},"summed":{}}},"modernbert-oos0/k100":{"model":"modernbert-oos0","k":100,"oos_train":0,"kind":"encoder","seeds":[42,43,44],"run_names":["ModernBERT-base-k100-oos0-seed42","ModernBERT-base-k100-oos0-seed43","ModernBERT-base-k100-oos0-seed44"],"logits_sha256":["c83e3c569facc09d5eb14cc3f837883fb59b0d5d68583257e9366e3750066be6","09029343a7529ac08bbc145fef45edcc1bfb04f6b976dc956473c6c546175343","6edfa2106d6d72ae26449b5598cff759ae601c44c42648de129ceab6e5530a61"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.000525},{"mean":0.000727,"std":0.0},{"mean":0.001212,"std":0.000606},{"mean":0.001385,"std":0.0003},{"mean":0.001818,"std":0.000787},{"mean":0.002559,"std":0.000617},{"mean":0.002909,"std":0.00063},{"mean":0.003636,"std":0.000573},{"mean":0.004444,"std":0.000463},{"mean":0.005967,"std":0.000982},{"mean":0.008052,"std":0.000779},{"mean":0.014626,"std":0.001852},{"mean":0.032197,"std":0.001933},{"mean":0.066809,"std":0.000864},{"mean":0.105253,"std":0.0007},{"mean":0.149091,"std":0.000383},{"mean":0.190909,"std":0.000364}],"operating":{"tau":[{"mean":0.999991,"std":1e-06},{"mean":0.999985,"std":1e-06},{"mean":0.999979,"std":1e-06},{"mean":0.999971,"std":2e-06},{"mean":0.999963,"std":2e-06},{"mean":0.999952,"std":3e-06},{"mean":0.999939,"std":5e-06},{"mean":0.999922,"std":7e-06},{"mean":0.999901,"std":9e-06},{"mean":0.999873,"std":1.1e-05},{"mean":0.999833,"std":1.8e-05},{"mean":0.999774,"std":2.8e-05},{"mean":0.999683,"std":3.6e-05},{"mean":0.999498,"std":5.5e-05},{"mean":0.999146,"std":5.5e-05},{"mean":0.998351,"std":0.000124},{"mean":0.99544,"std":0.000517},{"mean":0.976008,"std":0.001175},{"mean":0.752628,"std":0.014199},{"mean":0.086392,"std":0.017794}],"coverage":[{"mean":0.042606,"std":0.004131},{"mean":0.081879,"std":0.004215},{"mean":0.127273,"std":0.004287},{"mean":0.174848,"std":0.010197},{"mean":0.216424,"std":0.015926},{"mean":0.260667,"std":0.015601},{"mean":0.298606,"std":0.016948},{"mean":0.342303,"std":0.017339},{"mean":0.381818,"std":0.011555},{"mean":0.419636,"std":0.008491},{"mean":0.461939,"std":0.011677},{"mean":0.50303,"std":0.012523},{"mean":0.541152,"std":0.007094},{"mean":0.584364,"std":0.004567},{"mean":0.629818,"std":0.002144},{"mean":0.670364,"std":0.007613},{"mean":0.714424,"std":0.004187},{"mean":0.767152,"std":0.002956},{"mean":0.837697,"std":0.001277},{"mean":0.99903,"std":0.001001}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000843,"std":6.4e-05},{"mean":0.000699,"std":4.2e-05},{"mean":0.001234,"std":0.000638},{"mean":0.001596,"std":8e-05},{"mean":0.001593,"std":0.000323},{"mean":0.00232,"std":0.000712},{"mean":0.002504,"std":0.00067},{"mean":0.002779,"std":0.000487},{"mean":0.00359,"std":0.000747},{"mean":0.004045,"std":0.000313},{"mean":0.005195,"std":0.000482},{"mean":0.006689,"std":0.000368},{"mean":0.009159,"std":0.000868},{"mean":0.019039,"std":0.002037},{"mean":0.057082,"std":0.001627},{"mean":0.190123,"std":0.001029}],"accuracy_8":[{"mean":0.825152,"std":0.001211},{"mean":0.829515,"std":0.002911},{"mean":0.834788,"std":0.003},{"mean":0.838727,"std":0.003367},{"mean":0.843212,"std":0.003374},{"mean":0.848424,"std":0.002854},{"mean":0.852545,"std":0.003933},{"mean":0.856909,"std":0.003406},{"mean":0.860667,"std":0.003403},{"mean":0.864182,"std":0.00365},{"mean":0.869758,"std":0.002917},{"mean":0.874909,"std":0.003912},{"mean":0.878485,"std":0.003661},{"mean":0.883818,"std":0.002545},{"mean":0.889636,"std":0.002271},{"mean":0.893939,"std":0.001546},{"mean":0.89903,"std":0.000735},{"mean":0.901818,"std":0.000909},{"mean":0.892061,"std":0.000378},{"mean":0.809879,"std":0.001001}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.000577},{"mean":0.002,"std":0.001},{"mean":0.002333,"std":0.000577},{"mean":0.004333,"std":0.000577},{"mean":0.005,"std":0.001},{"mean":0.006333,"std":0.000577},{"mean":0.008667,"std":0.002082},{"mean":0.011,"std":0.001},{"mean":0.015333,"std":0.001528},{"mean":0.021667,"std":0.001528},{"mean":0.032,"std":0.003606},{"mean":0.072667,"std":0.008021},{"mean":0.241667,"std":0.005859},{"mean":0.994667,"std":0.005508}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.433667,"std":0.001155},{"mean":0.434,"std":0.001},{"mean":0.434333,"std":0.001528},{"mean":0.435333,"std":0.002517},{"mean":0.436667,"std":0.002082},{"mean":0.437333,"std":0.002082},{"mean":0.441667,"std":0.001155},{"mean":0.446333,"std":0.002309},{"mean":0.463333,"std":0.001155},{"mean":0.541333,"std":0.004933},{"mean":0.995667,"std":0.004041}],"llm_call_rate":[{"mean":0.957394,"std":0.004131},{"mean":0.918121,"std":0.004215},{"mean":0.872727,"std":0.004287},{"mean":0.825152,"std":0.010197},{"mean":0.783576,"std":0.015926},{"mean":0.739333,"std":0.015601},{"mean":0.701394,"std":0.016948},{"mean":0.657697,"std":0.017339},{"mean":0.618182,"std":0.011555},{"mean":0.580364,"std":0.008491},{"mean":0.538061,"std":0.011677},{"mean":0.49697,"std":0.012523},{"mean":0.458848,"std":0.007094},{"mean":0.415636,"std":0.004567},{"mean":0.370182,"std":0.002144},{"mean":0.329636,"std":0.007613},{"mean":0.285576,"std":0.004187},{"mean":0.232848,"std":0.002956},{"mean":0.162303,"std":0.001277},{"mean":0.00097,"std":0.001001}]}},"entropy":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000909,"std":0.0},{"mean":0.00097,"std":0.00042},{"mean":0.00101,"std":0.00035},{"mean":0.001385,"std":0.0003},{"mean":0.001818,"std":0.000787},{"mean":0.002559,"std":0.000617},{"mean":0.002909,"std":0.00063},{"mean":0.003747,"std":0.000505},{"mean":0.004242,"std":0.000525},{"mean":0.005594,"std":0.000839},{"mean":0.007879,"std":0.000654},{"mean":0.013899,"std":0.001378},{"mean":0.030227,"std":0.002083},{"mean":0.060535,"std":0.000932},{"mean":0.103704,"std":0.000421},{"mean":0.148517,"std":0.000574},{"mean":0.190909,"std":0.000364}],"operating":{"tau":[{"mean":-0.000137,"std":1e-05},{"mean":-0.000223,"std":1e-05},{"mean":-0.000316,"std":1.4e-05},{"mean":-0.000419,"std":2.1e-05},{"mean":-0.000531,"std":2.8e-05},{"mean":-0.000657,"std":3.6e-05},{"mean":-0.000816,"std":5e-05},{"mean":-0.001017,"std":7e-05},{"mean":-0.001278,"std":9e-05},{"mean":-0.001607,"std":0.000134},{"mean":-0.002038,"std":0.000185},{"mean":-0.002717,"std":0.000293},{"mean":-0.003686,"std":0.000415},{"mean":-0.00552,"std":0.000514},{"mean":-0.008877,"std":0.000653},{"mean":-0.015405,"std":0.00102},{"mean":-0.037329,"std":0.00399},{"mean":-0.157563,"std":0.008977},{"mean":-0.834926,"std":0.031835},{"mean":-3.949794,"std":0.042275}],"coverage":[{"mean":0.042242,"std":0.003789},{"mean":0.082061,"std":0.003555},{"mean":0.128545,"std":0.005068},{"mean":0.174545,"std":0.010092},{"mean":0.216909,"std":0.015156},{"mean":0.259455,"std":0.015461},{"mean":0.298364,"std":0.016726},{"mean":0.340121,"std":0.015334},{"mean":0.381455,"std":0.010303},{"mean":0.419394,"std":0.008818},{"mean":0.461576,"std":0.011824},{"mean":0.504121,"std":0.013144},{"mean":0.542303,"std":0.009209},{"mean":0.584667,"std":0.006209},{"mean":0.629758,"std":0.005079},{"mean":0.669455,"std":0.008056},{"mean":0.713333,"std":0.005195},{"mean":0.767818,"std":0.003912},{"mean":0.83103,"std":0.000378},{"mean":0.999939,"std":0.000105}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000841,"std":6.1e-05},{"mean":0.000952,"std":0.000471},{"mean":0.00103,"std":0.000395},{"mean":0.001436,"std":0.000361},{"mean":0.001431,"std":3.9e-05},{"mean":0.00247,"std":0.000959},{"mean":0.002635,"std":0.000526},{"mean":0.002649,"std":0.000277},{"mean":0.003806,"std":0.00058},{"mean":0.003938,"std":0.000459},{"mean":0.004904,"std":0.000722},{"mean":0.006153,"std":0.000835},{"mean":0.008918,"std":0.000617},{"mean":0.018547,"std":0.001728},{"mean":0.045654,"std":0.00141},{"mean":0.19086,"std":0.000439}],"accuracy_8":[{"mean":0.82503,"std":0.001183},{"mean":0.829333,"std":0.002819},{"mean":0.834848,"std":0.003193},{"mean":0.838545,"std":0.003217},{"mean":0.843091,"std":0.003154},{"mean":0.848303,"std":0.0029},{"mean":0.852727,"std":0.003581},{"mean":0.85703,"std":0.003692},{"mean":0.860545,"std":0.003318},{"mean":0.864303,"std":0.003093},{"mean":0.869515,"std":0.003315},{"mean":0.875152,"std":0.004006},{"mean":0.878606,"std":0.004107},{"mean":0.883455,"std":0.002857},{"mean":0.889515,"std":0.001994},{"mean":0.894121,"std":0.001277},{"mean":0.898788,"std":0.001169},{"mean":0.902242,"std":0.000458},{"mean":0.897212,"std":0.000555},{"mean":0.809152,"std":0.000458}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001,"std":0.0},{"mean":0.001667,"std":0.000577},{"mean":0.002,"std":0.001},{"mean":0.004667,"std":0.001155},{"mean":0.005333,"std":0.000577},{"mean":0.006,"std":0.0},{"mean":0.009333,"std":0.001528},{"mean":0.010667,"std":0.001528},{"mean":0.014333,"std":0.002517},{"mean":0.019667,"std":0.003215},{"mean":0.030667,"std":0.002082},{"mean":0.070667,"std":0.007371},{"mean":0.186,"std":0.006083},{"mean":0.999667,"std":0.000577}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433667,"std":0.001155},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.435333,"std":0.002517},{"mean":0.436333,"std":0.002309},{"mean":0.437,"std":0.002},{"mean":0.44,"std":0.0},{"mean":0.445333,"std":0.002309},{"mean":0.461333,"std":0.000577},{"mean":0.513333,"std":0.004163},{"mean":0.999667,"std":0.000577}],"llm_call_rate":[{"mean":0.957758,"std":0.003789},{"mean":0.917939,"std":0.003555},{"mean":0.871455,"std":0.005068},{"mean":0.825455,"std":0.010092},{"mean":0.783091,"std":0.015156},{"mean":0.740545,"std":0.015461},{"mean":0.701636,"std":0.016726},{"mean":0.659879,"std":0.015334},{"mean":0.618545,"std":0.010303},{"mean":0.580606,"std":0.008818},{"mean":0.538424,"std":0.011824},{"mean":0.495879,"std":0.013144},{"mean":0.457697,"std":0.009209},{"mean":0.415333,"std":0.006209},{"mean":0.370242,"std":0.005079},{"mean":0.330545,"std":0.008056},{"mean":0.286667,"std":0.005195},{"mean":0.232182,"std":0.003912},{"mean":0.16897,"std":0.000378},{"mean":6.1e-05,"std":0.000105}]}},"margin":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000404,"std":0.0007},{"mean":0.000909,"std":0.000909},{"mean":0.00097,"std":0.001111},{"mean":0.001818,"std":0.000606},{"mean":0.002078,"std":0.000519},{"mean":0.002727,"std":0.001203},{"mean":0.002828,"std":0.001457},{"mean":0.003273,"std":0.001311},{"mean":0.004077,"std":0.000763},{"mean":0.006061,"std":0.001321},{"mean":0.007459,"std":0.000582},{"mean":0.010043,"std":0.001227},{"mean":0.01899,"std":0.003431},{"mean":0.037576,"std":0.00273},{"mean":0.074225,"std":0.000566},{"mean":0.114613,"std":0.001184},{"mean":0.15311,"std":0.000506},{"mean":0.190909,"std":0.000364}],"operating":{"tau":[{"mean":8.794198,"std":0.093849},{"mean":8.405874,"std":0.119563},{"mean":8.125384,"std":0.088764},{"mean":7.886135,"std":0.094117},{"mean":7.686995,"std":0.098954},{"mean":7.480884,"std":0.111692},{"mean":7.291912,"std":0.124329},{"mean":7.100677,"std":0.124409},{"mean":6.902363,"std":0.128804},{"mean":6.71314,"std":0.126262},{"mean":6.507833,"std":0.122567},{"mean":6.268752,"std":0.123287},{"mean":5.998427,"std":0.133426},{"mean":5.686351,"std":0.139242},{"mean":5.285365,"std":0.137177},{"mean":4.754737,"std":0.119659},{"mean":4.055149,"std":0.128912},{"mean":2.951159,"std":0.021441},{"mean":1.16803,"std":0.029019},{"mean":0.009622,"std":0.006818}],"coverage":[{"mean":0.044485,"std":0.000278},{"mean":0.086182,"std":0.005366},{"mean":0.129152,"std":0.003728},{"mean":0.170545,"std":0.01095},{"mean":0.210303,"std":0.009483},{"mean":0.256667,"std":0.016397},{"mean":0.297879,"std":0.019645},{"mean":0.340182,"std":0.01994},{"mean":0.383697,"std":0.016925},{"mean":0.420909,"std":0.016872},{"mean":0.459818,"std":0.011196},{"mean":0.500788,"std":0.007624},{"mean":0.541697,"std":0.006434},{"mean":0.582,"std":0.002909},{"mean":0.625091,"std":0.003677},{"mean":0.671455,"std":0.008196},{"mean":0.716788,"std":0.005376},{"mean":0.767091,"std":0.000833},{"mean":0.853152,"std":0.002612},{"mean":0.998061,"std":0.000897}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000736,"std":0.000638},{"mean":0.000892,"std":0.000904},{"mean":0.000992,"std":0.001158},{"mean":0.002053,"std":0.000477},{"mean":0.002143,"std":0.000127},{"mean":0.002386,"std":0.00058},{"mean":0.002919,"std":0.001454},{"mean":0.002922,"std":0.001357},{"mean":0.003401,"std":0.001229},{"mean":0.004031,"std":0.000618},{"mean":0.004996,"std":0.00124},{"mean":0.006783,"std":0.000892},{"mean":0.008031,"std":0.000343},{"mean":0.011831,"std":0.001863},{"mean":0.022755,"std":0.002482},{"mean":0.076717,"std":0.002177},{"mean":0.189397,"std":0.000701}],"accuracy_8":[{"mean":0.825212,"std":0.000735},{"mean":0.829515,"std":0.002107},{"mean":0.834121,"std":0.002723},{"mean":0.838485,"std":0.003858},{"mean":0.843212,"std":0.004265},{"mean":0.847818,"std":0.004182},{"mean":0.851152,"std":0.004543},{"mean":0.856182,"std":0.003217},{"mean":0.861273,"std":0.002834},{"mean":0.865333,"std":0.002777},{"mean":0.87,"std":0.003397},{"mean":0.874606,"std":0.003884},{"mean":0.879273,"std":0.003004},{"mean":0.883818,"std":0.002745},{"mean":0.888545,"std":0.001616},{"mean":0.894303,"std":0.001065},{"mean":0.898485,"std":0.000584},{"mean":0.899939,"std":0.001365},{"mean":0.88103,"std":0.002107},{"mean":0.810364,"std":0.000656}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.001},{"mean":0.002333,"std":0.000577},{"mean":0.003,"std":0.001},{"mean":0.004,"std":0.0},{"mean":0.005667,"std":0.002082},{"mean":0.006,"std":0.002646},{"mean":0.008,"std":0.002646},{"mean":0.010333,"std":0.001528},{"mean":0.014,"std":0.003},{"mean":0.021,"std":0.002646},{"mean":0.026667,"std":0.001528},{"mean":0.042667,"std":0.006658},{"mean":0.088667,"std":0.008386},{"mean":0.338667,"std":0.014978},{"mean":0.99,"std":0.005292}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433333,"std":0.000577},{"mean":0.434,"std":0.001},{"mean":0.434667,"std":0.002082},{"mean":0.435,"std":0.002646},{"mean":0.436,"std":0.002646},{"mean":0.436667,"std":0.002517},{"mean":0.437333,"std":0.002082},{"mean":0.440667,"std":0.001528},{"mean":0.444333,"std":0.001528},{"mean":0.452333,"std":0.001528},{"mean":0.475,"std":0.002},{"mean":0.598667,"std":0.015567},{"mean":0.993333,"std":0.004163}],"llm_call_rate":[{"mean":0.955515,"std":0.000278},{"mean":0.913818,"std":0.005366},{"mean":0.870848,"std":0.003728},{"mean":0.829455,"std":0.01095},{"mean":0.789697,"std":0.009483},{"mean":0.743333,"std":0.016397},{"mean":0.702121,"std":0.019645},{"mean":0.659818,"std":0.01994},{"mean":0.616303,"std":0.016925},{"mean":0.579091,"std":0.016872},{"mean":0.540182,"std":0.011196},{"mean":0.499212,"std":0.007624},{"mean":0.458303,"std":0.006434},{"mean":0.418,"std":0.002909},{"mean":0.374909,"std":0.003677},{"mean":0.328545,"std":0.008196},{"mean":0.283212,"std":0.005376},{"mean":0.232909,"std":0.000833},{"mean":0.146848,"std":0.002612},{"mean":0.001939,"std":0.000897}]}},"msp_t":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.000525},{"mean":0.00097,"std":0.00042},{"mean":0.00101,"std":0.00035},{"mean":0.001385,"std":0.0003},{"mean":0.001818,"std":0.000455},{"mean":0.002424,"std":0.000404},{"mean":0.002909,"std":0.000364},{"mean":0.003526,"std":0.000505},{"mean":0.003838,"std":0.000175},{"mean":0.005035,"std":0.00028},{"mean":0.007532,"std":0.0},{"mean":0.01301,"std":0.00142},{"mean":0.028788,"std":0.002062},{"mean":0.06082,"std":0.000617},{"mean":0.10431,"std":0.000421},{"mean":0.148772,"std":0.000442},{"mean":0.190909,"std":0.000364}],"operating":{"tau":[{"mean":0.998502,"std":0.000198},{"mean":0.997911,"std":0.000265},{"mean":0.997364,"std":0.000287},{"mean":0.996822,"std":0.000297},{"mean":0.996271,"std":0.000347},{"mean":0.99567,"std":0.000416},{"mean":0.995004,"std":0.000507},{"mean":0.994169,"std":0.000642},{"mean":0.993249,"std":0.000711},{"mean":0.992049,"std":0.000977},{"mean":0.990646,"std":0.001162},{"mean":0.988778,"std":0.001477},{"mean":0.98602,"std":0.00201},{"mean":0.981746,"std":0.002522},{"mean":0.975197,"std":0.003434},{"mean":0.9642,"std":0.004262},{"mean":0.935723,"std":0.006994},{"mean":0.831855,"std":0.007938},{"mean":0.494164,"std":0.006169},{"mean":0.048299,"std":0.008281}],"coverage":[{"mean":0.041273,"std":0.004299},{"mean":0.082909,"std":0.003946},{"mean":0.128727,"std":0.008876},{"mean":0.171879,"std":0.008637},{"mean":0.215939,"std":0.015002},{"mean":0.259394,"std":0.014264},{"mean":0.299697,"std":0.010735},{"mean":0.341455,"std":0.010647},{"mean":0.380182,"std":0.010219},{"mean":0.421879,"std":0.010864},{"mean":0.462485,"std":0.009504},{"mean":0.502485,"std":0.01076},{"mean":0.544485,"std":0.009549},{"mean":0.587818,"std":0.005101},{"mean":0.628303,"std":0.005124},{"mean":0.670121,"std":0.004312},{"mean":0.711697,"std":0.002637},{"mean":0.767697,"std":0.004798},{"mean":0.831636,"std":0.001443},{"mean":0.999212,"std":0.000735}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.00034,"std":0.00059},{"mean":0.000541,"std":0.000468},{"mean":0.000951,"std":0.000468},{"mean":0.001019,"std":0.000376},{"mean":0.001243,"std":0.000307},{"mean":0.001758,"std":0.000308},{"mean":0.002159,"std":0.000464},{"mean":0.002758,"std":0.00045},{"mean":0.002898,"std":0.000388},{"mean":0.003453,"std":0.000248},{"mean":0.003815,"std":0.000183},{"mean":0.004627,"std":0.000543},{"mean":0.005878,"std":0.001089},{"mean":0.007749,"std":0.000271},{"mean":0.017129,"std":0.001116},{"mean":0.046494,"std":0.000669},{"mean":0.190271,"std":0.000869}],"accuracy_8":[{"mean":0.824788,"std":0.00084},{"mean":0.82903,"std":0.002939},{"mean":0.834303,"std":0.003833},{"mean":0.838909,"std":0.003059},{"mean":0.843152,"std":0.003413},{"mean":0.848727,"std":0.00317},{"mean":0.853273,"std":0.003102},{"mean":0.85697,"std":0.003611},{"mean":0.86097,"std":0.003028},{"mean":0.865152,"std":0.003022},{"mean":0.869697,"std":0.003274},{"mean":0.874727,"std":0.00365},{"mean":0.87903,"std":0.003555},{"mean":0.884606,"std":0.002482},{"mean":0.888909,"std":0.002144},{"mean":0.893939,"std":0.001034},{"mean":0.898727,"std":0.000833},{"mean":0.902606,"std":0.000105},{"mean":0.89703,"std":0.001096},{"mean":0.809758,"std":0.000897}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001,"std":0.0},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.004,"std":0.001},{"mean":0.005667,"std":0.000577},{"mean":0.006333,"std":0.000577},{"mean":0.008333,"std":0.000577},{"mean":0.01,"std":0.001},{"mean":0.013333,"std":0.002082},{"mean":0.018333,"std":0.003786},{"mean":0.026333,"std":0.000577},{"mean":0.064,"std":0.005292},{"mean":0.19,"std":0.002646},{"mean":0.995667,"std":0.004041}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433333,"std":0.000577},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434667,"std":0.001528},{"mean":0.435667,"std":0.002082},{"mean":0.437,"std":0.002},{"mean":0.439,"std":0.001},{"mean":0.443333,"std":0.002082},{"mean":0.458,"std":0.001732},{"mean":0.515,"std":0.006083},{"mean":0.996333,"std":0.003215}],"llm_call_rate":[{"mean":0.958727,"std":0.004299},{"mean":0.917091,"std":0.003946},{"mean":0.871273,"std":0.008876},{"mean":0.828121,"std":0.008637},{"mean":0.784061,"std":0.015002},{"mean":0.740606,"std":0.014264},{"mean":0.700303,"std":0.010735},{"mean":0.658545,"std":0.010647},{"mean":0.619818,"std":0.010219},{"mean":0.578121,"std":0.010864},{"mean":0.537515,"std":0.009504},{"mean":0.497515,"std":0.01076},{"mean":0.455515,"std":0.009549},{"mean":0.412182,"std":0.005101},{"mean":0.371697,"std":0.005124},{"mean":0.329879,"std":0.004312},{"mean":0.288303,"std":0.002637},{"mean":0.232303,"std":0.004798},{"mean":0.168364,"std":0.001443},{"mean":0.000788,"std":0.000735}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000404,"std":0.0007},{"mean":0.000606,"std":0.000525},{"mean":0.000727,"std":0.0},{"mean":0.00101,"std":0.00035},{"mean":0.001558,"std":0.000519},{"mean":0.00197,"std":0.000694},{"mean":0.002559,"std":0.000467},{"mean":0.002788,"std":0.000915},{"mean":0.003416,"std":0.000505},{"mean":0.003636,"std":0.000525},{"mean":0.004662,"std":0.000704},{"mean":0.007359,"std":0.000654},{"mean":0.01196,"std":0.00014},{"mean":0.024394,"std":0.000919},{"mean":0.059037,"std":0.000932},{"mean":0.103973,"std":0.000816},{"mean":0.14807,"std":0.000797},{"mean":0.190182,"std":0.000727}],"operating":{"tau":[{"mean":0.999996,"std":0.0},{"mean":0.999993,"std":0.0},{"mean":0.999991,"std":1e-06},{"mean":0.999987,"std":1e-06},{"mean":0.999983,"std":2e-06},{"mean":0.999978,"std":2e-06},{"mean":0.999973,"std":2e-06},{"mean":0.999966,"std":3e-06},{"mean":0.999957,"std":3e-06},{"mean":0.999945,"std":4e-06},{"mean":0.999929,"std":4e-06},{"mean":0.999905,"std":3e-06},{"mean":0.999868,"std":7e-06},{"mean":0.999808,"std":1.4e-05},{"mean":0.999694,"std":9e-06},{"mean":0.999433,"std":1.9e-05},{"mean":0.998705,"std":6.9e-05},{"mean":0.994666,"std":0.000128},{"mean":0.935217,"std":0.004165},{"mean":0.226828,"std":0.013507}],"coverage":[{"mean":0.043394,"std":0.002068},{"mean":0.080121,"std":0.002911},{"mean":0.121515,"std":0.00382},{"mean":0.170061,"std":0.007168},{"mean":0.214121,"std":0.012608},{"mean":0.256,"std":0.00997},{"mean":0.295273,"std":0.006732},{"mean":0.335394,"std":0.010401},{"mean":0.375636,"std":0.006793},{"mean":0.42103,"std":0.00648},{"mean":0.459455,"std":0.005827},{"mean":0.50103,"std":0.003679},{"mean":0.545879,"std":0.006491},{"mean":0.587091,"std":0.007742},{"mean":0.629152,"std":0.002222},{"mean":0.674545,"std":0.003632},{"mean":0.716061,"std":0.002854},{"mean":0.763818,"std":0.001734},{"mean":0.826303,"std":0.000933},{"mean":0.998848,"std":0.000735}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000697,"std":0.000604},{"mean":0.000548,"std":0.000475},{"mean":0.000711,"std":2.8e-05},{"mean":0.001032,"std":0.000373},{"mean":0.001616,"std":0.000494},{"mean":0.001929,"std":0.000955},{"mean":0.002302,"std":0.000656},{"mean":0.002643,"std":0.000633},{"mean":0.002786,"std":0.000929},{"mean":0.003335,"std":0.000613},{"mean":0.003512,"std":0.0005},{"mean":0.004141,"std":0.000321},{"mean":0.00584,"std":0.00122},{"mean":0.008125,"std":0.000419},{"mean":0.014046,"std":0.000889},{"mean":0.039973,"std":0.000847},{"mean":0.189248,"std":0.001192}],"accuracy_8":[{"mean":0.823394,"std":0.000458},{"mean":0.826303,"std":0.00191},{"mean":0.829879,"std":0.002649},{"mean":0.834667,"std":0.003518},{"mean":0.839212,"std":0.003379},{"mean":0.843273,"std":0.003507},{"mean":0.847939,"std":0.002442},{"mean":0.852667,"std":0.003264},{"mean":0.856303,"std":0.003203},{"mean":0.861091,"std":0.003623},{"mean":0.865515,"std":0.003098},{"mean":0.869636,"std":0.002926},{"mean":0.874909,"std":0.0025},{"mean":0.880121,"std":0.002922},{"mean":0.885576,"std":0.002573},{"mean":0.891576,"std":0.001944},{"mean":0.897636,"std":0.001967},{"mean":0.901939,"std":0.000278},{"mean":0.899212,"std":0.000525},{"mean":0.810848,"std":0.001365}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001333,"std":0.000577},{"mean":0.002333,"std":0.001155},{"mean":0.003,"std":0.001732},{"mean":0.004333,"std":0.001155},{"mean":0.005333,"std":0.000577},{"mean":0.006333,"std":0.001528},{"mean":0.008333,"std":0.001528},{"mean":0.009333,"std":0.001528},{"mean":0.012,"std":0.001},{"mean":0.018667,"std":0.004509},{"mean":0.028667,"std":0.001528},{"mean":0.053333,"std":0.003512},{"mean":0.168667,"std":0.002082},{"mean":0.993667,"std":0.004041}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433667,"std":0.001528},{"mean":0.434333,"std":0.001528},{"mean":0.435,"std":0.002},{"mean":0.435667,"std":0.002082},{"mean":0.437667,"std":0.001155},{"mean":0.441667,"std":0.001155},{"mean":0.452333,"std":0.001528},{"mean":0.499667,"std":0.001528},{"mean":0.994333,"std":0.004509}],"llm_call_rate":[{"mean":0.956606,"std":0.002068},{"mean":0.919879,"std":0.002911},{"mean":0.878485,"std":0.00382},{"mean":0.829939,"std":0.007168},{"mean":0.785879,"std":0.012608},{"mean":0.744,"std":0.00997},{"mean":0.704727,"std":0.006732},{"mean":0.664606,"std":0.010401},{"mean":0.624364,"std":0.006793},{"mean":0.57897,"std":0.00648},{"mean":0.540545,"std":0.005827},{"mean":0.49897,"std":0.003679},{"mean":0.454121,"std":0.006491},{"mean":0.412909,"std":0.007742},{"mean":0.370848,"std":0.002222},{"mean":0.325455,"std":0.003632},{"mean":0.283939,"std":0.002854},{"mean":0.236182,"std":0.001734},{"mean":0.173697,"std":0.000933},{"mean":0.001152,"std":0.000735}]}},"entropy":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000404,"std":0.0007},{"mean":0.000606,"std":0.000525},{"mean":0.000727,"std":0.0},{"mean":0.00101,"std":0.00035},{"mean":0.001558,"std":0.000519},{"mean":0.00197,"std":0.000694},{"mean":0.002559,"std":0.000467},{"mean":0.002667,"std":0.000757},{"mean":0.003306,"std":0.000573},{"mean":0.003636,"std":0.000525},{"mean":0.004569,"std":0.000704},{"mean":0.007273,"std":0.000687},{"mean":0.011879,"std":0.000242},{"mean":0.023561,"std":0.00086},{"mean":0.057469,"std":0.000891},{"mean":0.102492,"std":0.001113},{"mean":0.147815,"std":0.000904},{"mean":0.190182,"std":0.000727}],"operating":{"tau":[{"mean":-6.1e-05,"std":3e-06},{"mean":-9.4e-05,"std":4e-06},{"mean":-0.000131,"std":8e-06},{"mean":-0.000179,"std":1.3e-05},{"mean":-0.000229,"std":2.3e-05},{"mean":-0.000284,"std":2.4e-05},{"mean":-0.000345,"std":2.9e-05},{"mean":-0.00043,"std":3.5e-05},{"mean":-0.000532,"std":3.3e-05},{"mean":-0.000676,"std":4.2e-05},{"mean":-0.00085,"std":4.1e-05},{"mean":-0.001105,"std":4.2e-05},{"mean":-0.001472,"std":7e-05},{"mean":-0.002089,"std":0.000139},{"mean":-0.003191,"std":0.000108},{"mean":-0.005572,"std":0.00013},{"mean":-0.011512,"std":0.000357},{"mean":-0.039631,"std":0.000541},{"mean":-0.314844,"std":0.021662},{"mean":-1.846056,"std":0.022957}],"coverage":[{"mean":0.043455,"std":0.00189},{"mean":0.079939,"std":0.003274},{"mean":0.121152,"std":0.00516},{"mean":0.170182,"std":0.007266},{"mean":0.214121,"std":0.013643},{"mean":0.256242,"std":0.010308},{"mean":0.294424,"std":0.00714},{"mean":0.334727,"std":0.010595},{"mean":0.375758,"std":0.006671},{"mean":0.420242,"std":0.00691},{"mean":0.459758,"std":0.00589},{"mean":0.501394,"std":0.00447},{"mean":0.545515,"std":0.006701},{"mean":0.586667,"std":0.007285},{"mean":0.629515,"std":0.002131},{"mean":0.674727,"std":0.003004},{"mean":0.715939,"std":0.002107},{"mean":0.763818,"std":0.001443},{"mean":0.826061,"std":0.000278},{"mean":0.997939,"std":0.001857}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000696,"std":0.000603},{"mean":0.000547,"std":0.000474},{"mean":0.00071,"std":2.9e-05},{"mean":0.001035,"std":0.000375},{"mean":0.001619,"std":0.000495},{"mean":0.001929,"std":0.000956},{"mean":0.002159,"std":0.000854},{"mean":0.002509,"std":0.000485},{"mean":0.002664,"std":0.000775},{"mean":0.003336,"std":0.000608},{"mean":0.003206,"std":0.000508},{"mean":0.004139,"std":0.00032},{"mean":0.005839,"std":0.001219},{"mean":0.007872,"std":0.000431},{"mean":0.013966,"std":0.000979},{"mean":0.039325,"std":0.000984},{"mean":0.188508,"std":0.001771}],"accuracy_8":[{"mean":0.823394,"std":0.000458},{"mean":0.826364,"std":0.002182},{"mean":0.829879,"std":0.002819},{"mean":0.834788,"std":0.003569},{"mean":0.839212,"std":0.003379},{"mean":0.843636,"std":0.003406},{"mean":0.847939,"std":0.002586},{"mean":0.852667,"std":0.003177},{"mean":0.856424,"std":0.003254},{"mean":0.86103,"std":0.003679},{"mean":0.865576,"std":0.00305},{"mean":0.869758,"std":0.002783},{"mean":0.87497,"std":0.00268},{"mean":0.880182,"std":0.003367},{"mean":0.885939,"std":0.002573},{"mean":0.891576,"std":0.001944},{"mean":0.897939,"std":0.001794},{"mean":0.901879,"std":0.00021},{"mean":0.899576,"std":0.000555},{"mean":0.811455,"std":0.001734}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001333,"std":0.000577},{"mean":0.002333,"std":0.001155},{"mean":0.003,"std":0.001732},{"mean":0.004,"std":0.001732},{"mean":0.005,"std":0.0},{"mean":0.006,"std":0.001},{"mean":0.008333,"std":0.001528},{"mean":0.008667,"std":0.001155},{"mean":0.012,"std":0.001},{"mean":0.018667,"std":0.004509},{"mean":0.027667,"std":0.001528},{"mean":0.052667,"std":0.003786},{"mean":0.164333,"std":0.002517},{"mean":0.988667,"std":0.010214}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432667,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433333,"std":0.001155},{"mean":0.434333,"std":0.001528},{"mean":0.434667,"std":0.001528},{"mean":0.435667,"std":0.002082},{"mean":0.437667,"std":0.001155},{"mean":0.441333,"std":0.000577},{"mean":0.452333,"std":0.001528},{"mean":0.497,"std":0.001},{"mean":0.991,"std":0.007937}],"llm_call_rate":[{"mean":0.956545,"std":0.00189},{"mean":0.920061,"std":0.003274},{"mean":0.878848,"std":0.00516},{"mean":0.829818,"std":0.007266},{"mean":0.785879,"std":0.013643},{"mean":0.743758,"std":0.010308},{"mean":0.705576,"std":0.00714},{"mean":0.665273,"std":0.010595},{"mean":0.624242,"std":0.006671},{"mean":0.579758,"std":0.00691},{"mean":0.540242,"std":0.00589},{"mean":0.498606,"std":0.00447},{"mean":0.454485,"std":0.006701},{"mean":0.413333,"std":0.007285},{"mean":0.370485,"std":0.002131},{"mean":0.325273,"std":0.003004},{"mean":0.284061,"std":0.002107},{"mean":0.236182,"std":0.001443},{"mean":0.173939,"std":0.000278},{"mean":0.002061,"std":0.001857}]}},"margin":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000303,"std":0.000525},{"mean":0.000485,"std":0.00042},{"mean":0.001212,"std":0.000606},{"mean":0.001732,"std":0.0003},{"mean":0.002424,"std":0.000262},{"mean":0.002694,"std":0.000841},{"mean":0.00303,"std":0.000915},{"mean":0.003636,"std":0.000875},{"mean":0.00404,"std":0.000175},{"mean":0.004848,"std":0.000982},{"mean":0.007532,"std":0.000687},{"mean":0.012687,"std":0.000505},{"mean":0.024697,"std":0.001252},{"mean":0.060107,"std":0.000214},{"mean":0.105253,"std":0.001069},{"mean":0.149091,"std":0.000383},{"mean":0.190182,"std":0.000727}],"operating":{"tau":[{"mean":8.143297,"std":0.134349},{"mean":7.814998,"std":0.102719},{"mean":7.591836,"std":0.081063},{"mean":7.379861,"std":0.070598},{"mean":7.192279,"std":0.055091},{"mean":7.02989,"std":0.060605},{"mean":6.874881,"std":0.072415},{"mean":6.717854,"std":0.070462},{"mean":6.550981,"std":0.076046},{"mean":6.389038,"std":0.090014},{"mean":6.222509,"std":0.097208},{"mean":6.041062,"std":0.0971},{"mean":5.818176,"std":0.116111},{"mean":5.571505,"std":0.110558},{"mean":5.281098,"std":0.117833},{"mean":4.894836,"std":0.096699},{"mean":4.340884,"std":0.105767},{"mean":3.465839,"std":0.122196},{"mean":1.872189,"std":0.056381},{"mean":0.006722,"std":0.005623}],"coverage":[{"mean":0.043697,"std":0.001728},{"mean":0.082606,"std":0.00348},{"mean":0.122667,"std":0.005486},{"mean":0.168061,"std":0.004937},{"mean":0.215455,"std":0.008043},{"mean":0.256485,"std":0.004914},{"mean":0.297515,"std":0.004395},{"mean":0.339515,"std":0.006728},{"mean":0.381636,"std":0.00387},{"mean":0.422424,"std":0.001866},{"mean":0.462242,"std":0.004246},{"mean":0.502182,"std":0.005724},{"mean":0.545394,"std":0.010343},{"mean":0.588,"std":0.009522},{"mean":0.628727,"std":0.002025},{"mean":0.669697,"std":0.001458},{"mean":0.713697,"std":0.002723},{"mean":0.764303,"std":0.002302},{"mean":0.829576,"std":0.001001},{"mean":0.999152,"std":0.001169}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000355,"std":0.000615},{"mean":0.000566,"std":0.000491},{"mean":0.000707,"std":0.000699},{"mean":0.00122,"std":0.000609},{"mean":0.001782,"std":0.000277},{"mean":0.002382,"std":2.4e-05},{"mean":0.002583,"std":0.000439},{"mean":0.00276,"std":0.00107},{"mean":0.003144,"std":0.000793},{"mean":0.003449,"std":0.000555},{"mean":0.003921,"std":0.000423},{"mean":0.004338,"std":0.000499},{"mean":0.00588,"std":0.001624},{"mean":0.008407,"std":0.000249},{"mean":0.01459,"std":0.000878},{"mean":0.042591,"std":0.001282},{"mean":0.189554,"std":0.001014}],"accuracy_8":[{"mean":0.823152,"std":0.00042},{"mean":0.826242,"std":0.001169},{"mean":0.829333,"std":0.002169},{"mean":0.833455,"std":0.002691},{"mean":0.837394,"std":0.003364},{"mean":0.841394,"std":0.004006},{"mean":0.846121,"std":0.003546},{"mean":0.851697,"std":0.003161},{"mean":0.856182,"std":0.003107},{"mean":0.859697,"std":0.003384},{"mean":0.864424,"std":0.003728},{"mean":0.869091,"std":0.003577},{"mean":0.874545,"std":0.003818},{"mean":0.879515,"std":0.003508},{"mean":0.885273,"std":0.003154},{"mean":0.890485,"std":0.001821},{"mean":0.896545,"std":0.002641},{"mean":0.901394,"std":0.000555},{"mean":0.899091,"std":0.000727},{"mean":0.810424,"std":0.000915}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001667,"std":0.001155},{"mean":0.002333,"std":0.000577},{"mean":0.003667,"std":0.001155},{"mean":0.004667,"std":0.000577},{"mean":0.005667,"std":0.002082},{"mean":0.007,"std":0.001732},{"mean":0.008667,"std":0.001528},{"mean":0.010667,"std":0.001155},{"mean":0.012667,"std":0.001528},{"mean":0.019333,"std":0.005508},{"mean":0.029667,"std":0.000577},{"mean":0.055333,"std":0.003512},{"mean":0.182667,"std":0.005508},{"mean":0.996333,"std":0.004726}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001155},{"mean":0.433,"std":0.001732},{"mean":0.434,"std":0.002},{"mean":0.435,"std":0.002},{"mean":0.435333,"std":0.002517},{"mean":0.436333,"std":0.002517},{"mean":0.438333,"std":0.000577},{"mean":0.442,"std":0.002},{"mean":0.451667,"std":0.002517},{"mean":0.501333,"std":0.003786},{"mean":0.997,"std":0.003606}],"llm_call_rate":[{"mean":0.956303,"std":0.001728},{"mean":0.917394,"std":0.00348},{"mean":0.877333,"std":0.005486},{"mean":0.831939,"std":0.004937},{"mean":0.784545,"std":0.008043},{"mean":0.743515,"std":0.004914},{"mean":0.702485,"std":0.004395},{"mean":0.660485,"std":0.006728},{"mean":0.618364,"std":0.00387},{"mean":0.577576,"std":0.001866},{"mean":0.537758,"std":0.004246},{"mean":0.497818,"std":0.005724},{"mean":0.454606,"std":0.010343},{"mean":0.412,"std":0.009522},{"mean":0.371273,"std":0.002025},{"mean":0.330303,"std":0.001458},{"mean":0.286303,"std":0.002723},{"mean":0.235697,"std":0.002302},{"mean":0.170424,"std":0.001001},{"mean":0.000848,"std":0.001169}]}},"msp_t":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.000525},{"mean":0.000485,"std":0.00042},{"mean":0.000808,"std":0.00035},{"mean":0.001732,"std":0.0003},{"mean":0.002121,"std":0.000694},{"mean":0.002559,"std":0.000467},{"mean":0.002545,"std":0.000727},{"mean":0.003085,"std":0.000505},{"mean":0.003434,"std":0.000463},{"mean":0.004382,"std":0.000427},{"mean":0.006753,"std":0.001039},{"mean":0.011475,"std":0.00061},{"mean":0.0225,"std":0.001041},{"mean":0.056756,"std":0.000891},{"mean":0.102694,"std":0.001299},{"mean":0.147943,"std":0.00069},{"mean":0.190182,"std":0.000727}],"operating":{"tau":[{"mean":0.999062,"std":0.000142},{"mean":0.998736,"std":0.000145},{"mean":0.998396,"std":0.000146},{"mean":0.998034,"std":0.000138},{"mean":0.997681,"std":0.00015},{"mean":0.997313,"std":0.00019},{"mean":0.996921,"std":0.000234},{"mean":0.996405,"std":0.000291},{"mean":0.995833,"std":0.000346},{"mean":0.995093,"std":0.000511},{"mean":0.994245,"std":0.000612},{"mean":0.993104,"std":0.000819},{"mean":0.99156,"std":0.00091},{"mean":0.989248,"std":0.001496},{"mean":0.98557,"std":0.001785},{"mean":0.979168,"std":0.001911},{"mean":0.966233,"std":0.003391},{"mean":0.922491,"std":0.008109},{"mean":0.722724,"std":0.00956},{"mean":0.18539,"std":0.019404}],"coverage":[{"mean":0.044121,"std":0.003494},{"mean":0.080606,"std":0.00165},{"mean":0.12303,"std":0.007168},{"mean":0.172061,"std":0.011245},{"mean":0.213152,"std":0.012651},{"mean":0.256182,"std":0.012303},{"mean":0.293152,"std":0.008548},{"mean":0.338182,"std":0.008456},{"mean":0.377758,"std":0.006647},{"mean":0.421515,"std":0.004187},{"mean":0.46103,"std":0.006278},{"mean":0.503455,"std":0.008429},{"mean":0.545758,"std":0.005046},{"mean":0.588485,"std":0.009204},{"mean":0.630242,"std":0.003918},{"mean":0.67303,"std":0.00447},{"mean":0.71503,"std":0.00164},{"mean":0.763879,"std":0.002027},{"mean":0.826788,"std":0.00129},{"mean":0.999333,"std":0.000735}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000338,"std":0.000586},{"mean":0.00055,"std":0.000476},{"mean":0.00046,"std":0.000399},{"mean":0.000834,"std":0.000388},{"mean":0.00179,"std":0.000281},{"mean":0.002078,"std":0.0007},{"mean":0.002301,"std":0.000251},{"mean":0.002502,"std":0.000485},{"mean":0.002536,"std":0.000761},{"mean":0.003113,"std":0.000539},{"mean":0.003299,"std":0.000397},{"mean":0.00423,"std":0.000311},{"mean":0.005315,"std":0.000705},{"mean":0.007882,"std":0.000906},{"mean":0.013407,"std":0.001038},{"mean":0.038337,"std":0.000405},{"mean":0.189641,"std":0.00075}],"accuracy_8":[{"mean":0.823515,"std":0.000378},{"mean":0.826364,"std":0.002025},{"mean":0.830303,"std":0.003527},{"mean":0.835333,"std":0.003754},{"mean":0.839576,"std":0.003364},{"mean":0.844364,"std":0.002715},{"mean":0.848667,"std":0.002288},{"mean":0.853636,"std":0.003154},{"mean":0.857515,"std":0.003098},{"mean":0.862061,"std":0.003858},{"mean":0.865939,"std":0.003379},{"mean":0.871152,"std":0.002777},{"mean":0.876182,"std":0.002385},{"mean":0.881394,"std":0.003518},{"mean":0.88697,"std":0.002338},{"mean":0.891455,"std":0.00142},{"mean":0.897697,"std":0.001857},{"mean":0.902,"std":0.000182},{"mean":0.900485,"std":0.000639},{"mean":0.810364,"std":0.000833}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.002333,"std":0.000577},{"mean":0.003333,"std":0.001155},{"mean":0.004333,"std":0.000577},{"mean":0.005,"std":0.0},{"mean":0.005667,"std":0.001155},{"mean":0.007667,"std":0.001155},{"mean":0.009,"std":0.001},{"mean":0.012,"std":0.001},{"mean":0.016333,"std":0.002082},{"mean":0.027333,"std":0.002887},{"mean":0.05,"std":0.004359},{"mean":0.159333,"std":0.004509},{"mean":0.996333,"std":0.004041}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434667,"std":0.001528},{"mean":0.435667,"std":0.002082},{"mean":0.437667,"std":0.001155},{"mean":0.441,"std":0.0},{"mean":0.45,"std":0.001},{"mean":0.491667,"std":0.003055},{"mean":0.997,"std":0.003606}],"llm_call_rate":[{"mean":0.955879,"std":0.003494},{"mean":0.919394,"std":0.00165},{"mean":0.87697,"std":0.007168},{"mean":0.827939,"std":0.011245},{"mean":0.786848,"std":0.012651},{"mean":0.743818,"std":0.012303},{"mean":0.706848,"std":0.008548},{"mean":0.661818,"std":0.008456},{"mean":0.622242,"std":0.006647},{"mean":0.578485,"std":0.004187},{"mean":0.53897,"std":0.006278},{"mean":0.496545,"std":0.008429},{"mean":0.454242,"std":0.005046},{"mean":0.411515,"std":0.009204},{"mean":0.369758,"std":0.003918},{"mean":0.32697,"std":0.00447},{"mean":0.28497,"std":0.00164},{"mean":0.236121,"std":0.002027},{"mean":0.173212,"std":0.00129},{"mean":0.000667,"std":0.000735}]}}}}},"modernbert/k1":{"model":"modernbert","k":1,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["ModernBERT-base-k1-seed42","ModernBERT-base-k1-seed43","ModernBERT-base-k1-seed44"],"logits_sha256":["aaa5c7cc5e068eed5da1dc4a55a91587761246e273c254564fd6bf69a27c2ae2","c2eae04634a5224685fed57cf0600993eb975a269a9ef61ec97f33598705786b","84067b0f3e6ad7ab14ed6b9b919b1b4b24ed1d0c3399dd81573f325adf2021b4"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.169697,"std":0.026306},{"mean":0.257576,"std":0.038153},{"mean":0.328485,"std":0.014696},{"mean":0.374545,"std":0.026175},{"mean":0.409212,"std":0.02069},{"mean":0.435152,"std":0.014252},{"mean":0.461126,"std":0.017434},{"mean":0.478636,"std":0.024104},{"mean":0.495623,"std":0.022645},{"mean":0.511273,"std":0.01974},{"mean":0.523196,"std":0.018395},{"mean":0.535253,"std":0.018236},{"mean":0.548998,"std":0.018591},{"mean":0.560087,"std":0.017395},{"mean":0.57099,"std":0.01835},{"mean":0.579697,"std":0.018274},{"mean":0.58852,"std":0.016406},{"mean":0.59899,"std":0.016569},{"mean":0.607592,"std":0.015174},{"mean":0.616242,"std":0.013153}],"operating":{"tau":[{"mean":0.78177,"std":0.020505},{"mean":0.59582,"std":0.019658},{"mean":0.472352,"std":0.021869},{"mean":0.388968,"std":0.018468},{"mean":0.334719,"std":0.017879},{"mean":0.294797,"std":0.015018},{"mean":0.264354,"std":0.009653},{"mean":0.239482,"std":0.009417},{"mean":0.219031,"std":0.009135},{"mean":0.200212,"std":0.007622},{"mean":0.184565,"std":0.006502},{"mean":0.169367,"std":0.005769},{"mean":0.155326,"std":0.004684},{"mean":0.143895,"std":0.004606},{"mean":0.132104,"std":0.003723},{"mean":0.121245,"std":0.003232},{"mean":0.109229,"std":0.002514},{"mean":0.098136,"std":0.002712},{"mean":0.083878,"std":0.001685},{"mean":0.046433,"std":0.003008}],"coverage":[{"mean":0.040788,"std":0.0029},{"mean":0.084848,"std":0.007532},{"mean":0.132182,"std":0.008574},{"mean":0.185636,"std":0.007455},{"mean":0.236606,"std":0.011151},{"mean":0.287697,"std":0.013023},{"mean":0.334121,"std":0.007637},{"mean":0.379939,"std":0.012061},{"mean":0.42897,"std":0.010996},{"mean":0.478303,"std":0.007497},{"mean":0.524061,"std":0.006283},{"mean":0.578061,"std":0.005932},{"mean":0.633576,"std":0.009721},{"mean":0.688545,"std":0.010383},{"mean":0.740606,"std":0.012006},{"mean":0.786788,"std":0.013247},{"mean":0.843697,"std":0.006185},{"mean":0.892061,"std":0.000584},{"mean":0.948424,"std":0.004246},{"mean":0.999636,"std":0.00063}],"selective_risk":[{"mean":0.146367,"std":0.028035},{"mean":0.237877,"std":0.044954},{"mean":0.304715,"std":0.036242},{"mean":0.367361,"std":0.022663},{"mean":0.40118,"std":0.031305},{"mean":0.426937,"std":0.023897},{"mean":0.452632,"std":0.020379},{"mean":0.47217,"std":0.025754},{"mean":0.48668,"std":0.025032},{"mean":0.504942,"std":0.021652},{"mean":0.517237,"std":0.020192},{"mean":0.529811,"std":0.01808},{"mean":0.544404,"std":0.020231},{"mean":0.557342,"std":0.019482},{"mean":0.569483,"std":0.018909},{"mean":0.577098,"std":0.020026},{"mean":0.587457,"std":0.017036},{"mean":0.596981,"std":0.016698},{"mean":0.607341,"std":0.015776},{"mean":0.616106,"std":0.013022}],"accuracy_8":[{"mean":0.81903,"std":0.00105},{"mean":0.811515,"std":0.00469},{"mean":0.798788,"std":0.007598},{"mean":0.780788,"std":0.008199},{"mean":0.763152,"std":0.012003},{"mean":0.744485,"std":0.01392},{"mean":0.724667,"std":0.013258},{"mean":0.705212,"std":0.018489},{"mean":0.684061,"std":0.01788},{"mean":0.660848,"std":0.014918},{"mean":0.639455,"std":0.015458},{"mean":0.614121,"std":0.015676},{"mean":0.585939,"std":0.017581},{"mean":0.557152,"std":0.017147},{"mean":0.528788,"std":0.018383},{"mean":0.504848,"std":0.019919},{"mean":0.474485,"std":0.013682},{"mean":0.447212,"std":0.014308},{"mean":0.413273,"std":0.014936},{"mean":0.384061,"std":0.012867}],"high_conf_oos_misroute_rate":[{"mean":0.009333,"std":0.002082},{"mean":0.027667,"std":0.008386},{"mean":0.053667,"std":0.01115},{"mean":0.093333,"std":0.018037},{"mean":0.134,"std":0.019672},{"mean":0.174,"std":0.019},{"mean":0.207333,"std":0.01701},{"mean":0.251333,"std":0.022502},{"mean":0.293667,"std":0.026577},{"mean":0.342667,"std":0.030105},{"mean":0.386333,"std":0.02967},{"mean":0.439667,"std":0.024173},{"mean":0.503,"std":0.030806},{"mean":0.56,"std":0.041243},{"mean":0.614333,"std":0.037072},{"mean":0.665,"std":0.040731},{"mean":0.728667,"std":0.035838},{"mean":0.786667,"std":0.034078},{"mean":0.852333,"std":0.037687},{"mean":0.919333,"std":0.028148}],"oos_misroute_rate":[{"mean":0.434,"std":0.001},{"mean":0.44,"std":0.005292},{"mean":0.455,"std":0.007937},{"mean":0.472,"std":0.00866},{"mean":0.492667,"std":0.012097},{"mean":0.509667,"std":0.012858},{"mean":0.524667,"std":0.016503},{"mean":0.547,"std":0.024515},{"mean":0.573667,"std":0.025423},{"mean":0.600333,"std":0.025146},{"mean":0.621333,"std":0.02635},{"mean":0.651,"std":0.026153},{"mean":0.685667,"std":0.027429},{"mean":0.716,"std":0.032741},{"mean":0.746667,"std":0.028148},{"mean":0.774667,"std":0.029501},{"mean":0.81,"std":0.025159},{"mean":0.839667,"std":0.025146},{"mean":0.882333,"std":0.028042},{"mean":0.919333,"std":0.028148}],"llm_call_rate":[{"mean":0.959212,"std":0.0029},{"mean":0.915152,"std":0.007532},{"mean":0.867818,"std":0.008574},{"mean":0.814364,"std":0.007455},{"mean":0.763394,"std":0.011151},{"mean":0.712303,"std":0.013023},{"mean":0.665879,"std":0.007637},{"mean":0.620061,"std":0.012061},{"mean":0.57103,"std":0.010996},{"mean":0.521697,"std":0.007497},{"mean":0.475939,"std":0.006283},{"mean":0.421939,"std":0.005932},{"mean":0.366424,"std":0.009721},{"mean":0.311455,"std":0.010383},{"mean":0.259394,"std":0.012006},{"mean":0.213212,"std":0.013247},{"mean":0.156303,"std":0.006185},{"mean":0.107939,"std":0.000584},{"mean":0.051576,"std":0.004246},{"mean":0.000364,"std":0.00063}]}},"entropy":{"risk_coverage":[{"mean":0.163636,"std":0.022119},{"mean":0.253939,"std":0.030279},{"mean":0.319596,"std":0.021829},{"mean":0.373333,"std":0.019441},{"mean":0.408727,"std":0.022709},{"mean":0.431717,"std":0.023413},{"mean":0.452987,"std":0.02333},{"mean":0.472121,"std":0.018229},{"mean":0.489832,"std":0.019231},{"mean":0.504606,"std":0.020212},{"mean":0.521653,"std":0.02026},{"mean":0.534545,"std":0.01829},{"mean":0.54676,"std":0.017121},{"mean":0.558268,"std":0.018082},{"mean":0.56897,"std":0.01644},{"mean":0.579015,"std":0.01615},{"mean":0.589661,"std":0.016077},{"mean":0.598114,"std":0.016806},{"mean":0.607656,"std":0.015098},{"mean":0.616242,"std":0.013153}],"operating":{"tau":[{"mean":-1.277668,"std":0.060023},{"mean":-2.070776,"std":0.082533},{"mean":-2.508326,"std":0.09054},{"mean":-2.802261,"std":0.072873},{"mean":-2.996113,"std":0.057721},{"mean":-3.152038,"std":0.056013},{"mean":-3.26233,"std":0.047188},{"mean":-3.35479,"std":0.04569},{"mean":-3.431281,"std":0.041067},{"mean":-3.500737,"std":0.037769},{"mean":-3.564762,"std":0.034855},{"mean":-3.621958,"std":0.030142},{"mean":-3.676969,"std":0.024801},{"mean":-3.732727,"std":0.020613},{"mean":-3.786784,"std":0.019194},{"mean":-3.836819,"std":0.014321},{"mean":-3.889431,"std":0.012331},{"mean":-3.956274,"std":0.011779},{"mean":-4.03976,"std":0.009349},{"mean":-4.39049,"std":0.012825}],"coverage":[{"mean":0.04097,"std":0.00347},{"mean":0.085273,"std":0.008123},{"mean":0.131091,"std":0.010063},{"mean":0.179636,"std":0.013009},{"mean":0.231879,"std":0.012417},{"mean":0.28497,"std":0.011288},{"mean":0.332788,"std":0.011164},{"mean":0.378364,"std":0.008214},{"mean":0.425455,"std":0.005203},{"mean":0.473455,"std":0.005852},{"mean":0.527333,"std":0.004369},{"mean":0.577939,"std":0.007237},{"mean":0.62697,"std":0.007488},{"mean":0.683152,"std":0.008321},{"mean":0.73497,"std":0.008652},{"mean":0.783152,"std":0.010691},{"mean":0.830485,"std":0.007488},{"mean":0.889091,"std":0.005101},{"mean":0.943515,"std":0.006718},{"mean":0.999939,"std":0.000105}],"selective_risk":[{"mean":0.144488,"std":0.036415},{"mean":0.245202,"std":0.036581},{"mean":0.300208,"std":0.029009},{"mean":0.349461,"std":0.030387},{"mean":0.400614,"std":0.028503},{"mean":0.425255,"std":0.029307},{"mean":0.444609,"std":0.025961},{"mean":0.464285,"std":0.02238},{"mean":0.481102,"std":0.022395},{"mean":0.497256,"std":0.019904},{"mean":0.513938,"std":0.021511},{"mean":0.52771,"std":0.02056},{"mean":0.540972,"std":0.018329},{"mean":0.554638,"std":0.017342},{"mean":0.56587,"std":0.016889},{"mean":0.576106,"std":0.015639},{"mean":0.585227,"std":0.017356},{"mean":0.595908,"std":0.016446},{"mean":0.606332,"std":0.016511},{"mean":0.61622,"std":0.013131}],"accuracy_8":[{"mean":0.81897,"std":0.00163},{"mean":0.810485,"std":0.003437},{"mean":0.79903,"std":0.006986},{"mean":0.783818,"std":0.010662},{"mean":0.763697,"std":0.011747},{"mean":0.744909,"std":0.013651},{"mean":0.726242,"std":0.013669},{"mean":0.706909,"std":0.013331},{"mean":0.686727,"std":0.013151},{"mean":0.664667,"std":0.01292},{"mean":0.638485,"std":0.014075},{"mean":0.613697,"std":0.015782},{"mean":0.589152,"std":0.014294},{"mean":0.559273,"std":0.013372},{"mean":0.532364,"std":0.012354},{"mean":0.50697,"std":0.011467},{"mean":0.480727,"std":0.0138},{"mean":0.447636,"std":0.014024},{"mean":0.417152,"std":0.016565},{"mean":0.383818,"std":0.013095}],"high_conf_oos_misroute_rate":[{"mean":0.009,"std":0.002646},{"mean":0.027333,"std":0.006658},{"mean":0.05,"std":0.012124},{"mean":0.082667,"std":0.016921},{"mean":0.123333,"std":0.021385},{"mean":0.164333,"std":0.023245},{"mean":0.201667,"std":0.019088},{"mean":0.245,"std":0.02498},{"mean":0.284,"std":0.02358},{"mean":0.328667,"std":0.021221},{"mean":0.378667,"std":0.03421},{"mean":0.425667,"std":0.030989},{"mean":0.477,"std":0.026514},{"mean":0.531667,"std":0.029704},{"mean":0.589667,"std":0.031262},{"mean":0.649,"std":0.027514},{"mean":0.702,"std":0.026},{"mean":0.772667,"std":0.025541},{"mean":0.841667,"std":0.028148},{"mean":0.919333,"std":0.028148}],"oos_misroute_rate":[{"mean":0.433667,"std":0.002082},{"mean":0.439,"std":0.001},{"mean":0.451667,"std":0.009452},{"mean":0.465333,"std":0.013279},{"mean":0.483667,"std":0.01365},{"mean":0.505667,"std":0.010214},{"mean":0.523333,"std":0.010017},{"mean":0.545667,"std":0.01365},{"mean":0.562333,"std":0.013204},{"mean":0.586,"std":0.017349},{"mean":0.616,"std":0.030315},{"mean":0.643,"std":0.028},{"mean":0.67,"std":0.020952},{"mean":0.701667,"std":0.020404},{"mean":0.732,"std":0.021656},{"mean":0.762667,"std":0.021385},{"mean":0.793,"std":0.023643},{"mean":0.833667,"std":0.023116},{"mean":0.872,"std":0.029052},{"mean":0.919333,"std":0.028148}],"llm_call_rate":[{"mean":0.95903,"std":0.00347},{"mean":0.914727,"std":0.008123},{"mean":0.868909,"std":0.010063},{"mean":0.820364,"std":0.013009},{"mean":0.768121,"std":0.012417},{"mean":0.71503,"std":0.011288},{"mean":0.667212,"std":0.011164},{"mean":0.621636,"std":0.008214},{"mean":0.574545,"std":0.005203},{"mean":0.526545,"std":0.005852},{"mean":0.472667,"std":0.004369},{"mean":0.422061,"std":0.007237},{"mean":0.37303,"std":0.007488},{"mean":0.316848,"std":0.008321},{"mean":0.26503,"std":0.008652},{"mean":0.216848,"std":0.010691},{"mean":0.169515,"std":0.007488},{"mean":0.110909,"std":0.005101},{"mean":0.056485,"std":0.006718},{"mean":6.1e-05,"std":0.000105}]}},"margin":{"risk_coverage":[{"mean":0.192727,"std":0.015851},{"mean":0.286667,"std":0.033788},{"mean":0.349091,"std":0.020391},{"mean":0.398788,"std":0.022348},{"mean":0.436848,"std":0.027274},{"mean":0.463434,"std":0.023041},{"mean":0.49039,"std":0.017377},{"mean":0.505909,"std":0.017273},{"mean":0.520808,"std":0.018051},{"mean":0.535758,"std":0.016744},{"mean":0.545234,"std":0.018713},{"mean":0.557778,"std":0.016471},{"mean":0.567925,"std":0.018555},{"mean":0.57645,"std":0.019006},{"mean":0.582788,"std":0.015261},{"mean":0.58947,"std":0.014869},{"mean":0.597718,"std":0.014318},{"mean":0.60404,"std":0.014648},{"mean":0.610207,"std":0.012841},{"mean":0.616242,"std":0.013153}],"operating":{"tau":[{"mean":2.250133,"std":0.164764},{"mean":1.591437,"std":0.133551},{"mean":1.264702,"std":0.112701},{"mean":1.044612,"std":0.09219},{"mean":0.882479,"std":0.069037},{"mean":0.759264,"std":0.061148},{"mean":0.657975,"std":0.059979},{"mean":0.575541,"std":0.053101},{"mean":0.498087,"std":0.040056},{"mean":0.431197,"std":0.034034},{"mean":0.369053,"std":0.027287},{"mean":0.315412,"std":0.0199},{"mean":0.267895,"std":0.017443},{"mean":0.21909,"std":0.012185},{"mean":0.177856,"std":0.013864},{"mean":0.137354,"std":0.008886},{"mean":0.10247,"std":0.007412},{"mean":0.067086,"std":0.005864},{"mean":0.033342,"std":0.002102},{"mean":0.000255,"std":0.000219}],"coverage":[{"mean":0.041091,"std":0.003511},{"mean":0.088485,"std":0.002373},{"mean":0.136485,"std":0.003093},{"mean":0.18497,"std":0.005498},{"mean":0.239091,"std":0.003963},{"mean":0.289212,"std":0.007017},{"mean":0.339636,"std":0.010264},{"mean":0.386545,"std":0.01244},{"mean":0.440061,"std":0.008148},{"mean":0.491152,"std":0.006304},{"mean":0.543394,"std":0.006864},{"mean":0.593091,"std":0.008998},{"mean":0.641333,"std":0.005407},{"mean":0.696303,"std":0.008174},{"mean":0.747818,"std":0.001273},{"mean":0.798061,"std":0.002442},{"mean":0.849515,"std":0.000555},{"mean":0.900667,"std":0.004484},{"mean":0.95,"std":0.002321},{"mean":0.999697,"std":0.000378}],"selective_risk":[{"mean":0.171551,"std":0.008348},{"mean":0.265272,"std":0.031014},{"mean":0.336215,"std":0.026328},{"mean":0.385812,"std":0.025791},{"mean":0.429351,"std":0.029863},{"mean":0.45575,"std":0.028899},{"mean":0.485087,"std":0.022823},{"mean":0.502845,"std":0.020863},{"mean":0.51847,"std":0.019614},{"mean":0.534717,"std":0.017522},{"mean":0.543866,"std":0.018315},{"mean":0.555224,"std":0.014675},{"mean":0.566867,"std":0.017983},{"mean":0.576073,"std":0.017164},{"mean":0.5828,"std":0.015269},{"mean":0.589178,"std":0.01458},{"mean":0.59756,"std":0.014195},{"mean":0.604184,"std":0.014793},{"mean":0.609957,"std":0.012576},{"mean":0.616127,"std":0.013126}],"accuracy_8":[{"mean":0.818182,"std":0.000364},{"mean":0.809333,"std":0.003555},{"mean":0.794909,"std":0.005101},{"mean":0.778242,"std":0.007959},{"mean":0.756485,"std":0.01148},{"mean":0.737515,"std":0.014794},{"mean":0.714424,"std":0.015343},{"mean":0.694182,"std":0.016775},{"mean":0.669758,"std":0.016042},{"mean":0.645939,"std":0.012812},{"mean":0.621758,"std":0.013247},{"mean":0.596788,"std":0.009926},{"mean":0.571697,"std":0.012494},{"mean":0.543152,"std":0.010205},{"mean":0.517818,"std":0.012448},{"mean":0.492848,"std":0.010791},{"mean":0.46497,"std":0.012232},{"mean":0.437576,"std":0.012911},{"mean":0.410667,"std":0.010448},{"mean":0.384,"std":0.013122}],"high_conf_oos_misroute_rate":[{"mean":0.011333,"std":0.003215},{"mean":0.030667,"std":0.006658},{"mean":0.064667,"std":0.007767},{"mean":0.102667,"std":0.012741},{"mean":0.148,"std":0.013077},{"mean":0.194333,"std":0.022143},{"mean":0.244,"std":0.024759},{"mean":0.282333,"std":0.024007},{"mean":0.333667,"std":0.012662},{"mean":0.386,"std":0.016371},{"mean":0.437667,"std":0.019858},{"mean":0.490333,"std":0.012702},{"mean":0.545667,"std":0.022121},{"mean":0.605,"std":0.026287},{"mean":0.656,"std":0.031193},{"mean":0.709333,"std":0.030089},{"mean":0.762667,"std":0.03691},{"mean":0.812,"std":0.035043},{"mean":0.868,"std":0.028583},{"mean":0.919,"std":0.028213}],"oos_misroute_rate":[{"mean":0.435667,"std":0.000577},{"mean":0.443667,"std":0.002082},{"mean":0.463,"std":0.004583},{"mean":0.481,"std":0.010536},{"mean":0.503333,"std":0.014742},{"mean":0.524667,"std":0.023116},{"mean":0.551667,"std":0.02318},{"mean":0.570333,"std":0.021385},{"mean":0.596,"std":0.015},{"mean":0.620333,"std":0.009018},{"mean":0.649,"std":0.016},{"mean":0.683333,"std":0.002082},{"mean":0.712333,"std":0.012503},{"mean":0.746333,"std":0.010066},{"mean":0.775,"std":0.015395},{"mean":0.804333,"std":0.015948},{"mean":0.834333,"std":0.023029},{"mean":0.861667,"std":0.022502},{"mean":0.893333,"std":0.02318},{"mean":0.919333,"std":0.028148}],"llm_call_rate":[{"mean":0.958909,"std":0.003511},{"mean":0.911515,"std":0.002373},{"mean":0.863515,"std":0.003093},{"mean":0.81503,"std":0.005498},{"mean":0.760909,"std":0.003963},{"mean":0.710788,"std":0.007017},{"mean":0.660364,"std":0.010264},{"mean":0.613455,"std":0.01244},{"mean":0.559939,"std":0.008148},{"mean":0.508848,"std":0.006304},{"mean":0.456606,"std":0.006864},{"mean":0.406909,"std":0.008998},{"mean":0.358667,"std":0.005407},{"mean":0.303697,"std":0.008174},{"mean":0.252182,"std":0.001273},{"mean":0.201939,"std":0.002442},{"mean":0.150485,"std":0.000555},{"mean":0.099333,"std":0.004484},{"mean":0.05,"std":0.002321},{"mean":0.000303,"std":0.000378}]}},"msp_t":{"risk_coverage":[{"mean":0.167273,"std":0.025455},{"mean":0.253939,"std":0.029894},{"mean":0.320404,"std":0.018356},{"mean":0.373636,"std":0.02441},{"mean":0.406545,"std":0.020089},{"mean":0.433131,"std":0.01759},{"mean":0.45645,"std":0.019749},{"mean":0.476061,"std":0.025137},{"mean":0.491044,"std":0.022985},{"mean":0.510303,"std":0.018407},{"mean":0.521873,"std":0.017932},{"mean":0.534747,"std":0.018313},{"mean":0.548159,"std":0.017128},{"mean":0.558874,"std":0.016796},{"mean":0.569293,"std":0.017296},{"mean":0.579621,"std":0.018028},{"mean":0.587665,"std":0.016762},{"mean":0.598316,"std":0.016825},{"mean":0.607656,"std":0.015421},{"mean":0.616242,"std":0.013153}],"operating":{"tau":[{"mean":0.436825,"std":0.064187},{"mean":0.29399,"std":0.050037},{"mean":0.226481,"std":0.038723},{"mean":0.186208,"std":0.028129},{"mean":0.160081,"std":0.02472},{"mean":0.142301,"std":0.021203},{"mean":0.128603,"std":0.016742},{"mean":0.118433,"std":0.01527},{"mean":0.109591,"std":0.01325},{"mean":0.101557,"std":0.01217},{"mean":0.094625,"std":0.010797},{"mean":0.088311,"std":0.009898},{"mean":0.082519,"std":0.009445},{"mean":0.077236,"std":0.008422},{"mean":0.07203,"std":0.007498},{"mean":0.066872,"std":0.006837},{"mean":0.061931,"std":0.00594},{"mean":0.056451,"std":0.004806},{"mean":0.049794,"std":0.004318},{"mean":0.030694,"std":0.001643}],"coverage":[{"mean":0.040424,"std":0.002599},{"mean":0.083939,"std":0.006102},{"mean":0.131273,"std":0.010122},{"mean":0.182061,"std":0.006599},{"mean":0.234848,"std":0.012091},{"mean":0.286606,"std":0.013393},{"mean":0.334667,"std":0.004365},{"mean":0.380606,"std":0.013511},{"mean":0.428061,"std":0.01064},{"mean":0.476667,"std":0.00447},{"mean":0.523091,"std":0.007086},{"mean":0.576667,"std":0.003532},{"mean":0.629697,"std":0.00812},{"mean":0.685818,"std":0.011073},{"mean":0.738727,"std":0.011977},{"mean":0.789455,"std":0.012993},{"mean":0.838909,"std":0.010619},{"mean":0.892303,"std":0.003259},{"mean":0.948242,"std":0.004131},{"mean":0.999818,"std":0.000315}],"selective_risk":[{"mean":0.142655,"std":0.03329},{"mean":0.235775,"std":0.037122},{"mean":0.294986,"std":0.036802},{"mean":0.360394,"std":0.024032},{"mean":0.39789,"std":0.033331},{"mean":0.425528,"std":0.024974},{"mean":0.450758,"std":0.019114},{"mean":0.468458,"std":0.026641},{"mean":0.484005,"std":0.025775},{"mean":0.501484,"std":0.021607},{"mean":0.514929,"std":0.018348},{"mean":0.529419,"std":0.018795},{"mean":0.542797,"std":0.017717},{"mean":0.555783,"std":0.018121},{"mean":0.566911,"std":0.019708},{"mean":0.577171,"std":0.019167},{"mean":0.585341,"std":0.017636},{"mean":0.596569,"std":0.016444},{"mean":0.607138,"std":0.016446},{"mean":0.616174,"std":0.013087}],"accuracy_8":[{"mean":0.819091,"std":0.00142},{"mean":0.811697,"std":0.003754},{"mean":0.800061,"std":0.007659},{"mean":0.782303,"std":0.00812},{"mean":0.763576,"std":0.012409},{"mean":0.744667,"std":0.014188},{"mean":0.725152,"std":0.011741},{"mean":0.706424,"std":0.01835},{"mean":0.685091,"std":0.016515},{"mean":0.662242,"std":0.014546},{"mean":0.641212,"std":0.014091},{"mean":0.614364,"std":0.015156},{"mean":0.588182,"std":0.016294},{"mean":0.558909,"std":0.017574},{"mean":0.531212,"std":0.017952},{"mean":0.503394,"std":0.018946},{"mean":0.478121,"std":0.015711},{"mean":0.447333,"std":0.013333},{"mean":0.413636,"std":0.015781},{"mean":0.383879,"std":0.013037}],"high_conf_oos_misroute_rate":[{"mean":0.008667,"std":0.002082},{"mean":0.026,"std":0.007},{"mean":0.051,"std":0.014177},{"mean":0.087667,"std":0.014572},{"mean":0.130667,"std":0.020257},{"mean":0.169,"std":0.015875},{"mean":0.207333,"std":0.016166},{"mean":0.246667,"std":0.023587},{"mean":0.291667,"std":0.024846},{"mean":0.335667,"std":0.034933},{"mean":0.383333,"std":0.027429},{"mean":0.435,"std":0.027},{"mean":0.493333,"std":0.030238},{"mean":0.552333,"std":0.032884},{"mean":0.609,"std":0.038039},{"mean":0.663667,"std":0.039004},{"mean":0.718333,"std":0.034501},{"mean":0.784667,"std":0.028501},{"mean":0.849,"std":0.039962},{"mean":0.919333,"std":0.028148}],"oos_misroute_rate":[{"mean":0.433333,"std":0.001528},{"mean":0.438667,"std":0.002309},{"mean":0.452,"std":0.009539},{"mean":0.469667,"std":0.009292},{"mean":0.490667,"std":0.012097},{"mean":0.507333,"std":0.01124},{"mean":0.522667,"std":0.012858},{"mean":0.543333,"std":0.021197},{"mean":0.570667,"std":0.022189},{"mean":0.597,"std":0.027074},{"mean":0.619,"std":0.028478},{"mean":0.65,"std":0.023388},{"mean":0.679,"std":0.026153},{"mean":0.712333,"std":0.031005},{"mean":0.742667,"std":0.027502},{"mean":0.773,"std":0.027622},{"mean":0.804,"std":0.02571},{"mean":0.839,"std":0.020664},{"mean":0.879333,"std":0.03107},{"mean":0.919333,"std":0.028148}],"llm_call_rate":[{"mean":0.959576,"std":0.002599},{"mean":0.916061,"std":0.006102},{"mean":0.868727,"std":0.010122},{"mean":0.817939,"std":0.006599},{"mean":0.765152,"std":0.012091},{"mean":0.713394,"std":0.013393},{"mean":0.665333,"std":0.004365},{"mean":0.619394,"std":0.013511},{"mean":0.571939,"std":0.01064},{"mean":0.523333,"std":0.00447},{"mean":0.476909,"std":0.007086},{"mean":0.423333,"std":0.003532},{"mean":0.370303,"std":0.00812},{"mean":0.314182,"std":0.011073},{"mean":0.261273,"std":0.011977},{"mean":0.210545,"std":0.012993},{"mean":0.161091,"std":0.010619},{"mean":0.107697,"std":0.003259},{"mean":0.051758,"std":0.004131},{"mean":0.000182,"std":0.000315}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.130909,"std":0.015851},{"mean":0.170909,"std":0.018453},{"mean":0.208889,"std":0.029125},{"mean":0.244545,"std":0.027318},{"mean":0.275152,"std":0.028966},{"mean":0.305253,"std":0.032277},{"mean":0.330909,"std":0.035451},{"mean":0.356515,"std":0.037307},{"mean":0.380202,"std":0.037086},{"mean":0.406182,"std":0.033211},{"mean":0.428981,"std":0.035496},{"mean":0.449697,"std":0.035819},{"mean":0.467506,"std":0.035552},{"mean":0.487965,"std":0.034656},{"mean":0.504808,"std":0.03318},{"mean":0.521667,"std":0.031877},{"mean":0.537184,"std":0.030744},{"mean":0.55138,"std":0.027389},{"mean":0.566954,"std":0.026817},{"mean":0.582909,"std":0.024735}],"operating":{"tau":[{"mean":0.86213,"std":0.015753},{"mean":0.737387,"std":0.023936},{"mean":0.651239,"std":0.025276},{"mean":0.588698,"std":0.020948},{"mean":0.538398,"std":0.014839},{"mean":0.497017,"std":0.015835},{"mean":0.461485,"std":0.014485},{"mean":0.432915,"std":0.011265},{"mean":0.408924,"std":0.009134},{"mean":0.385636,"std":0.005434},{"mean":0.365335,"std":0.006587},{"mean":0.3464,"std":0.004324},{"mean":0.32855,"std":0.004187},{"mean":0.313587,"std":0.003564},{"mean":0.298834,"std":0.004091},{"mean":0.283844,"std":0.003947},{"mean":0.267873,"std":0.003191},{"mean":0.251751,"std":0.001924},{"mean":0.230792,"std":0.001098},{"mean":0.174863,"std":0.007233}],"coverage":[{"mean":0.040485,"std":0.003888},{"mean":0.084182,"std":0.007119},{"mean":0.129697,"std":0.008976},{"mean":0.175333,"std":0.009135},{"mean":0.220788,"std":0.001065},{"mean":0.268485,"std":0.002699},{"mean":0.318667,"std":0.004528},{"mean":0.367879,"std":0.002599},{"mean":0.417333,"std":0.002222},{"mean":0.467758,"std":0.000859},{"mean":0.519697,"std":0.005819},{"mean":0.570667,"std":0.001546},{"mean":0.626364,"std":0.005068},{"mean":0.676,"std":0.00629},{"mean":0.727515,"std":0.011443},{"mean":0.781818,"std":0.011619},{"mean":0.839758,"std":0.009535},{"mean":0.888848,"std":0.002612},{"mean":0.944182,"std":0.001734},{"mean":0.999394,"std":0.000555}],"selective_risk":[{"mean":0.117119,"std":0.020439},{"mean":0.165125,"std":0.028787},{"mean":0.194204,"std":0.03496},{"mean":0.228071,"std":0.033384},{"mean":0.257108,"std":0.030437},{"mean":0.286461,"std":0.032875},{"mean":0.315818,"std":0.035806},{"mean":0.341833,"std":0.037732},{"mean":0.363088,"std":0.037831},{"mean":0.389888,"std":0.037578},{"mean":0.414891,"std":0.032555},{"mean":0.437662,"std":0.035743},{"mean":0.459114,"std":0.037757},{"mean":0.478373,"std":0.03705},{"mean":0.497782,"std":0.036976},{"mean":0.516156,"std":0.035994},{"mean":0.534875,"std":0.031576},{"mean":0.548416,"std":0.028318},{"mean":0.565158,"std":0.02719},{"mean":0.582708,"std":0.024951}],"accuracy_8":[{"mean":0.819818,"std":0.00063},{"mean":0.815879,"std":0.002917},{"mean":0.810424,"std":0.004818},{"mean":0.801152,"std":0.007913},{"mean":0.792121,"std":0.008552},{"mean":0.780364,"std":0.010418},{"mean":0.764909,"std":0.013451},{"mean":0.749091,"std":0.015293},{"mean":0.731939,"std":0.016676},{"mean":0.710667,"std":0.017937},{"mean":0.687697,"std":0.018164},{"mean":0.663515,"std":0.022757},{"mean":0.635515,"std":0.026847},{"mean":0.609273,"std":0.028819},{"mean":0.580121,"std":0.032137},{"mean":0.549818,"std":0.032897},{"mean":0.515515,"std":0.029956},{"mean":0.488121,"std":0.026249},{"mean":0.454061,"std":0.025448},{"mean":0.417394,"std":0.025012}],"high_conf_oos_misroute_rate":[{"mean":0.008333,"std":0.002082},{"mean":0.022667,"std":0.005686},{"mean":0.041,"std":0.007},{"mean":0.062,"std":0.008544},{"mean":0.09,"std":0.009849},{"mean":0.119667,"std":0.01124},{"mean":0.162,"std":0.018028},{"mean":0.205667,"std":0.019009},{"mean":0.245667,"std":0.016258},{"mean":0.3,"std":0.019975},{"mean":0.358,"std":0.022517},{"mean":0.416,"std":0.019975},{"mean":0.478333,"std":0.029939},{"mean":0.530667,"std":0.030105},{"mean":0.594333,"std":0.035796},{"mean":0.668667,"std":0.040104},{"mean":0.751,"std":0.04859},{"mean":0.816333,"std":0.026502},{"mean":0.888,"std":0.019053},{"mean":0.981333,"std":0.007371}],"oos_misroute_rate":[{"mean":0.433667,"std":0.000577},{"mean":0.435,"std":0.001732},{"mean":0.444,"std":0.005292},{"mean":0.453667,"std":0.005859},{"mean":0.464667,"std":0.005132},{"mean":0.476667,"std":0.005132},{"mean":0.499,"std":0.008888},{"mean":0.519333,"std":0.009866},{"mean":0.537333,"std":0.008963},{"mean":0.568,"std":0.01179},{"mean":0.602,"std":0.012166},{"mean":0.636,"std":0.012166},{"mean":0.674333,"std":0.01861},{"mean":0.706,"std":0.020664},{"mean":0.744667,"std":0.023544},{"mean":0.788667,"std":0.024338},{"mean":0.839,"std":0.032357},{"mean":0.876,"std":0.019079},{"mean":0.918333,"std":0.013614},{"mean":0.982,"std":0.007211}],"llm_call_rate":[{"mean":0.959515,"std":0.003888},{"mean":0.915818,"std":0.007119},{"mean":0.870303,"std":0.008976},{"mean":0.824667,"std":0.009135},{"mean":0.779212,"std":0.001065},{"mean":0.731515,"std":0.002699},{"mean":0.681333,"std":0.004528},{"mean":0.632121,"std":0.002599},{"mean":0.582667,"std":0.002222},{"mean":0.532242,"std":0.000859},{"mean":0.480303,"std":0.005819},{"mean":0.429333,"std":0.001546},{"mean":0.373636,"std":0.005068},{"mean":0.324,"std":0.00629},{"mean":0.272485,"std":0.011443},{"mean":0.218182,"std":0.011619},{"mean":0.160242,"std":0.009535},{"mean":0.111152,"std":0.002612},{"mean":0.055818,"std":0.001734},{"mean":0.000606,"std":0.000555}]}},"entropy":{"risk_coverage":[{"mean":0.130909,"std":0.016664},{"mean":0.181212,"std":0.020274},{"mean":0.214545,"std":0.027958},{"mean":0.246667,"std":0.026696},{"mean":0.28,"std":0.029524},{"mean":0.309899,"std":0.034574},{"mean":0.333333,"std":0.034624},{"mean":0.364091,"std":0.035422},{"mean":0.38936,"std":0.037663},{"mean":0.412485,"std":0.03454},{"mean":0.431956,"std":0.033606},{"mean":0.453232,"std":0.033777},{"mean":0.471795,"std":0.032903},{"mean":0.49039,"std":0.033183},{"mean":0.507071,"std":0.032757},{"mean":0.523864,"std":0.030953},{"mean":0.537968,"std":0.029107},{"mean":0.553266,"std":0.027594},{"mean":0.568485,"std":0.026983},{"mean":0.582909,"std":0.024735}],"operating":{"tau":[{"mean":-0.613079,"std":0.0466},{"mean":-0.989318,"std":0.056081},{"mean":-1.187154,"std":0.049778},{"mean":-1.317382,"std":0.041472},{"mean":-1.407273,"std":0.028793},{"mean":-1.480261,"std":0.019166},{"mean":-1.536548,"std":0.019293},{"mean":-1.581741,"std":0.015047},{"mean":-1.617402,"std":0.009122},{"mean":-1.651706,"std":0.008238},{"mean":-1.680305,"std":0.006451},{"mean":-1.708367,"std":0.004164},{"mean":-1.733882,"std":0.003058},{"mean":-1.758562,"std":0.002191},{"mean":-1.783097,"std":0.00084},{"mean":-1.804972,"std":0.002585},{"mean":-1.829852,"std":0.002714},{"mean":-1.857409,"std":0.003148},{"mean":-1.896378,"std":0.004014},{"mean":-2.040678,"std":0.014022}],"coverage":[{"mean":0.04,"std":0.003779},{"mean":0.084182,"std":0.008188},{"mean":0.128364,"std":0.007805},{"mean":0.174303,"std":0.010093},{"mean":0.218606,"std":0.006577},{"mean":0.269515,"std":0.002612},{"mean":0.316727,"std":0.002204},{"mean":0.365091,"std":0.002234},{"mean":0.412545,"std":0.005197},{"mean":0.463879,"std":0.003442},{"mean":0.512121,"std":0.000639},{"mean":0.564,"std":0.003004},{"mean":0.617273,"std":0.003026},{"mean":0.672667,"std":0.000278},{"mean":0.730848,"std":0.007373},{"mean":0.780606,"std":0.004615},{"mean":0.832,"std":0.004062},{"mean":0.883515,"std":0.006352},{"mean":0.939091,"std":0.00882},{"mean":0.999576,"std":0.000584}],"selective_risk":[{"mean":0.117083,"std":0.023064},{"mean":0.166521,"std":0.033338},{"mean":0.203998,"std":0.034281},{"mean":0.231004,"std":0.032713},{"mean":0.258633,"std":0.032554},{"mean":0.294254,"std":0.03364},{"mean":0.319017,"std":0.036586},{"mean":0.342539,"std":0.034135},{"mean":0.371483,"std":0.034384},{"mean":0.395861,"std":0.037021},{"mean":0.417372,"std":0.034746},{"mean":0.438319,"std":0.035652},{"mean":0.460175,"std":0.034631},{"mean":0.480953,"std":0.032521},{"mean":0.500774,"std":0.034661},{"mean":0.517127,"std":0.032753},{"mean":0.533062,"std":0.030031},{"mean":0.548809,"std":0.030342},{"mean":0.56534,"std":0.027636},{"mean":0.582731,"std":0.024761}],"accuracy_8":[{"mean":0.819818,"std":0.00063},{"mean":0.816061,"std":0.003135},{"mean":0.809212,"std":0.004927},{"mean":0.801636,"std":0.007662},{"mean":0.792182,"std":0.009291},{"mean":0.777636,"std":0.010356},{"mean":0.763152,"std":0.012143},{"mean":0.748121,"std":0.012547},{"mean":0.72903,"std":0.014762},{"mean":0.708424,"std":0.018644},{"mean":0.687212,"std":0.021686},{"mean":0.663394,"std":0.02341},{"mean":0.637394,"std":0.026156},{"mean":0.608424,"std":0.02507},{"mean":0.576667,"std":0.028343},{"mean":0.548848,"std":0.027766},{"mean":0.520061,"std":0.025928},{"mean":0.489697,"std":0.028506},{"mean":0.455091,"std":0.02576},{"mean":0.417515,"std":0.024793}],"high_conf_oos_misroute_rate":[{"mean":0.007667,"std":0.001528},{"mean":0.021667,"std":0.004726},{"mean":0.039,"std":0.007},{"mean":0.060667,"std":0.01106},{"mean":0.084,"std":0.013115},{"mean":0.116667,"std":0.010263},{"mean":0.147,"std":0.010149},{"mean":0.181667,"std":0.012014},{"mean":0.224,"std":0.011136},{"mean":0.275333,"std":0.020793},{"mean":0.320667,"std":0.023459},{"mean":0.37,"std":0.026851},{"mean":0.435333,"std":0.031214},{"mean":0.495667,"std":0.035698},{"mean":0.563333,"std":0.036295},{"mean":0.626667,"std":0.043155},{"mean":0.692,"std":0.042509},{"mean":0.767,"std":0.052115},{"mean":0.858667,"std":0.052975},{"mean":0.982,"std":0.006557}],"oos_misroute_rate":[{"mean":0.433333,"std":0.000577},{"mean":0.434333,"std":0.001528},{"mean":0.443,"std":0.003606},{"mean":0.452667,"std":0.004933},{"mean":0.459667,"std":0.004041},{"mean":0.474333,"std":0.005686},{"mean":0.488667,"std":0.004933},{"mean":0.502667,"std":0.003215},{"mean":0.522667,"std":0.003512},{"mean":0.548667,"std":0.006658},{"mean":0.575333,"std":0.006658},{"mean":0.603333,"std":0.009815},{"mean":0.642333,"std":0.015567},{"mean":0.679333,"std":0.015535},{"mean":0.719,"std":0.021},{"mean":0.757333,"std":0.021221},{"mean":0.793667,"std":0.021362},{"mean":0.839667,"std":0.024846},{"mean":0.901,"std":0.029547},{"mean":0.982,"std":0.006557}],"llm_call_rate":[{"mean":0.96,"std":0.003779},{"mean":0.915818,"std":0.008188},{"mean":0.871636,"std":0.007805},{"mean":0.825697,"std":0.010093},{"mean":0.781394,"std":0.006577},{"mean":0.730485,"std":0.002612},{"mean":0.683273,"std":0.002204},{"mean":0.634909,"std":0.002234},{"mean":0.587455,"std":0.005197},{"mean":0.536121,"std":0.003442},{"mean":0.487879,"std":0.000639},{"mean":0.436,"std":0.003004},{"mean":0.382727,"std":0.003026},{"mean":0.327333,"std":0.000278},{"mean":0.269152,"std":0.007373},{"mean":0.219394,"std":0.004615},{"mean":0.168,"std":0.004062},{"mean":0.116485,"std":0.006352},{"mean":0.060909,"std":0.00882},{"mean":0.000424,"std":0.000584}]}},"margin":{"risk_coverage":[{"mean":0.10303,"std":0.021919},{"mean":0.140606,"std":0.015463},{"mean":0.175354,"std":0.011647},{"mean":0.215152,"std":0.022532},{"mean":0.257212,"std":0.027982},{"mean":0.296162,"std":0.032718},{"mean":0.327792,"std":0.045564},{"mean":0.360152,"std":0.044998},{"mean":0.391111,"std":0.04064},{"mean":0.419152,"std":0.041064},{"mean":0.442534,"std":0.037903},{"mean":0.464747,"std":0.034028},{"mean":0.48317,"std":0.033602},{"mean":0.500952,"std":0.032655},{"mean":0.517899,"std":0.028987},{"mean":0.534773,"std":0.029249},{"mean":0.547879,"std":0.027865},{"mean":0.560404,"std":0.025621},{"mean":0.572376,"std":0.026063},{"mean":0.582909,"std":0.024735}],"operating":{"tau":[{"mean":1.697157,"std":0.265983},{"mean":1.274462,"std":0.220265},{"mean":1.048227,"std":0.170127},{"mean":0.887158,"std":0.150696},{"mean":0.764262,"std":0.128904},{"mean":0.66316,"std":0.098464},{"mean":0.579245,"std":0.081125},{"mean":0.504166,"std":0.071738},{"mean":0.440284,"std":0.061913},{"mean":0.384593,"std":0.053511},{"mean":0.330083,"std":0.043193},{"mean":0.282119,"std":0.03753},{"mean":0.23916,"std":0.031389},{"mean":0.19962,"std":0.025692},{"mean":0.161417,"std":0.020242},{"mean":0.125517,"std":0.017173},{"mean":0.093283,"std":0.014998},{"mean":0.063165,"std":0.00862},{"mean":0.031635,"std":0.005791},{"mean":0.000379,"std":0.000208}],"coverage":[{"mean":0.040909,"std":0.00513},{"mean":0.083576,"std":0.006058},{"mean":0.126909,"std":0.003469},{"mean":0.174545,"std":0.002926},{"mean":0.221818,"std":0.007406},{"mean":0.270848,"std":0.002624},{"mean":0.31697,"std":0.004506},{"mean":0.369818,"std":0.003469},{"mean":0.419576,"std":0.004001},{"mean":0.466424,"std":0.005425},{"mean":0.519576,"std":0.008984},{"mean":0.572,"std":0.011273},{"mean":0.624788,"std":0.011},{"mean":0.677576,"std":0.013379},{"mean":0.733091,"std":0.011202},{"mean":0.78703,"std":0.011151},{"mean":0.837697,"std":0.013715},{"mean":0.888121,"std":0.006357},{"mean":0.942545,"std":0.008405},{"mean":0.999212,"std":0.000897}],"selective_risk":[{"mean":0.097995,"std":0.030877},{"mean":0.124712,"std":0.016428},{"mean":0.154761,"std":0.021082},{"mean":0.19499,"std":0.018069},{"mean":0.233201,"std":0.025786},{"mean":0.2731,"std":0.033493},{"mean":0.307883,"std":0.034163},{"mean":0.341366,"std":0.048926},{"mean":0.372269,"std":0.047904},{"mean":0.401618,"std":0.043252},{"mean":0.428853,"std":0.042987},{"mean":0.453134,"std":0.03953},{"mean":0.473765,"std":0.037218},{"mean":0.493362,"std":0.037188},{"mean":0.512923,"std":0.033205},{"mean":0.530361,"std":0.032867},{"mean":0.544965,"std":0.030415},{"mean":0.556852,"std":0.027597},{"mean":0.570834,"std":0.027773},{"mean":0.582837,"std":0.024673}],"accuracy_8":[{"mean":0.820182,"std":0.000656},{"mean":0.818303,"std":0.001766},{"mean":0.814,"std":0.002097},{"mean":0.805394,"std":0.003442},{"mean":0.795879,"std":0.007364},{"mean":0.781818,"std":0.011028},{"mean":0.766606,"std":0.012962},{"mean":0.747758,"std":0.020614},{"mean":0.727152,"std":0.023396},{"mean":0.70503,"std":0.024064},{"mean":0.680182,"std":0.027266},{"mean":0.652848,"std":0.028161},{"mean":0.62703,"std":0.027694},{"mean":0.597879,"std":0.029346},{"mean":0.567879,"std":0.02916},{"mean":0.537273,"std":0.030185},{"mean":0.509394,"std":0.031288},{"mean":0.481758,"std":0.027015},{"mean":0.449576,"std":0.028752},{"mean":0.417576,"std":0.024231}],"high_conf_oos_misroute_rate":[{"mean":0.009,"std":0.004},{"mean":0.022,"std":0.004359},{"mean":0.039333,"std":0.01097},{"mean":0.063,"std":0.007211},{"mean":0.099,"std":0.013528},{"mean":0.143667,"std":0.01193},{"mean":0.188333,"std":0.021197},{"mean":0.238,"std":0.018},{"mean":0.298333,"std":0.018009},{"mean":0.354333,"std":0.018556},{"mean":0.411667,"std":0.025423},{"mean":0.470667,"std":0.035388},{"mean":0.530333,"std":0.028024},{"mean":0.594667,"std":0.030892},{"mean":0.658,"std":0.029103},{"mean":0.728667,"std":0.029687},{"mean":0.799333,"std":0.024007},{"mean":0.853333,"std":0.017156},{"mean":0.920333,"std":0.020232},{"mean":0.980667,"std":0.007638}],"oos_misroute_rate":[{"mean":0.433333,"std":0.000577},{"mean":0.435333,"std":0.002309},{"mean":0.441333,"std":0.001528},{"mean":0.450333,"std":0.004933},{"mean":0.467,"std":0.006083},{"mean":0.491,"std":0.006083},{"mean":0.515333,"std":0.005686},{"mean":0.540333,"std":0.008327},{"mean":0.575333,"std":0.009866},{"mean":0.606333,"std":0.009866},{"mean":0.637667,"std":0.018824},{"mean":0.676333,"std":0.026312},{"mean":0.711,"std":0.017321},{"mean":0.752667,"std":0.016921},{"mean":0.790333,"std":0.018502},{"mean":0.831333,"std":0.021362},{"mean":0.874,"std":0.018735},{"mean":0.906333,"std":0.014224},{"mean":0.946,"std":0.013856},{"mean":0.981,"std":0.00755}],"llm_call_rate":[{"mean":0.959091,"std":0.00513},{"mean":0.916424,"std":0.006058},{"mean":0.873091,"std":0.003469},{"mean":0.825455,"std":0.002926},{"mean":0.778182,"std":0.007406},{"mean":0.729152,"std":0.002624},{"mean":0.68303,"std":0.004506},{"mean":0.630182,"std":0.003469},{"mean":0.580424,"std":0.004001},{"mean":0.533576,"std":0.005425},{"mean":0.480424,"std":0.008984},{"mean":0.428,"std":0.011273},{"mean":0.375212,"std":0.011},{"mean":0.322424,"std":0.013379},{"mean":0.266909,"std":0.011202},{"mean":0.21297,"std":0.011151},{"mean":0.162303,"std":0.013715},{"mean":0.111879,"std":0.006357},{"mean":0.057455,"std":0.008405},{"mean":0.000788,"std":0.000897}]}},"msp_t":{"risk_coverage":[{"mean":0.105455,"std":0.007273},{"mean":0.126061,"std":0.005845},{"mean":0.159596,"std":0.017411},{"mean":0.193636,"std":0.021262},{"mean":0.230303,"std":0.027215},{"mean":0.266869,"std":0.029099},{"mean":0.29974,"std":0.031612},{"mean":0.334545,"std":0.03285},{"mean":0.369562,"std":0.033653},{"mean":0.400485,"std":0.033688},{"mean":0.424463,"std":0.03611},{"mean":0.450202,"std":0.036303},{"mean":0.471235,"std":0.035065},{"mean":0.489957,"std":0.033938},{"mean":0.507071,"std":0.034823},{"mean":0.524621,"std":0.032557},{"mean":0.539893,"std":0.030723},{"mean":0.553939,"std":0.02815},{"mean":0.569314,"std":0.025649},{"mean":0.582909,"std":0.024735}],"operating":{"tau":[{"mean":0.619433,"std":0.06027},{"mean":0.514728,"std":0.052185},{"mean":0.462448,"std":0.042799},{"mean":0.421269,"std":0.035306},{"mean":0.391764,"std":0.03018},{"mean":0.366432,"std":0.025136},{"mean":0.348097,"std":0.021663},{"mean":0.32912,"std":0.018245},{"mean":0.314761,"std":0.015795},{"mean":0.302089,"std":0.014055},{"mean":0.290953,"std":0.011485},{"mean":0.281137,"std":0.010829},{"mean":0.271619,"std":0.009075},{"mean":0.262014,"std":0.007706},{"mean":0.253364,"std":0.007292},{"mean":0.244006,"std":0.006252},{"mean":0.234103,"std":0.005471},{"mean":0.224132,"std":0.004743},{"mean":0.211452,"std":0.003497},{"mean":0.169548,"std":0.005553}],"coverage":[{"mean":0.040788,"std":0.004987},{"mean":0.084,"std":0.006222},{"mean":0.125455,"std":0.002727},{"mean":0.173697,"std":0.004001},{"mean":0.218121,"std":0.003494},{"mean":0.266545,"std":0.004166},{"mean":0.309636,"std":0.00619},{"mean":0.36503,"std":0.005948},{"mean":0.413939,"std":0.005954},{"mean":0.463091,"std":0.007266},{"mean":0.51697,"std":0.006289},{"mean":0.567576,"std":0.007836},{"mean":0.620909,"std":0.005928},{"mean":0.675879,"std":0.005522},{"mean":0.730485,"std":0.006748},{"mean":0.786364,"std":0.007695},{"mean":0.841697,"std":0.005739},{"mean":0.89303,"std":0.006331},{"mean":0.943576,"std":0.004203},{"mean":0.998788,"std":0.001211}],"selective_risk":[{"mean":0.096092,"std":0.009064},{"mean":0.115872,"std":0.011851},{"mean":0.143755,"std":0.014769},{"mean":0.178652,"std":0.015608},{"mean":0.20596,"std":0.025154},{"mean":0.242078,"std":0.029273},{"mean":0.273938,"std":0.025485},{"mean":0.311095,"std":0.028107},{"mean":0.345582,"std":0.035094},{"mean":0.379209,"std":0.033747},{"mean":0.408698,"std":0.034978},{"mean":0.434645,"std":0.039186},{"mean":0.45834,"std":0.038052},{"mean":0.480444,"std":0.036784},{"mean":0.499176,"std":0.036128},{"mean":0.519308,"std":0.035847},{"mean":0.537587,"std":0.032596},{"mean":0.551469,"std":0.030407},{"mean":0.567288,"std":0.026738},{"mean":0.582445,"std":0.025138}],"accuracy_8":[{"mean":0.820727,"std":0.0},{"mean":0.818667,"std":0.001001},{"mean":0.81497,"std":0.002184},{"mean":0.808182,"std":0.002526},{"mean":0.800667,"std":0.006247},{"mean":0.789152,"std":0.008556},{"mean":0.776788,"std":0.009057},{"mean":0.758727,"std":0.012466},{"mean":0.738424,"std":0.016435},{"mean":0.715212,"std":0.018693},{"mean":0.690364,"std":0.021604},{"mean":0.665455,"std":0.027386},{"mean":0.638061,"std":0.028711},{"mean":0.608242,"std":0.02934},{"mean":0.578,"std":0.030234},{"mean":0.546,"std":0.033708},{"mean":0.513333,"std":0.03034},{"mean":0.484364,"std":0.029479},{"mean":0.452061,"std":0.025678},{"mean":0.417939,"std":0.025416}],"high_conf_oos_misroute_rate":[{"mean":0.008333,"std":0.001528},{"mean":0.020333,"std":0.003512},{"mean":0.030667,"std":0.006429},{"mean":0.052333,"std":0.004933},{"mean":0.079,"std":0.01044},{"mean":0.119,"std":0.008185},{"mean":0.151667,"std":0.01124},{"mean":0.203,"std":0.020298},{"mean":0.255667,"std":0.027574},{"mean":0.314667,"std":0.033126},{"mean":0.373,"std":0.040731},{"mean":0.433333,"std":0.047962},{"mean":0.495333,"std":0.043386},{"mean":0.560667,"std":0.044106},{"mean":0.621,"std":0.04564},{"mean":0.691,"std":0.049},{"mean":0.755,"std":0.047655},{"mean":0.820333,"std":0.03926},{"mean":0.889333,"std":0.026312},{"mean":0.98,"std":0.010817}],"oos_misroute_rate":[{"mean":0.432333,"std":0.000577},{"mean":0.435,"std":0.0},{"mean":0.436333,"std":0.001528},{"mean":0.442333,"std":0.001528},{"mean":0.454667,"std":0.001528},{"mean":0.469667,"std":0.005686},{"mean":0.484667,"std":0.000577},{"mean":0.51,"std":0.005},{"mean":0.539333,"std":0.01222},{"mean":0.571667,"std":0.013317},{"mean":0.601667,"std":0.018037},{"mean":0.636333,"std":0.025146},{"mean":0.671667,"std":0.023288},{"mean":0.714,"std":0.026889},{"mean":0.757,"std":0.023431},{"mean":0.802333,"std":0.028042},{"mean":0.840333,"std":0.028937},{"mean":0.880333,"std":0.022811},{"mean":0.924,"std":0.018193},{"mean":0.980667,"std":0.009713}],"llm_call_rate":[{"mean":0.959212,"std":0.004987},{"mean":0.916,"std":0.006222},{"mean":0.874545,"std":0.002727},{"mean":0.826303,"std":0.004001},{"mean":0.781879,"std":0.003494},{"mean":0.733455,"std":0.004166},{"mean":0.690364,"std":0.00619},{"mean":0.63497,"std":0.005948},{"mean":0.586061,"std":0.005954},{"mean":0.536909,"std":0.007266},{"mean":0.48303,"std":0.006289},{"mean":0.432424,"std":0.007836},{"mean":0.379091,"std":0.005928},{"mean":0.324121,"std":0.005522},{"mean":0.269515,"std":0.006748},{"mean":0.213636,"std":0.007695},{"mean":0.158303,"std":0.005739},{"mean":0.10697,"std":0.006331},{"mean":0.056424,"std":0.004203},{"mean":0.001212,"std":0.001211}]}}}}},"modernbert/k10":{"model":"modernbert","k":10,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["ModernBERT-base-k10-seed42","ModernBERT-base-k10-seed43","ModernBERT-base-k10-seed44"],"logits_sha256":["6fff9cd41ff6f9c828b9337b3c190f6e4cb1e41dff62667329f5e0647f81afba","f08779235eb7af011ab4c6402e86fe393d5cb9cc0d341ecfb01992fe97d52004","6c20ced25748c4ed64688269d0ab4fe74f0b32ae9af565c8ccd4cf6ed72140d9"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001818,"std":0.001818},{"mean":0.00202,"std":0.0014},{"mean":0.003333,"std":0.001892},{"mean":0.004364,"std":0.002519},{"mean":0.005455,"std":0.002185},{"mean":0.006407,"std":0.00167},{"mean":0.007727,"std":0.001203},{"mean":0.010236,"std":0.000233},{"mean":0.012606,"std":0.000757},{"mean":0.017631,"std":0.000954},{"mean":0.023939,"std":0.000606},{"mean":0.031515,"std":0.003337},{"mean":0.042597,"std":0.004949},{"mean":0.055273,"std":0.006563},{"mean":0.074167,"std":0.006827},{"mean":0.09697,"std":0.00733},{"mean":0.122155,"std":0.007983},{"mean":0.149537,"std":0.009677},{"mean":0.180061,"std":0.011099}],"operating":{"tau":[{"mean":0.998654,"std":0.000256},{"mean":0.997265,"std":0.000625},{"mean":0.995245,"std":0.00124},{"mean":0.992695,"std":0.002228},{"mean":0.989,"std":0.003516},{"mean":0.984019,"std":0.005089},{"mean":0.976616,"std":0.008262},{"mean":0.966275,"std":0.01156},{"mean":0.950397,"std":0.016477},{"mean":0.928179,"std":0.021914},{"mean":0.894865,"std":0.033592},{"mean":0.849024,"std":0.049128},{"mean":0.788161,"std":0.058676},{"mean":0.71552,"std":0.067442},{"mean":0.625673,"std":0.062564},{"mean":0.529639,"std":0.057925},{"mean":0.43908,"std":0.049504},{"mean":0.328556,"std":0.035935},{"mean":0.223168,"std":0.021547},{"mean":0.067537,"std":0.006748}],"coverage":[{"mean":0.045212,"std":0.003785},{"mean":0.089758,"std":0.006559},{"mean":0.135515,"std":0.006455},{"mean":0.17503,"std":0.00873},{"mean":0.216667,"std":0.01117},{"mean":0.254545,"std":0.003933},{"mean":0.292364,"std":0.006147},{"mean":0.334364,"std":0.002834},{"mean":0.380667,"std":0.00347},{"mean":0.424667,"std":0.002345},{"mean":0.474909,"std":0.001311},{"mean":0.519515,"std":0.00447},{"mean":0.567515,"std":0.004042},{"mean":0.612424,"std":0.002662},{"mean":0.661939,"std":0.00134},{"mean":0.712,"std":0.00512},{"mean":0.761091,"std":0.005357},{"mean":0.824424,"std":0.006924},{"mean":0.896303,"std":0.009431},{"mean":0.999394,"std":0.000278}],"selective_risk":[{"mean":0.001437,"std":0.002489},{"mean":0.001405,"std":0.001219},{"mean":0.002283,"std":0.001683},{"mean":0.003101,"std":0.001984},{"mean":0.00386,"std":0.00222},{"mean":0.004515,"std":0.00227},{"mean":0.005165,"std":0.001921},{"mean":0.005796,"std":0.001098},{"mean":0.007161,"std":0.001232},{"mean":0.009134,"std":0.000264},{"mean":0.011229,"std":0.000559},{"mean":0.014926,"std":0.001199},{"mean":0.019222,"std":0.0005},{"mean":0.025232,"std":0.001186},{"mean":0.033786,"std":0.003135},{"mean":0.045932,"std":0.007189},{"mean":0.059763,"std":0.008373},{"mean":0.084119,"std":0.01007},{"mean":0.120888,"std":0.011608},{"mean":0.179624,"std":0.011175}],"accuracy_8":[{"mean":0.823212,"std":0.00082},{"mean":0.826788,"std":0.00042},{"mean":0.829939,"std":0.000639},{"mean":0.833333,"std":0.001377},{"mean":0.836848,"std":0.002044},{"mean":0.840182,"std":0.001373},{"mean":0.843455,"std":0.002049},{"mean":0.848121,"std":0.002421},{"mean":0.852424,"std":0.003119},{"mean":0.85703,"std":0.002956},{"mean":0.862485,"std":0.002753},{"mean":0.866303,"std":0.003066},{"mean":0.871697,"std":0.002989},{"mean":0.874606,"std":0.002795},{"mean":0.876606,"std":0.003259},{"mean":0.876545,"std":0.003232},{"mean":0.875515,"std":0.004791},{"mean":0.868909,"std":0.007406},{"mean":0.854424,"std":0.008623},{"mean":0.820182,"std":0.011236}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.000577},{"mean":0.002333,"std":0.002309},{"mean":0.003333,"std":0.002517},{"mean":0.004667,"std":0.002082},{"mean":0.006,"std":0.001732},{"mean":0.009,"std":0.002646},{"mean":0.012667,"std":0.002517},{"mean":0.017667,"std":0.000577},{"mean":0.025,"std":0.002646},{"mean":0.034667,"std":0.001528},{"mean":0.049667,"std":0.003786},{"mean":0.072667,"std":0.008021},{"mean":0.112,"std":0.017321},{"mean":0.158667,"std":0.022855},{"mean":0.251,"std":0.035043},{"mean":0.406,"std":0.050715},{"mean":0.709667,"std":0.054418}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001155},{"mean":0.433,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.435,"std":0.001732},{"mean":0.434667,"std":0.001155},{"mean":0.435667,"std":0.001155},{"mean":0.437333,"std":0.001528},{"mean":0.44,"std":0.001},{"mean":0.446,"std":0.001732},{"mean":0.461,"std":0.007},{"mean":0.474333,"std":0.008386},{"mean":0.507333,"std":0.020817},{"mean":0.571333,"std":0.034962},{"mean":0.711,"std":0.054745}],"llm_call_rate":[{"mean":0.954788,"std":0.003785},{"mean":0.910242,"std":0.006559},{"mean":0.864485,"std":0.006455},{"mean":0.82497,"std":0.00873},{"mean":0.783333,"std":0.01117},{"mean":0.745455,"std":0.003933},{"mean":0.707636,"std":0.006147},{"mean":0.665636,"std":0.002834},{"mean":0.619333,"std":0.00347},{"mean":0.575333,"std":0.002345},{"mean":0.525091,"std":0.001311},{"mean":0.480485,"std":0.00447},{"mean":0.432485,"std":0.004042},{"mean":0.387576,"std":0.002662},{"mean":0.338061,"std":0.00134},{"mean":0.288,"std":0.00512},{"mean":0.238909,"std":0.005357},{"mean":0.175576,"std":0.006924},{"mean":0.103697,"std":0.009431},{"mean":0.000606,"std":0.000278}]}},"entropy":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001818,"std":0.001818},{"mean":0.002424,"std":0.001212},{"mean":0.003333,"std":0.00105},{"mean":0.004364,"std":0.002519},{"mean":0.004848,"std":0.001603},{"mean":0.006234,"std":0.001873},{"mean":0.007727,"std":0.001203},{"mean":0.009293,"std":0.0},{"mean":0.012121,"std":0.000915},{"mean":0.016749,"std":0.000954},{"mean":0.021313,"std":0.0007},{"mean":0.028159,"std":0.002688},{"mean":0.038182,"std":0.005498},{"mean":0.052848,"std":0.008189},{"mean":0.071288,"std":0.006759},{"mean":0.094617,"std":0.007121},{"mean":0.121347,"std":0.008182},{"mean":0.148708,"std":0.008458},{"mean":0.180061,"std":0.011099}],"operating":{"tau":[{"mean":-0.014497,"std":0.002573},{"mean":-0.027419,"std":0.005618},{"mean":-0.044338,"std":0.010682},{"mean":-0.066083,"std":0.018363},{"mean":-0.093853,"std":0.026965},{"mean":-0.128718,"std":0.036804},{"mean":-0.174684,"std":0.05244},{"mean":-0.23934,"std":0.068619},{"mean":-0.329726,"std":0.089301},{"mean":-0.449558,"std":0.118078},{"mean":-0.591526,"std":0.13978},{"mean":-0.785495,"std":0.189273},{"mean":-1.003542,"std":0.216929},{"mean":-1.239539,"std":0.242856},{"mean":-1.546928,"std":0.274034},{"mean":-1.875696,"std":0.236067},{"mean":-2.271427,"std":0.240907},{"mean":-2.750297,"std":0.152199},{"mean":-3.214918,"std":0.096347},{"mean":-4.033278,"std":0.016357}],"coverage":[{"mean":0.04503,"std":0.003413},{"mean":0.089273,"std":0.006735},{"mean":0.133939,"std":0.007136},{"mean":0.17703,"std":0.011407},{"mean":0.217697,"std":0.011099},{"mean":0.254909,"std":0.004329},{"mean":0.29097,"std":0.006632},{"mean":0.331879,"std":0.001689},{"mean":0.379636,"std":0.001488},{"mean":0.424788,"std":0.004547},{"mean":0.47297,"std":0.005308},{"mean":0.52,"std":0.007919},{"mean":0.567091,"std":0.001585},{"mean":0.610485,"std":0.003403},{"mean":0.656,"std":0.003691},{"mean":0.700545,"std":0.002097},{"mean":0.75103,"std":0.00559},{"mean":0.817515,"std":0.004746},{"mean":0.892848,"std":0.005462},{"mean":0.999091,"std":0.000315}],"selective_risk":[{"mean":0.001443,"std":0.002499},{"mean":0.001414,"std":0.001227},{"mean":0.002322,"std":0.001738},{"mean":0.003064,"std":0.001938},{"mean":0.004392,"std":0.002486},{"mean":0.004271,"std":0.002439},{"mean":0.004766,"std":0.001806},{"mean":0.00584,"std":0.001359},{"mean":0.007503,"std":0.000997},{"mean":0.008277,"std":0.000329},{"mean":0.011268,"std":0.001235},{"mean":0.014198,"std":0.002135},{"mean":0.017633,"std":0.001093},{"mean":0.02263,"std":0.001169},{"mean":0.028813,"std":0.003856},{"mean":0.03823,"std":0.00571},{"mean":0.053047,"std":0.0104},{"mean":0.079384,"std":0.006094},{"mean":0.117801,"std":0.010107},{"mean":0.17968,"std":0.010915}],"accuracy_8":[{"mean":0.823273,"std":0.000656},{"mean":0.826485,"std":0.000688},{"mean":0.829515,"std":0.00134},{"mean":0.833636,"std":0.001575},{"mean":0.836727,"std":0.001488},{"mean":0.839879,"std":0.001377},{"mean":0.843212,"std":0.002302},{"mean":0.847758,"std":0.002259},{"mean":0.852121,"std":0.003193},{"mean":0.856606,"std":0.002508},{"mean":0.861394,"std":0.003437},{"mean":0.865818,"std":0.003333},{"mean":0.870424,"std":0.003642},{"mean":0.874485,"std":0.001944},{"mean":0.876848,"std":0.002554},{"mean":0.878182,"std":0.003107},{"mean":0.877636,"std":0.00512},{"mean":0.870242,"std":0.004832},{"mean":0.856303,"std":0.006629},{"mean":0.820121,"std":0.011209}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.000577},{"mean":0.002667,"std":0.002082},{"mean":0.003,"std":0.002646},{"mean":0.004333,"std":0.002517},{"mean":0.006,"std":0.002646},{"mean":0.008667,"std":0.001528},{"mean":0.011333,"std":0.002082},{"mean":0.016333,"std":0.002309},{"mean":0.022667,"std":0.003215},{"mean":0.03,"std":0.004359},{"mean":0.04,"std":0.002646},{"mean":0.053,"std":0.005196},{"mean":0.080667,"std":0.009866},{"mean":0.129667,"std":0.028042},{"mean":0.225667,"std":0.022502},{"mean":0.384333,"std":0.038136},{"mean":0.709667,"std":0.053003}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001155},{"mean":0.432667,"std":0.001155},{"mean":0.433667,"std":0.001155},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.0},{"mean":0.434667,"std":0.001528},{"mean":0.434667,"std":0.002082},{"mean":0.434667,"std":0.001528},{"mean":0.437,"std":0.002},{"mean":0.439333,"std":0.003055},{"mean":0.44,"std":0.001},{"mean":0.448,"std":0.004},{"mean":0.464333,"std":0.008505},{"mean":0.498333,"std":0.010017},{"mean":0.557,"std":0.02512},{"mean":0.711333,"std":0.054372}],"llm_call_rate":[{"mean":0.95497,"std":0.003413},{"mean":0.910727,"std":0.006735},{"mean":0.866061,"std":0.007136},{"mean":0.82297,"std":0.011407},{"mean":0.782303,"std":0.011099},{"mean":0.745091,"std":0.004329},{"mean":0.70903,"std":0.006632},{"mean":0.668121,"std":0.001689},{"mean":0.620364,"std":0.001488},{"mean":0.575212,"std":0.004547},{"mean":0.52703,"std":0.005308},{"mean":0.48,"std":0.007919},{"mean":0.432909,"std":0.001585},{"mean":0.389515,"std":0.003403},{"mean":0.344,"std":0.003691},{"mean":0.299455,"std":0.002097},{"mean":0.24897,"std":0.00559},{"mean":0.182485,"std":0.004746},{"mean":0.107152,"std":0.005462},{"mean":0.000909,"std":0.000315}]}},"margin":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.002424,"std":0.00105},{"mean":0.003636,"std":0.002099},{"mean":0.004545,"std":0.001818},{"mean":0.006061,"std":0.00183},{"mean":0.006465,"std":0.0007},{"mean":0.007446,"std":0.001081},{"mean":0.008636,"std":0.000455},{"mean":0.012525,"std":0.0007},{"mean":0.018424,"std":0.00042},{"mean":0.025895,"std":0.001336},{"mean":0.033232,"std":0.003183},{"mean":0.044476,"std":0.006506},{"mean":0.059134,"std":0.008099},{"mean":0.078545,"std":0.009957},{"mean":0.099242,"std":0.009646},{"mean":0.118859,"std":0.011348},{"mean":0.138249,"std":0.011457},{"mean":0.158915,"std":0.010972},{"mean":0.180061,"std":0.011099}],"operating":{"tau":[{"mean":9.29027,"std":0.093286},{"mean":8.383416,"std":0.157384},{"mean":7.6969,"std":0.215619},{"mean":7.143053,"std":0.21672},{"mean":6.658004,"std":0.210046},{"mean":6.180256,"std":0.241445},{"mean":5.71155,"std":0.222207},{"mean":5.179433,"std":0.279285},{"mean":4.701083,"std":0.309533},{"mean":4.23369,"std":0.324788},{"mean":3.755685,"std":0.319432},{"mean":3.236233,"std":0.311309},{"mean":2.752025,"std":0.289477},{"mean":2.264802,"std":0.275042},{"mean":1.760646,"std":0.209913},{"mean":1.322082,"std":0.20555},{"mean":0.929189,"std":0.152718},{"mean":0.568067,"std":0.105459},{"mean":0.265068,"std":0.041187},{"mean":0.001492,"std":0.001679}],"coverage":[{"mean":0.04503,"std":0.002911},{"mean":0.090121,"std":0.005853},{"mean":0.133879,"std":0.009381},{"mean":0.175636,"std":0.008649},{"mean":0.214485,"std":0.00548},{"mean":0.254121,"std":0.005993},{"mean":0.296545,"std":0.000793},{"mean":0.344909,"std":0.005235},{"mean":0.388848,"std":0.004147},{"mean":0.43103,"std":0.004462},{"mean":0.473818,"std":0.003273},{"mean":0.521697,"std":0.005157},{"mean":0.569697,"std":0.006632},{"mean":0.620061,"std":0.006536},{"mean":0.679455,"std":0.004643},{"mean":0.736667,"std":0.014448},{"mean":0.795515,"std":0.016906},{"mean":0.858667,"std":0.01786},{"mean":0.928606,"std":0.008136},{"mean":0.999818,"std":0.000182}],"selective_risk":[{"mean":0.001437,"std":0.002489},{"mean":0.002735,"std":0.001335},{"mean":0.003187,"std":0.001615},{"mean":0.003751,"std":0.002468},{"mean":0.004795,"std":0.002077},{"mean":0.005952,"std":0.001719},{"mean":0.00654,"std":0.000715},{"mean":0.007035,"std":0.000877},{"mean":0.008417,"std":8.9e-05},{"mean":0.011529,"std":0.000624},{"mean":0.014968,"std":0.000827},{"mean":0.02194,"std":0.002631},{"mean":0.028612,"std":0.002499},{"mean":0.037334,"std":0.004162},{"mean":0.051791,"std":0.008516},{"mean":0.073513,"std":0.015496},{"mean":0.097584,"std":0.016717},{"mean":0.121943,"std":0.017672},{"mean":0.15004,"std":0.014511},{"mean":0.180032,"std":0.011059}],"accuracy_8":[{"mean":0.823273,"std":0.001443},{"mean":0.82703,"std":0.000735},{"mean":0.830121,"std":0.000735},{"mean":0.833879,"std":0.00129},{"mean":0.837576,"std":0.00105},{"mean":0.840485,"std":0.00165},{"mean":0.844667,"std":0.002956},{"mean":0.849576,"std":0.002548},{"mean":0.854485,"std":0.003354},{"mean":0.858545,"std":0.003713},{"mean":0.862848,"std":0.003193},{"mean":0.867152,"std":0.002723},{"mean":0.870182,"std":0.00387},{"mean":0.87097,"std":0.005173},{"mean":0.871636,"std":0.00629},{"mean":0.868364,"std":0.00863},{"mean":0.860667,"std":0.010683},{"mean":0.851636,"std":0.012637},{"mean":0.837879,"std":0.012899},{"mean":0.82,"std":0.011064}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.001333,"std":0.000577},{"mean":0.001667,"std":0.001155},{"mean":0.003333,"std":0.001528},{"mean":0.004667,"std":0.001155},{"mean":0.006,"std":0.001},{"mean":0.008,"std":0.001},{"mean":0.011333,"std":0.000577},{"mean":0.018667,"std":0.002517},{"mean":0.026,"std":0.003},{"mean":0.041667,"std":0.006028},{"mean":0.057667,"std":0.008622},{"mean":0.084667,"std":0.014364},{"mean":0.132667,"std":0.024987},{"mean":0.206333,"std":0.048387},{"mean":0.300667,"std":0.059079},{"mean":0.412333,"std":0.068966},{"mean":0.554667,"std":0.059501},{"mean":0.712,"std":0.053926}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001155},{"mean":0.433333,"std":0.001528},{"mean":0.433667,"std":0.000577},{"mean":0.434333,"std":0.000577},{"mean":0.433667,"std":0.001528},{"mean":0.434,"std":0.001732},{"mean":0.434667,"std":0.001155},{"mean":0.434667,"std":0.001155},{"mean":0.438,"std":0.003606},{"mean":0.442,"std":0.0},{"mean":0.456,"std":0.005292},{"mean":0.469333,"std":0.01193},{"mean":0.492,"std":0.02488},{"mean":0.527,"std":0.032512},{"mean":0.574,"std":0.049487},{"mean":0.641,"std":0.057715},{"mean":0.712,"std":0.053926}],"llm_call_rate":[{"mean":0.95497,"std":0.002911},{"mean":0.909879,"std":0.005853},{"mean":0.866121,"std":0.009381},{"mean":0.824364,"std":0.008649},{"mean":0.785515,"std":0.00548},{"mean":0.745879,"std":0.005993},{"mean":0.703455,"std":0.000793},{"mean":0.655091,"std":0.005235},{"mean":0.611152,"std":0.004147},{"mean":0.56897,"std":0.004462},{"mean":0.526182,"std":0.003273},{"mean":0.478303,"std":0.005157},{"mean":0.430303,"std":0.006632},{"mean":0.379939,"std":0.006536},{"mean":0.320545,"std":0.004643},{"mean":0.263333,"std":0.014448},{"mean":0.204485,"std":0.016906},{"mean":0.141333,"std":0.01786},{"mean":0.071394,"std":0.008136},{"mean":0.000182,"std":0.000182}]}},"msp_t":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001818,"std":0.001818},{"mean":0.00202,"std":0.0014},{"mean":0.003636,"std":0.002405},{"mean":0.004606,"std":0.002338},{"mean":0.005455,"std":0.002099},{"mean":0.006753,"std":0.001374},{"mean":0.007424,"std":0.001144},{"mean":0.010909,"std":0.0},{"mean":0.013091,"std":0.00063},{"mean":0.018292,"std":0.000382},{"mean":0.025051,"std":0.000926},{"mean":0.03338,"std":0.003181},{"mean":0.044502,"std":0.005363},{"mean":0.058828,"std":0.005687},{"mean":0.075152,"std":0.007087},{"mean":0.097184,"std":0.007128},{"mean":0.122559,"std":0.008182},{"mean":0.14941,"std":0.009627},{"mean":0.180061,"std":0.011099}],"operating":{"tau":[{"mean":0.999585,"std":5.6e-05},{"mean":0.999067,"std":0.000166},{"mean":0.998287,"std":0.000315},{"mean":0.997115,"std":0.000641},{"mean":0.995492,"std":0.001113},{"mean":0.993152,"std":0.001503},{"mean":0.989352,"std":0.003004},{"mean":0.983725,"std":0.004514},{"mean":0.974313,"std":0.007081},{"mean":0.960673,"std":0.010147},{"mean":0.937926,"std":0.017538},{"mean":0.903031,"std":0.029226},{"mean":0.853731,"std":0.04021},{"mean":0.788688,"std":0.052727},{"mean":0.700621,"std":0.053604},{"mean":0.602264,"std":0.052901},{"mean":0.501095,"std":0.046639},{"mean":0.387385,"std":0.033195},{"mean":0.268235,"std":0.016855},{"mean":0.077414,"std":0.006973}],"coverage":[{"mean":0.046061,"std":0.004444},{"mean":0.089879,"std":0.006491},{"mean":0.135455,"std":0.005827},{"mean":0.177394,"std":0.01094},{"mean":0.216788,"std":0.010795},{"mean":0.253758,"std":0.004199},{"mean":0.293697,"std":0.005446},{"mean":0.335636,"std":0.004252},{"mean":0.381879,"std":0.003203},{"mean":0.425091,"std":0.002622},{"mean":0.474364,"std":0.002622},{"mean":0.520667,"std":0.002877},{"mean":0.567697,"std":0.003413},{"mean":0.613455,"std":0.004299},{"mean":0.664303,"std":0.00165},{"mean":0.713091,"std":0.006052},{"mean":0.764848,"std":0.0097},{"mean":0.827697,"std":0.008419},{"mean":0.896606,"std":0.010313},{"mean":0.999394,"std":0.000278}],"selective_risk":[{"mean":0.001437,"std":0.002489},{"mean":0.001403,"std":0.001217},{"mean":0.002278,"std":0.001667},{"mean":0.003054,"std":0.001931},{"mean":0.004157,"std":0.002044},{"mean":0.004529,"std":0.002261},{"mean":0.005355,"std":0.002307},{"mean":0.006131,"std":0.001303},{"mean":0.007453,"std":0.001146},{"mean":0.00884,"std":0.000266},{"mean":0.011243,"std":0.000433},{"mean":0.014898,"std":0.00085},{"mean":0.020387,"std":0.001344},{"mean":0.02696,"std":0.002346},{"mean":0.035938,"std":0.004464},{"mean":0.048488,"std":0.007696},{"mean":0.06323,"std":0.009698},{"mean":0.0872,"std":0.012057},{"mean":0.120971,"std":0.012203},{"mean":0.179624,"std":0.011175}],"accuracy_8":[{"mean":0.823394,"std":0.00105},{"mean":0.826667,"std":0.000378},{"mean":0.829939,"std":0.000555},{"mean":0.833939,"std":0.00165},{"mean":0.836788,"std":0.001718},{"mean":0.839818,"std":0.001636},{"mean":0.843818,"std":0.001488},{"mean":0.848061,"std":0.002945},{"mean":0.852667,"std":0.003093},{"mean":0.857455,"std":0.002993},{"mean":0.862848,"std":0.002442},{"mean":0.867212,"std":0.003413},{"mean":0.871394,"std":0.003692},{"mean":0.874606,"std":0.003259},{"mean":0.876303,"std":0.003611},{"mean":0.876303,"std":0.003656},{"mean":0.873879,"std":0.006015},{"mean":0.867818,"std":0.008153},{"mean":0.854727,"std":0.008752},{"mean":0.820182,"std":0.011236}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.000577},{"mean":0.002333,"std":0.002309},{"mean":0.003333,"std":0.002309},{"mean":0.004667,"std":0.002082},{"mean":0.006333,"std":0.002082},{"mean":0.01,"std":0.002646},{"mean":0.012333,"std":0.002082},{"mean":0.018,"std":0.001},{"mean":0.026333,"std":0.003055},{"mean":0.037667,"std":0.003786},{"mean":0.053333,"std":0.005859},{"mean":0.08,"std":0.011358},{"mean":0.121667,"std":0.021221},{"mean":0.172,"std":0.028792},{"mean":0.264333,"std":0.04087},{"mean":0.408333,"std":0.053715},{"mean":0.709667,"std":0.054418}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001155},{"mean":0.433333,"std":0.000577},{"mean":0.434,"std":0.001},{"mean":0.433667,"std":0.000577},{"mean":0.434667,"std":0.000577},{"mean":0.434333,"std":0.001155},{"mean":0.435,"std":0.001},{"mean":0.435,"std":0.001},{"mean":0.438,"std":0.001732},{"mean":0.440667,"std":0.000577},{"mean":0.448333,"std":0.003786},{"mean":0.462,"std":0.008},{"mean":0.479333,"std":0.01464},{"mean":0.513667,"std":0.025027},{"mean":0.570333,"std":0.034078},{"mean":0.711,"std":0.054745}],"llm_call_rate":[{"mean":0.953939,"std":0.004444},{"mean":0.910121,"std":0.006491},{"mean":0.864545,"std":0.005827},{"mean":0.822606,"std":0.01094},{"mean":0.783212,"std":0.010795},{"mean":0.746242,"std":0.004199},{"mean":0.706303,"std":0.005446},{"mean":0.664364,"std":0.004252},{"mean":0.618121,"std":0.003203},{"mean":0.574909,"std":0.002622},{"mean":0.525636,"std":0.002622},{"mean":0.479333,"std":0.002877},{"mean":0.432303,"std":0.003413},{"mean":0.386545,"std":0.004299},{"mean":0.335697,"std":0.00165},{"mean":0.286909,"std":0.006052},{"mean":0.235152,"std":0.0097},{"mean":0.172303,"std":0.008419},{"mean":0.103394,"std":0.010313},{"mean":0.000606,"std":0.000278}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.00105},{"mean":0.001616,"std":0.0007},{"mean":0.00303,"std":0.00105},{"mean":0.003394,"std":0.001514},{"mean":0.004242,"std":0.001212},{"mean":0.005714,"std":0.0009},{"mean":0.006667,"std":0.000694},{"mean":0.007138,"std":0.000233},{"mean":0.00897,"std":0.00164},{"mean":0.01135,"std":0.001631},{"mean":0.014848,"std":0.001321},{"mean":0.020979,"std":0.002947},{"mean":0.030909,"std":0.002029},{"mean":0.044687,"std":0.003089},{"mean":0.064394,"std":0.004823},{"mean":0.091052,"std":0.006985},{"mean":0.122761,"std":0.006858},{"mean":0.153493,"std":0.006845},{"mean":0.18497,"std":0.006941}],"operating":{"tau":[{"mean":0.99938,"std":0.000167},{"mean":0.998765,"std":0.000339},{"mean":0.997911,"std":0.000651},{"mean":0.996776,"std":0.001091},{"mean":0.995309,"std":0.001731},{"mean":0.99312,"std":0.002748},{"mean":0.990402,"std":0.003855},{"mean":0.986804,"std":0.004989},{"mean":0.981116,"std":0.007256},{"mean":0.972447,"std":0.009881},{"mean":0.96152,"std":0.013462},{"mean":0.945515,"std":0.019327},{"mean":0.923076,"std":0.028734},{"mean":0.888332,"std":0.035898},{"mean":0.843219,"std":0.045731},{"mean":0.776338,"std":0.044848},{"mean":0.674587,"std":0.047719},{"mean":0.535618,"std":0.038325},{"mean":0.407392,"std":0.019456},{"mean":0.191069,"std":0.008637}],"coverage":[{"mean":0.044242,"std":0.0029},{"mean":0.086727,"std":0.007446},{"mean":0.132788,"std":0.006843},{"mean":0.174121,"std":0.00742},{"mean":0.214303,"std":0.006592},{"mean":0.254848,"std":0.010598},{"mean":0.294061,"std":0.005095},{"mean":0.334485,"std":0.001277},{"mean":0.379091,"std":0.001106},{"mean":0.426909,"std":0.005548},{"mean":0.467939,"std":0.002302},{"mean":0.51097,"std":0.004756},{"mean":0.555455,"std":0.006472},{"mean":0.603515,"std":0.004099},{"mean":0.643697,"std":0.005459},{"mean":0.689091,"std":0.005091},{"mean":0.747152,"std":0.007191},{"mean":0.815091,"std":0.007599},{"mean":0.886242,"std":0.003588},{"mean":0.998667,"std":0.001065}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.001364,"std":0.001189},{"mean":0.001854,"std":0.000901},{"mean":0.002427,"std":0.000518},{"mean":0.003393,"std":0.001661},{"mean":0.003791,"std":0.001576},{"mean":0.004127,"std":0.001302},{"mean":0.005071,"std":0.001235},{"mean":0.006716,"std":0.000841},{"mean":0.006955,"std":0.000171},{"mean":0.007902,"std":0.000485},{"mean":0.009727,"std":0.000836},{"mean":0.01199,"std":0.002071},{"mean":0.015262,"std":0.001685},{"mean":0.019775,"std":0.00249},{"mean":0.028408,"std":0.000831},{"mean":0.043848,"std":0.005593},{"mean":0.072734,"std":0.009649},{"mean":0.113158,"std":0.008689},{"mean":0.184249,"std":0.006348}],"accuracy_8":[{"mean":0.822,"std":0.000481},{"mean":0.824545,"std":0.001311},{"mean":0.827879,"std":0.000639},{"mean":0.830424,"std":0.000933},{"mean":0.833879,"std":0.001365},{"mean":0.837879,"std":0.002482},{"mean":0.840485,"std":0.002554},{"mean":0.844121,"std":0.001821},{"mean":0.848242,"std":0.001718},{"mean":0.853212,"std":0.003},{"mean":0.857758,"std":0.0029},{"mean":0.862182,"std":0.002823},{"mean":0.867576,"std":0.00258},{"mean":0.872424,"std":0.002222},{"mean":0.876909,"std":0.002364},{"mean":0.879636,"std":0.002641},{"mean":0.881212,"std":0.003203},{"mean":0.876242,"std":0.004964},{"mean":0.860788,"std":0.006299},{"mean":0.815576,"std":0.006496}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.001155},{"mean":0.003333,"std":0.001528},{"mean":0.004,"std":0.001732},{"mean":0.006333,"std":0.001528},{"mean":0.010333,"std":0.001528},{"mean":0.011333,"std":0.001155},{"mean":0.014333,"std":0.002309},{"mean":0.019,"std":0.004359},{"mean":0.025333,"std":0.008386},{"mean":0.035,"std":0.006083},{"mean":0.046333,"std":0.008505},{"mean":0.072333,"std":0.00611},{"mean":0.123,"std":0.020075},{"mean":0.229667,"std":0.038423},{"mean":0.386667,"std":0.033975},{"mean":0.763667,"std":0.029023}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001155},{"mean":0.432667,"std":0.001155},{"mean":0.433,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.435,"std":0.001},{"mean":0.436,"std":0.001732},{"mean":0.437,"std":0.002},{"mean":0.437667,"std":0.002082},{"mean":0.438667,"std":0.001155},{"mean":0.444,"std":0.003},{"mean":0.455667,"std":0.006429},{"mean":0.492333,"std":0.015631},{"mean":0.555,"std":0.023643},{"mean":0.766,"std":0.029309}],"llm_call_rate":[{"mean":0.955758,"std":0.0029},{"mean":0.913273,"std":0.007446},{"mean":0.867212,"std":0.006843},{"mean":0.825879,"std":0.00742},{"mean":0.785697,"std":0.006592},{"mean":0.745152,"std":0.010598},{"mean":0.705939,"std":0.005095},{"mean":0.665515,"std":0.001277},{"mean":0.620909,"std":0.001106},{"mean":0.573091,"std":0.005548},{"mean":0.532061,"std":0.002302},{"mean":0.48903,"std":0.004756},{"mean":0.444545,"std":0.006472},{"mean":0.396485,"std":0.004099},{"mean":0.356303,"std":0.005459},{"mean":0.310909,"std":0.005091},{"mean":0.252848,"std":0.007191},{"mean":0.184909,"std":0.007599},{"mean":0.113758,"std":0.003588},{"mean":0.001333,"std":0.001065}]}},"entropy":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.00105},{"mean":0.001616,"std":0.0007},{"mean":0.00303,"std":0.00105},{"mean":0.003394,"std":0.001514},{"mean":0.004242,"std":0.001212},{"mean":0.005714,"std":0.001039},{"mean":0.006667,"std":0.000694},{"mean":0.006869,"std":0.0},{"mean":0.009333,"std":0.000915},{"mean":0.01124,"std":0.001192},{"mean":0.014949,"std":0.002063},{"mean":0.021725,"std":0.002688},{"mean":0.030823,"std":0.001843},{"mean":0.045172,"std":0.003711},{"mean":0.065379,"std":0.005244},{"mean":0.091979,"std":0.007091},{"mean":0.122357,"std":0.006873},{"mean":0.154258,"std":0.006812},{"mean":0.18497,"std":0.006941}],"operating":{"tau":[{"mean":-0.0062,"std":0.0015},{"mean":-0.011444,"std":0.002802},{"mean":-0.018154,"std":0.005053},{"mean":-0.026591,"std":0.007899},{"mean":-0.036902,"std":0.011283},{"mean":-0.051386,"std":0.01736},{"mean":-0.068415,"std":0.023408},{"mean":-0.089413,"std":0.028055},{"mean":-0.12173,"std":0.038621},{"mean":-0.166158,"std":0.049085},{"mean":-0.217167,"std":0.061894},{"mean":-0.28849,"std":0.080742},{"mean":-0.376783,"std":0.106929},{"mean":-0.504695,"std":0.117209},{"mean":-0.657756,"std":0.132768},{"mean":-0.839389,"std":0.11528},{"mean":-1.06621,"std":0.10197},{"mean":-1.328791,"std":0.059269},{"mean":-1.60029,"std":0.041173},{"mean":-2.010273,"std":0.016153}],"coverage":[{"mean":0.044303,"std":0.002956},{"mean":0.08697,"std":0.006924},{"mean":0.132727,"std":0.007767},{"mean":0.174182,"std":0.007216},{"mean":0.214424,"std":0.006102},{"mean":0.254788,"std":0.009999},{"mean":0.294121,"std":0.005555},{"mean":0.333818,"std":0.001443},{"mean":0.379697,"std":0.000278},{"mean":0.426606,"std":0.005909},{"mean":0.466788,"std":0.00164},{"mean":0.510788,"std":0.004099},{"mean":0.554727,"std":0.006309},{"mean":0.603455,"std":0.003059},{"mean":0.645636,"std":0.004367},{"mean":0.689273,"std":0.002446},{"mean":0.743152,"std":0.00444},{"mean":0.803818,"std":0.003946},{"mean":0.881576,"std":0.005302},{"mean":0.998848,"std":0.000458}],"selective_risk":[{"mean":0.001443,"std":0.002499},{"mean":0.001362,"std":0.001186},{"mean":0.00186,"std":0.000919},{"mean":0.002428,"std":0.000532},{"mean":0.002833,"std":0.001001},{"mean":0.00379,"std":0.001569},{"mean":0.003919,"std":0.001445},{"mean":0.0049,"std":0.000928},{"mean":0.006545,"std":0.000736},{"mean":0.00696,"std":0.000168},{"mean":0.007401,"std":0.000673},{"mean":0.009848,"std":0.001025},{"mean":0.011572,"std":0.001829},{"mean":0.015164,"std":0.001955},{"mean":0.021126,"std":0.002303},{"mean":0.027964,"std":0.00213},{"mean":0.043285,"std":0.005455},{"mean":0.067088,"std":0.006202},{"mean":0.110647,"std":0.009856},{"mean":0.184335,"std":0.006612}],"accuracy_8":[{"mean":0.822121,"std":0.000525},{"mean":0.824667,"std":0.001211},{"mean":0.828,"std":0.000833},{"mean":0.830606,"std":0.000639},{"mean":0.834061,"std":0.00129},{"mean":0.837697,"std":0.002662},{"mean":0.840485,"std":0.002302},{"mean":0.844121,"std":0.00163},{"mean":0.848303,"std":0.001944},{"mean":0.853576,"std":0.003093},{"mean":0.858303,"std":0.002753},{"mean":0.862545,"std":0.00246},{"mean":0.867939,"std":0.002222},{"mean":0.872606,"std":0.002401},{"mean":0.876121,"std":0.002917},{"mean":0.879273,"std":0.002641},{"mean":0.881212,"std":0.00382},{"mean":0.877939,"std":0.00447},{"mean":0.862848,"std":0.007214},{"mean":0.815576,"std":0.006711}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001667,"std":0.000577},{"mean":0.002333,"std":0.000577},{"mean":0.003333,"std":0.001528},{"mean":0.004,"std":0.001732},{"mean":0.006,"std":0.001},{"mean":0.01,"std":0.001},{"mean":0.011333,"std":0.001155},{"mean":0.013,"std":0.001},{"mean":0.018667,"std":0.004726},{"mean":0.024,"std":0.00781},{"mean":0.033667,"std":0.005508},{"mean":0.046,"std":0.007},{"mean":0.063,"std":0.005196},{"mean":0.107,"std":0.018083},{"mean":0.188,"std":0.020952},{"mean":0.364333,"std":0.039311},{"mean":0.764,"std":0.029547}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432667,"std":0.001155},{"mean":0.432667,"std":0.001155},{"mean":0.433,"std":0.001},{"mean":0.433667,"std":0.001155},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.436333,"std":0.002309},{"mean":0.436667,"std":0.002082},{"mean":0.437667,"std":0.002082},{"mean":0.439,"std":0.002646},{"mean":0.441333,"std":0.001528},{"mean":0.449333,"std":0.005508},{"mean":0.474,"std":0.01044},{"mean":0.541,"std":0.022605},{"mean":0.765667,"std":0.02967}],"llm_call_rate":[{"mean":0.955697,"std":0.002956},{"mean":0.91303,"std":0.006924},{"mean":0.867273,"std":0.007767},{"mean":0.825818,"std":0.007216},{"mean":0.785576,"std":0.006102},{"mean":0.745212,"std":0.009999},{"mean":0.705879,"std":0.005555},{"mean":0.666182,"std":0.001443},{"mean":0.620303,"std":0.000278},{"mean":0.573394,"std":0.005909},{"mean":0.533212,"std":0.00164},{"mean":0.489212,"std":0.004099},{"mean":0.445273,"std":0.006309},{"mean":0.396545,"std":0.003059},{"mean":0.354364,"std":0.004367},{"mean":0.310727,"std":0.002446},{"mean":0.256848,"std":0.00444},{"mean":0.196182,"std":0.003946},{"mean":0.118424,"std":0.005302},{"mean":0.001152,"std":0.000458}]}},"margin":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.001212,"std":0.00105},{"mean":0.002424,"std":0.001212},{"mean":0.00303,"std":0.001389},{"mean":0.004364,"std":0.002182},{"mean":0.004646,"std":0.001948},{"mean":0.006407,"std":0.000794},{"mean":0.006818,"std":0.000909},{"mean":0.007677,"std":0.001069},{"mean":0.009818,"std":0.001091},{"mean":0.011791,"std":0.00101},{"mean":0.016061,"std":0.001603},{"mean":0.023403,"std":0.003919},{"mean":0.03368,"std":0.003289},{"mean":0.048889,"std":0.004923},{"mean":0.069848,"std":0.006462},{"mean":0.095258,"std":0.007198},{"mean":0.12431,"std":0.007796},{"mean":0.155343,"std":0.00648},{"mean":0.18497,"std":0.006941}],"operating":{"tau":[{"mean":9.671165,"std":0.258516},{"mean":8.882638,"std":0.253589},{"mean":8.272679,"std":0.286815},{"mean":7.775194,"std":0.294897},{"mean":7.334016,"std":0.328573},{"mean":6.905433,"std":0.340159},{"mean":6.496926,"std":0.349602},{"mean":6.07213,"std":0.339559},{"mean":5.64569,"std":0.362304},{"mean":5.202521,"std":0.335312},{"mean":4.806483,"std":0.33025},{"mean":4.365012,"std":0.348139},{"mean":3.930258,"std":0.348828},{"mean":3.440099,"std":0.326884},{"mean":2.955575,"std":0.289661},{"mean":2.414929,"std":0.246282},{"mean":1.782918,"std":0.229037},{"mean":1.141187,"std":0.174175},{"mean":0.517509,"std":0.06866},{"mean":0.002456,"std":0.002018}],"coverage":[{"mean":0.046667,"std":0.003066},{"mean":0.088121,"std":0.006336},{"mean":0.13297,"std":0.005733},{"mean":0.175091,"std":0.001845},{"mean":0.214364,"std":0.005042},{"mean":0.252121,"std":0.003951},{"mean":0.292788,"std":0.00502},{"mean":0.338485,"std":0.002184},{"mean":0.385576,"std":0.000555},{"mean":0.429152,"std":0.003109},{"mean":0.472485,"std":0.001155},{"mean":0.515152,"std":0.002819},{"mean":0.55703,"std":0.004388},{"mean":0.602606,"std":0.005431},{"mean":0.645515,"std":0.009615},{"mean":0.694848,"std":0.007404},{"mean":0.752061,"std":0.00864},{"mean":0.821697,"std":0.012776},{"mean":0.905576,"std":0.010745},{"mean":0.999394,"std":0.000378}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.001374,"std":0.001199},{"mean":0.001808,"std":0.000717},{"mean":0.002082,"std":0.001058},{"mean":0.003104,"std":0.001729},{"mean":0.004332,"std":0.002158},{"mean":0.004148,"std":0.001578},{"mean":0.006448,"std":0.001098},{"mean":0.006601,"std":0.000465},{"mean":0.007488,"std":0.000924},{"mean":0.008724,"std":0.001369},{"mean":0.010002,"std":0.000926},{"mean":0.012509,"std":0.001318},{"mean":0.016301,"std":0.001502},{"mean":0.023015,"std":0.002437},{"mean":0.032772,"std":0.004323},{"mean":0.049985,"std":0.008078},{"mean":0.080488,"std":0.012452},{"mean":0.127481,"std":0.0124},{"mean":0.184656,"std":0.006979}],"accuracy_8":[{"mean":0.821879,"std":0.000584},{"mean":0.823758,"std":0.001183},{"mean":0.826788,"std":0.000897},{"mean":0.830061,"std":0.000458},{"mean":0.83297,"std":0.00082},{"mean":0.836364,"std":0.001666},{"mean":0.839515,"std":0.001821},{"mean":0.843758,"std":0.002092},{"mean":0.848545,"std":0.002565},{"mean":0.852242,"std":0.002455},{"mean":0.856909,"std":0.003059},{"mean":0.861636,"std":0.002545},{"mean":0.866848,"std":0.002939},{"mean":0.872606,"std":0.003546},{"mean":0.876545,"std":0.003818},{"mean":0.879455,"std":0.00246},{"mean":0.878545,"std":0.003713},{"mean":0.870848,"std":0.007727},{"mean":0.851576,"std":0.009086},{"mean":0.815212,"std":0.00677}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.001},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.001155},{"mean":0.004333,"std":0.001528},{"mean":0.004667,"std":0.001155},{"mean":0.009333,"std":0.002082},{"mean":0.010333,"std":0.000577},{"mean":0.013,"std":0.002646},{"mean":0.017,"std":0.004583},{"mean":0.021333,"std":0.004041},{"mean":0.028,"std":0.004359},{"mean":0.039,"std":0.006},{"mean":0.059667,"std":0.009292},{"mean":0.093333,"std":0.019035},{"mean":0.154333,"std":0.03371},{"mean":0.273667,"std":0.047522},{"mean":0.480667,"std":0.049541},{"mean":0.767333,"std":0.032347}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001155},{"mean":0.432667,"std":0.001155},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.435333,"std":0.001528},{"mean":0.436667,"std":0.000577},{"mean":0.437333,"std":0.001155},{"mean":0.436667,"std":0.000577},{"mean":0.441667,"std":0.004726},{"mean":0.450333,"std":0.006028},{"mean":0.470333,"std":0.011372},{"mean":0.518,"std":0.026851},{"mean":0.609,"std":0.034044},{"mean":0.767667,"std":0.031786}],"llm_call_rate":[{"mean":0.953333,"std":0.003066},{"mean":0.911879,"std":0.006336},{"mean":0.86703,"std":0.005733},{"mean":0.824909,"std":0.001845},{"mean":0.785636,"std":0.005042},{"mean":0.747879,"std":0.003951},{"mean":0.707212,"std":0.00502},{"mean":0.661515,"std":0.002184},{"mean":0.614424,"std":0.000555},{"mean":0.570848,"std":0.003109},{"mean":0.527515,"std":0.001155},{"mean":0.484848,"std":0.002819},{"mean":0.44297,"std":0.004388},{"mean":0.397394,"std":0.005431},{"mean":0.354485,"std":0.009615},{"mean":0.305152,"std":0.007404},{"mean":0.247939,"std":0.00864},{"mean":0.178303,"std":0.012776},{"mean":0.094424,"std":0.010745},{"mean":0.000606,"std":0.000378}]}},"msp_t":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.00105},{"mean":0.001616,"std":0.0007},{"mean":0.00303,"std":0.00105},{"mean":0.003636,"std":0.001455},{"mean":0.004242,"std":0.00105},{"mean":0.005887,"std":0.0012},{"mean":0.006667,"std":0.000694},{"mean":0.007138,"std":0.000233},{"mean":0.008848,"std":0.00164},{"mean":0.011129,"std":0.001252},{"mean":0.015152,"std":0.001093},{"mean":0.021445,"std":0.002816},{"mean":0.031602,"std":0.003227},{"mean":0.044768,"std":0.003482},{"mean":0.064621,"std":0.004553},{"mean":0.090553,"std":0.007179},{"mean":0.12101,"std":0.006788},{"mean":0.151834,"std":0.005912},{"mean":0.18497,"std":0.006941}],"operating":{"tau":[{"mean":0.999834,"std":3.1e-05},{"mean":0.999628,"std":8.8e-05},{"mean":0.999339,"std":0.000151},{"mean":0.99893,"std":0.000263},{"mean":0.998323,"std":0.000471},{"mean":0.997444,"std":0.000824},{"mean":0.996267,"std":0.00117},{"mean":0.994511,"std":0.001673},{"mean":0.991647,"std":0.002785},{"mean":0.987248,"std":0.003938},{"mean":0.980769,"std":0.006117},{"mean":0.971307,"std":0.009365},{"mean":0.956517,"std":0.015223},{"mean":0.931365,"std":0.020854},{"mean":0.896993,"std":0.028888},{"mean":0.839382,"std":0.031727},{"mean":0.740923,"std":0.038772},{"mean":0.593846,"std":0.035113},{"mean":0.445537,"std":0.01729},{"mean":0.196053,"std":0.017527}],"coverage":[{"mean":0.044242,"std":0.003494},{"mean":0.087939,"std":0.007341},{"mean":0.132424,"std":0.00648},{"mean":0.173636,"std":0.007098},{"mean":0.21497,"std":0.005853},{"mean":0.253758,"std":0.010313},{"mean":0.293515,"std":0.004462},{"mean":0.335758,"std":0.000555},{"mean":0.380848,"std":0.002649},{"mean":0.427576,"std":0.005699},{"mean":0.470545,"std":0.001488},{"mean":0.511758,"std":0.004234},{"mean":0.556909,"std":0.006623},{"mean":0.604,"std":0.001791},{"mean":0.642485,"std":0.006833},{"mean":0.689273,"std":0.007086},{"mean":0.748121,"std":0.006069},{"mean":0.815818,"std":0.007875},{"mean":0.889818,"std":0.004639},{"mean":0.99903,"std":0.001377}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.001346,"std":0.001173},{"mean":0.001858,"std":0.000898},{"mean":0.002432,"std":0.000512},{"mean":0.003376,"std":0.001642},{"mean":0.003809,"std":0.001588},{"mean":0.004126,"std":0.001262},{"mean":0.005595,"std":0.000818},{"mean":0.006686,"std":0.000522},{"mean":0.006945,"std":0.000181},{"mean":0.008373,"std":0.000982},{"mean":0.009473,"std":0.001323},{"mean":0.012069,"std":0.001704},{"mean":0.015252,"std":0.001147},{"mean":0.020007,"std":0.001757},{"mean":0.029104,"std":0.001201},{"mean":0.043958,"std":0.005752},{"mean":0.071921,"std":0.010353},{"mean":0.114662,"std":0.01034},{"mean":0.184424,"std":0.006473}],"accuracy_8":[{"mean":0.821939,"std":0.000525},{"mean":0.824606,"std":0.001365},{"mean":0.827636,"std":0.000481},{"mean":0.830424,"std":0.000897},{"mean":0.834,"std":0.001192},{"mean":0.837455,"std":0.002204},{"mean":0.840303,"std":0.002735},{"mean":0.844,"std":0.001924},{"mean":0.848485,"std":0.001568},{"mean":0.853333,"std":0.003077},{"mean":0.857879,"std":0.003109},{"mean":0.862727,"std":0.002823},{"mean":0.868,"std":0.002823},{"mean":0.872848,"std":0.002373},{"mean":0.876909,"std":0.002278},{"mean":0.879636,"std":0.002641},{"mean":0.881333,"std":0.004187},{"mean":0.876788,"std":0.005976},{"mean":0.859515,"std":0.007273},{"mean":0.815394,"std":0.006549}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.001155},{"mean":0.003333,"std":0.001528},{"mean":0.004,"std":0.001732},{"mean":0.007333,"std":0.001528},{"mean":0.010333,"std":0.000577},{"mean":0.011333,"std":0.001155},{"mean":0.015667,"std":0.003786},{"mean":0.019333,"std":0.004933},{"mean":0.025667,"std":0.007234},{"mean":0.035,"std":0.005196},{"mean":0.047667,"std":0.007638},{"mean":0.074667,"std":0.007024},{"mean":0.125,"std":0.021932},{"mean":0.228,"std":0.03995},{"mean":0.394667,"std":0.041789},{"mean":0.765,"std":0.031193}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001155},{"mean":0.432667,"std":0.001155},{"mean":0.433333,"std":0.001528},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.435667,"std":0.002082},{"mean":0.435667,"std":0.001155},{"mean":0.437,"std":0.002},{"mean":0.437667,"std":0.002082},{"mean":0.439,"std":0.001},{"mean":0.445333,"std":0.002517},{"mean":0.456667,"std":0.007638},{"mean":0.490667,"std":0.020133},{"mean":0.559,"std":0.026963},{"mean":0.767,"std":0.031097}],"llm_call_rate":[{"mean":0.955758,"std":0.003494},{"mean":0.912061,"std":0.007341},{"mean":0.867576,"std":0.00648},{"mean":0.826364,"std":0.007098},{"mean":0.78503,"std":0.005853},{"mean":0.746242,"std":0.010313},{"mean":0.706485,"std":0.004462},{"mean":0.664242,"std":0.000555},{"mean":0.619152,"std":0.002649},{"mean":0.572424,"std":0.005699},{"mean":0.529455,"std":0.001488},{"mean":0.488242,"std":0.004234},{"mean":0.443091,"std":0.006623},{"mean":0.396,"std":0.001791},{"mean":0.357515,"std":0.006833},{"mean":0.310727,"std":0.007086},{"mean":0.251879,"std":0.006069},{"mean":0.184182,"std":0.007875},{"mean":0.110182,"std":0.004639},{"mean":0.00097,"std":0.001377}]}}}}},"modernbert/k100":{"model":"modernbert","k":100,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["ModernBERT-base-k100-seed42","ModernBERT-base-k100-seed43","ModernBERT-base-k100-seed44"],"logits_sha256":["208f372ab9096fa4f95a006328c8ef766cb65144005d9df8f4bad1c391a100a9","58c3bd83838784f23d7e9977921b4a021797799b9a514b3c4cd449c641cd5257","37ebd645a97a654308f9f50cf4501b6b1c5bfed5bbd064fe591d8587a4873966"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.00105},{"mean":0.001212,"std":0.0},{"mean":0.001212,"std":0.000525},{"mean":0.001455,"std":0.000727},{"mean":0.001616,"std":0.00035},{"mean":0.001732,"std":0.000794},{"mean":0.00197,"std":0.000262},{"mean":0.00202,"std":0.0},{"mean":0.002182,"std":0.0},{"mean":0.002314,"std":0.0},{"mean":0.00303,"std":0.000802},{"mean":0.003636,"std":0.000559},{"mean":0.004589,"std":0.00075},{"mean":0.007596,"std":0.000851},{"mean":0.012045,"std":0.000787},{"mean":0.020463,"std":0.000751},{"mean":0.03367,"std":0.002103},{"mean":0.053206,"std":0.000331},{"mean":0.081273,"std":0.001273}],"operating":{"tau":[{"mean":0.99999,"std":1e-06},{"mean":0.999984,"std":0.0},{"mean":0.999977,"std":0.0},{"mean":0.99997,"std":1e-06},{"mean":0.999961,"std":1e-06},{"mean":0.999951,"std":1e-06},{"mean":0.999937,"std":3e-06},{"mean":0.99992,"std":5e-06},{"mean":0.999895,"std":4e-06},{"mean":0.999865,"std":7e-06},{"mean":0.999821,"std":9e-06},{"mean":0.999759,"std":1.6e-05},{"mean":0.99966,"std":1.9e-05},{"mean":0.999499,"std":2.5e-05},{"mean":0.999139,"std":2.9e-05},{"mean":0.998262,"std":0.000126},{"mean":0.995689,"std":0.000496},{"mean":0.982537,"std":0.002362},{"mean":0.8745,"std":0.008948},{"mean":0.180987,"std":0.017005}],"coverage":[{"mean":0.042,"std":0.005773},{"mean":0.086061,"std":0.003161},{"mean":0.128,"std":0.003511},{"mean":0.173333,"std":0.003135},{"mean":0.213818,"std":0.005715},{"mean":0.255576,"std":0.004853},{"mean":0.300061,"std":0.003413},{"mean":0.341152,"std":0.00348},{"mean":0.386485,"std":0.004433},{"mean":0.428,"std":0.00732},{"mean":0.467091,"std":0.007202},{"mean":0.505576,"std":0.00447},{"mean":0.545758,"std":0.003737},{"mean":0.585636,"std":0.004299},{"mean":0.631758,"std":0.006151},{"mean":0.682,"std":0.00142},{"mean":0.731394,"std":0.002842},{"mean":0.795455,"std":0.003549},{"mean":0.871879,"std":0.003679},{"mean":0.996667,"std":0.001728}],"selective_risk":[{"mean":0.001522,"std":0.002636},{"mean":0.00139,"std":0.001205},{"mean":0.000936,"std":0.000811},{"mean":0.0014,"std":0.000611},{"mean":0.001136,"std":0.000497},{"mean":0.00142,"std":0.000708},{"mean":0.001618,"std":0.000362},{"mean":0.001778,"std":0.000815},{"mean":0.002041,"std":0.000297},{"mean":0.001982,"std":0.000242},{"mean":0.002206,"std":0.000222},{"mean":0.002158,"std":1.9e-05},{"mean":0.002332,"std":1.6e-05},{"mean":0.003003,"std":0.000659},{"mean":0.003362,"std":0.000754},{"mean":0.004443,"std":0.001006},{"mean":0.00613,"std":0.000859},{"mean":0.01158,"std":0.000905},{"mean":0.025234,"std":0.002076},{"mean":0.078807,"std":0.001523}],"accuracy_8":[{"mean":0.824364,"std":0.001443},{"mean":0.828788,"std":0.001689},{"mean":0.833576,"std":0.001315},{"mean":0.838,"std":0.000727},{"mean":0.841818,"std":0.000545},{"mean":0.845818,"std":0.0},{"mean":0.851636,"std":0.001192},{"mean":0.856727,"std":0.000793},{"mean":0.862242,"std":0.00082},{"mean":0.866727,"std":0.000481},{"mean":0.870788,"std":0.000859},{"mean":0.875697,"std":0.001238},{"mean":0.880303,"std":0.00164},{"mean":0.885394,"std":0.000584},{"mean":0.890788,"std":0.00134},{"mean":0.896242,"std":0.002273},{"mean":0.903758,"std":0.002482},{"mean":0.91297,"std":0.003119},{"mean":0.922242,"std":0.002027},{"mean":0.919697,"std":0.001389}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.001155},{"mean":0.002,"std":0.0},{"mean":0.002333,"std":0.000577},{"mean":0.003333,"std":0.000577},{"mean":0.003667,"std":0.001155},{"mean":0.004333,"std":0.000577},{"mean":0.004667,"std":0.000577},{"mean":0.005333,"std":0.000577},{"mean":0.007333,"std":0.001528},{"mean":0.009,"std":0.002646},{"mean":0.013,"std":0.002646},{"mean":0.019667,"std":0.002309},{"mean":0.043333,"std":0.004041},{"mean":0.1,"std":0.00781},{"mean":0.374667,"std":0.009074}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.431333,"std":0.001155},{"mean":0.431,"std":0.001732},{"mean":0.430333,"std":0.003055},{"mean":0.430667,"std":0.002309},{"mean":0.429667,"std":0.002517},{"mean":0.429,"std":0.001732},{"mean":0.428333,"std":0.002887},{"mean":0.427,"std":0.003464},{"mean":0.425,"std":0.001732},{"mean":0.423667,"std":0.001528},{"mean":0.421333,"std":0.002309},{"mean":0.412333,"std":0.005859},{"mean":0.396,"std":0.008544},{"mean":0.365333,"std":0.006658},{"mean":0.384,"std":0.005292}],"llm_call_rate":[{"mean":0.958,"std":0.005773},{"mean":0.913939,"std":0.003161},{"mean":0.872,"std":0.003511},{"mean":0.826667,"std":0.003135},{"mean":0.786182,"std":0.005715},{"mean":0.744424,"std":0.004853},{"mean":0.699939,"std":0.003413},{"mean":0.658848,"std":0.00348},{"mean":0.613515,"std":0.004433},{"mean":0.572,"std":0.00732},{"mean":0.532909,"std":0.007202},{"mean":0.494424,"std":0.00447},{"mean":0.454242,"std":0.003737},{"mean":0.414364,"std":0.004299},{"mean":0.368242,"std":0.006151},{"mean":0.318,"std":0.00142},{"mean":0.268606,"std":0.002842},{"mean":0.204545,"std":0.003549},{"mean":0.128121,"std":0.003679},{"mean":0.003333,"std":0.001728}]}},"entropy":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.00105},{"mean":0.001212,"std":0.0},{"mean":0.001212,"std":0.000525},{"mean":0.001212,"std":0.00084},{"mean":0.001616,"std":0.00035},{"mean":0.001905,"std":0.0006},{"mean":0.00197,"std":0.000262},{"mean":0.002155,"std":0.000233},{"mean":0.002182,"std":0.0},{"mean":0.002314,"std":0.0},{"mean":0.002929,"std":0.000631},{"mean":0.00373,"std":0.000427},{"mean":0.004502,"std":0.000912},{"mean":0.007354,"std":0.00061},{"mean":0.011818,"std":0.000819},{"mean":0.01975,"std":0.000864},{"mean":0.032593,"std":0.003209},{"mean":0.053907,"std":0.001274},{"mean":0.081273,"std":0.001273}],"operating":{"tau":[{"mean":-0.000156,"std":1e-05},{"mean":-0.000244,"std":7e-06},{"mean":-0.000333,"std":6e-06},{"mean":-0.000431,"std":1.4e-05},{"mean":-0.000546,"std":1.5e-05},{"mean":-0.000675,"std":1.9e-05},{"mean":-0.000848,"std":4.8e-05},{"mean":-0.001053,"std":5.6e-05},{"mean":-0.001346,"std":5.6e-05},{"mean":-0.001694,"std":0.0001},{"mean":-0.002164,"std":0.000105},{"mean":-0.002858,"std":0.000182},{"mean":-0.003903,"std":0.000155},{"mean":-0.005389,"std":0.00021},{"mean":-0.008749,"std":0.000269},{"mean":-0.016495,"std":0.000859},{"mean":-0.03488,"std":0.002738},{"mean":-0.117994,"std":0.008833},{"mean":-0.547546,"std":0.064317},{"mean":-3.23616,"std":0.119982}],"coverage":[{"mean":0.041879,"std":0.006325},{"mean":0.085576,"std":0.003951},{"mean":0.127212,"std":0.002729},{"mean":0.171152,"std":0.003611},{"mean":0.213879,"std":0.005},{"mean":0.255515,"std":0.004686},{"mean":0.300303,"std":0.005339},{"mean":0.341212,"std":0.002154},{"mean":0.387152,"std":0.004425},{"mean":0.427758,"std":0.005842},{"mean":0.466,"std":0.00835},{"mean":0.507152,"std":0.002154},{"mean":0.547697,"std":0.003066},{"mean":0.585273,"std":0.004442},{"mean":0.630424,"std":0.005987},{"mean":0.681576,"std":0.002783},{"mean":0.729152,"std":0.000584},{"mean":0.794545,"std":0.000962},{"mean":0.868667,"std":0.006129},{"mean":0.996545,"std":0.002691}],"selective_risk":[{"mean":0.001543,"std":0.002673},{"mean":0.001389,"std":0.001204},{"mean":0.000947,"std":0.000821},{"mean":0.001422,"std":0.000637},{"mean":0.001134,"std":0.000491},{"mean":0.00142,"std":0.000709},{"mean":0.001617,"std":0.000366},{"mean":0.001953,"std":0.000609},{"mean":0.002037,"std":0.000295},{"mean":0.001986,"std":0.00027},{"mean":0.002211,"std":0.000217},{"mean":0.002151,"std":9e-06},{"mean":0.002324,"std":1.3e-05},{"mean":0.002798,"std":0.000552},{"mean":0.003271,"std":0.000467},{"mean":0.004269,"std":0.00093},{"mean":0.005819,"std":0.000524},{"mean":0.011212,"std":0.000814},{"mean":0.023785,"std":0.002182},{"mean":0.078692,"std":0.002518}],"accuracy_8":[{"mean":0.824424,"std":0.001377},{"mean":0.828727,"std":0.001753},{"mean":0.83297,"std":0.001238},{"mean":0.837515,"std":0.000757},{"mean":0.841939,"std":0.000735},{"mean":0.845818,"std":0.000481},{"mean":0.851879,"std":0.00165},{"mean":0.856242,"std":0.001001},{"mean":0.862485,"std":0.000378},{"mean":0.866667,"std":0.00021},{"mean":0.871091,"std":0.001106},{"mean":0.875697,"std":0.000915},{"mean":0.880606,"std":0.001389},{"mean":0.88497,"std":0.001155},{"mean":0.890485,"std":0.001111},{"mean":0.896242,"std":0.002259},{"mean":0.903394,"std":0.002939},{"mean":0.912545,"std":0.001924},{"mean":0.920727,"std":0.001585},{"mean":0.92,"std":0.002025}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.001155},{"mean":0.002,"std":0.0},{"mean":0.002667,"std":0.000577},{"mean":0.003333,"std":0.000577},{"mean":0.003667,"std":0.000577},{"mean":0.004333,"std":0.000577},{"mean":0.004667,"std":0.000577},{"mean":0.005333,"std":0.000577},{"mean":0.007,"std":0.001732},{"mean":0.008667,"std":0.001528},{"mean":0.012667,"std":0.002887},{"mean":0.018667,"std":0.002082},{"mean":0.042,"std":0.004359},{"mean":0.092667,"std":0.006429},{"mean":0.374333,"std":0.014224}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.431333,"std":0.001155},{"mean":0.431,"std":0.001732},{"mean":0.431333,"std":0.001528},{"mean":0.430667,"std":0.002309},{"mean":0.43,"std":0.002},{"mean":0.429333,"std":0.002082},{"mean":0.428333,"std":0.002887},{"mean":0.427333,"std":0.002887},{"mean":0.425333,"std":0.002082},{"mean":0.423667,"std":0.002309},{"mean":0.421667,"std":0.002517},{"mean":0.414,"std":0.008185},{"mean":0.398333,"std":0.007638},{"mean":0.374,"std":0.004},{"mean":0.382667,"std":0.008386}],"llm_call_rate":[{"mean":0.958121,"std":0.006325},{"mean":0.914424,"std":0.003951},{"mean":0.872788,"std":0.002729},{"mean":0.828848,"std":0.003611},{"mean":0.786121,"std":0.005},{"mean":0.744485,"std":0.004686},{"mean":0.699697,"std":0.005339},{"mean":0.658788,"std":0.002154},{"mean":0.612848,"std":0.004425},{"mean":0.572242,"std":0.005842},{"mean":0.534,"std":0.00835},{"mean":0.492848,"std":0.002154},{"mean":0.452303,"std":0.003066},{"mean":0.414727,"std":0.004442},{"mean":0.369576,"std":0.005987},{"mean":0.318424,"std":0.002783},{"mean":0.270848,"std":0.000584},{"mean":0.205455,"std":0.000962},{"mean":0.131333,"std":0.006129},{"mean":0.003455,"std":0.002691}]}},"margin":{"risk_coverage":[{"mean":0.002424,"std":0.002099},{"mean":0.001212,"std":0.00105},{"mean":0.001212,"std":0.0},{"mean":0.001515,"std":0.000525},{"mean":0.001455,"std":0.0},{"mean":0.001818,"std":0.000606},{"mean":0.001905,"std":0.0003},{"mean":0.001818,"std":0.0},{"mean":0.001886,"std":0.000233},{"mean":0.002667,"std":0.00042},{"mean":0.002975,"std":0.000661},{"mean":0.003333,"std":0.000802},{"mean":0.004103,"std":0.000982},{"mean":0.005887,"std":0.001227},{"mean":0.008485,"std":0.000874},{"mean":0.01303,"std":0.001074},{"mean":0.021105,"std":0.001716},{"mean":0.035354,"std":0.002128},{"mean":0.05614,"std":0.001121},{"mean":0.081273,"std":0.001273}],"operating":{"tau":[{"mean":10.471047,"std":0.069915},{"mean":9.966643,"std":0.057864},{"mean":9.659126,"std":0.03488},{"mean":9.369748,"std":0.028305},{"mean":9.138716,"std":0.027209},{"mean":8.908479,"std":0.033602},{"mean":8.678286,"std":0.032998},{"mean":8.466839,"std":0.033754},{"mean":8.246209,"std":0.018094},{"mean":7.997813,"std":0.000784},{"mean":7.750293,"std":0.015332},{"mean":7.45269,"std":0.046111},{"mean":7.135095,"std":0.02428},{"mean":6.757185,"std":0.040112},{"mean":6.285401,"std":0.051382},{"mean":5.707079,"std":0.065584},{"mean":4.880505,"std":0.160163},{"mean":3.753247,"std":0.153165},{"mean":2.01247,"std":0.123932},{"mean":0.005776,"std":0.005131}],"coverage":[{"mean":0.04103,"std":0.003109},{"mean":0.089818,"std":0.005421},{"mean":0.131879,"std":0.007221},{"mean":0.176121,"std":0.006876},{"mean":0.21503,"std":0.006058},{"mean":0.257758,"std":0.008788},{"mean":0.302182,"std":0.009418},{"mean":0.34,"std":0.010489},{"mean":0.379697,"std":0.006299},{"mean":0.423333,"std":0.002777},{"mean":0.460545,"std":0.004287},{"mean":0.504,"std":0.00682},{"mean":0.546909,"std":0.005818},{"mean":0.589394,"std":0.009018},{"mean":0.635273,"std":0.005724},{"mean":0.682909,"std":0.003831},{"mean":0.738121,"std":0.008406},{"mean":0.801152,"std":0.005157},{"mean":0.881576,"std":0.007386},{"mean":0.999576,"std":0.000458}],"selective_risk":[{"mean":0.002831,"std":0.002452},{"mean":0.001308,"std":0.001133},{"mean":0.001382,"std":7.8e-05},{"mean":0.00139,"std":0.000651},{"mean":0.001417,"std":0.000514},{"mean":0.001643,"std":0.000381},{"mean":0.001808,"std":0.000603},{"mean":0.001782,"std":0.000297},{"mean":0.001916,"std":3.2e-05},{"mean":0.001718,"std":1.1e-05},{"mean":0.002107,"std":0.000244},{"mean":0.002646,"std":0.000419},{"mean":0.002995,"std":0.000678},{"mean":0.003296,"std":0.000803},{"mean":0.003913,"std":0.000731},{"mean":0.00497,"std":0.000771},{"mean":0.0073,"std":0.001455},{"mean":0.013234,"std":0.001218},{"mean":0.028934,"std":0.002846},{"mean":0.081125,"std":0.00141}],"accuracy_8":[{"mean":0.824242,"std":0.000525},{"mean":0.829879,"std":0.000735},{"mean":0.834364,"std":0.001106},{"mean":0.839576,"std":0.00165},{"mean":0.842909,"std":0.001012},{"mean":0.848061,"std":0.002011},{"mean":0.852364,"std":0.002727},{"mean":0.856242,"std":0.002068},{"mean":0.86103,"std":0.001747},{"mean":0.865939,"std":0.001389},{"mean":0.870424,"std":0.001412},{"mean":0.875879,"std":0.001589},{"mean":0.881394,"std":0.00163},{"mean":0.886788,"std":0.001377},{"mean":0.891818,"std":0.002065},{"mean":0.898121,"std":0.002945},{"mean":0.905394,"std":0.002939},{"mean":0.914303,"std":0.002123},{"mean":0.923939,"std":0.00268},{"mean":0.918788,"std":0.001315}],"high_conf_oos_misroute_rate":[{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001333,"std":0.000577},{"mean":0.002,"std":0.001},{"mean":0.002333,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.003,"std":0.001},{"mean":0.003,"std":0.001},{"mean":0.004,"std":0.001},{"mean":0.005333,"std":0.001155},{"mean":0.006667,"std":0.001528},{"mean":0.008333,"std":0.002082},{"mean":0.011,"std":0.001732},{"mean":0.014667,"std":0.001155},{"mean":0.023667,"std":0.005132},{"mean":0.049,"std":0.003606},{"mean":0.117,"std":0.011},{"mean":0.388667,"std":0.004041}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.431667,"std":0.000577},{"mean":0.431333,"std":0.001155},{"mean":0.430667,"std":0.002309},{"mean":0.430333,"std":0.002887},{"mean":0.430667,"std":0.003215},{"mean":0.43,"std":0.003464},{"mean":0.43,"std":0.003464},{"mean":0.429333,"std":0.003786},{"mean":0.428333,"std":0.003786},{"mean":0.427333,"std":0.003786},{"mean":0.426333,"std":0.003786},{"mean":0.424667,"std":0.003215},{"mean":0.423,"std":0.002646},{"mean":0.420333,"std":0.003055},{"mean":0.415,"std":0.004583},{"mean":0.406333,"std":0.008505},{"mean":0.386667,"std":0.003512},{"mean":0.355,"std":0.00866},{"mean":0.388667,"std":0.004041}],"llm_call_rate":[{"mean":0.95897,"std":0.003109},{"mean":0.910182,"std":0.005421},{"mean":0.868121,"std":0.007221},{"mean":0.823879,"std":0.006876},{"mean":0.78497,"std":0.006058},{"mean":0.742242,"std":0.008788},{"mean":0.697818,"std":0.009418},{"mean":0.66,"std":0.010489},{"mean":0.620303,"std":0.006299},{"mean":0.576667,"std":0.002777},{"mean":0.539455,"std":0.004287},{"mean":0.496,"std":0.00682},{"mean":0.453091,"std":0.005818},{"mean":0.410606,"std":0.009018},{"mean":0.364727,"std":0.005724},{"mean":0.317091,"std":0.003831},{"mean":0.261879,"std":0.008406},{"mean":0.198848,"std":0.005157},{"mean":0.118424,"std":0.007386},{"mean":0.000424,"std":0.000458}]}},"msp_t":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.00105},{"mean":0.001616,"std":0.0007},{"mean":0.001212,"std":0.000525},{"mean":0.001455,"std":0.000727},{"mean":0.001818,"std":0.0},{"mean":0.001905,"std":0.0006},{"mean":0.002121,"std":0.000262},{"mean":0.00202,"std":0.0},{"mean":0.002182,"std":0.0},{"mean":0.002314,"std":0.0},{"mean":0.002929,"std":0.000631},{"mean":0.00373,"std":0.000427},{"mean":0.004675,"std":0.000937},{"mean":0.007354,"std":0.000505},{"mean":0.011742,"std":0.00086},{"mean":0.01975,"std":0.000653},{"mean":0.033468,"std":0.002944},{"mean":0.052951,"std":0.001245},{"mean":0.081273,"std":0.001273}],"operating":{"tau":[{"mean":0.999798,"std":2.5e-05},{"mean":0.999708,"std":1.9e-05},{"mean":0.999622,"std":1.9e-05},{"mean":0.999529,"std":1.6e-05},{"mean":0.999433,"std":2.3e-05},{"mean":0.999314,"std":2.6e-05},{"mean":0.99917,"std":2e-05},{"mean":0.998997,"std":3.5e-05},{"mean":0.998775,"std":2.2e-05},{"mean":0.998515,"std":2.5e-05},{"mean":0.998172,"std":4.2e-05},{"mean":0.997685,"std":1.1e-05},{"mean":0.997005,"std":6.7e-05},{"mean":0.99601,"std":4e-05},{"mean":0.993946,"std":0.000155},{"mean":0.989723,"std":0.000335},{"mean":0.979837,"std":0.001046},{"mean":0.941303,"std":0.006641},{"mean":0.763764,"std":0.020064},{"mean":0.133238,"std":0.010698}],"coverage":[{"mean":0.042909,"std":0.005827},{"mean":0.085455,"std":0.003186},{"mean":0.126788,"std":0.001857},{"mean":0.170545,"std":0.001753},{"mean":0.210303,"std":0.003135},{"mean":0.255515,"std":0.00401},{"mean":0.299879,"std":0.004131},{"mean":0.343091,"std":0.001636},{"mean":0.388242,"std":0.002649},{"mean":0.428667,"std":0.006015},{"mean":0.467333,"std":0.010413},{"mean":0.508242,"std":0.003728},{"mean":0.548424,"std":0.003093},{"mean":0.586727,"std":0.002678},{"mean":0.631515,"std":0.006881},{"mean":0.681515,"std":0.002783},{"mean":0.730909,"std":0.001311},{"mean":0.793515,"std":0.003374},{"mean":0.868485,"std":0.005954},{"mean":0.996303,"std":0.002003}],"selective_risk":[{"mean":0.001425,"std":0.002467},{"mean":0.000711,"std":0.001231},{"mean":0.000953,"std":0.000826},{"mean":0.001426,"std":0.000634},{"mean":0.001152,"std":0.000496},{"mean":0.001426,"std":0.000712},{"mean":0.001819,"std":2.5e-05},{"mean":0.001945,"std":0.000621},{"mean":0.00203,"std":0.000276},{"mean":0.002121,"std":3e-05},{"mean":0.002205,"std":0.00022},{"mean":0.002147,"std":1.6e-05},{"mean":0.002321,"std":1.3e-05},{"mean":0.002686,"std":0.00065},{"mean":0.003073,"std":0.000614},{"mean":0.004181,"std":0.000779},{"mean":0.005971,"std":0.000441},{"mean":0.011073,"std":0.001012},{"mean":0.023932,"std":0.001688},{"mean":0.078591,"std":0.002066}],"accuracy_8":[{"mean":0.824364,"std":0.001373},{"mean":0.828545,"std":0.001666},{"mean":0.832242,"std":0.001412},{"mean":0.837333,"std":0.00082},{"mean":0.841455,"std":0.00063},{"mean":0.845697,"std":0.000555},{"mean":0.851758,"std":0.001001},{"mean":0.856788,"std":0.000555},{"mean":0.862545,"std":0.000656},{"mean":0.866909,"std":0.000315},{"mean":0.871152,"std":0.001169},{"mean":0.875697,"std":0.001001},{"mean":0.880242,"std":0.001689},{"mean":0.884485,"std":0.001892},{"mean":0.890485,"std":0.001065},{"mean":0.895939,"std":0.002169},{"mean":0.903455,"std":0.002678},{"mean":0.912545,"std":0.001924},{"mean":0.920788,"std":0.00168},{"mean":0.919879,"std":0.001568}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.000577},{"mean":0.002333,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.003333,"std":0.000577},{"mean":0.003667,"std":0.000577},{"mean":0.004333,"std":0.000577},{"mean":0.004667,"std":0.000577},{"mean":0.005333,"std":0.000577},{"mean":0.006667,"std":0.002082},{"mean":0.008,"std":0.001732},{"mean":0.012333,"std":0.002309},{"mean":0.019333,"std":0.001528},{"mean":0.041667,"std":0.004619},{"mean":0.093667,"std":0.004726},{"mean":0.373333,"std":0.010693}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.431667,"std":0.000577},{"mean":0.431333,"std":0.001155},{"mean":0.431333,"std":0.001528},{"mean":0.430333,"std":0.002082},{"mean":0.430667,"std":0.002309},{"mean":0.429667,"std":0.002517},{"mean":0.428667,"std":0.002309},{"mean":0.427667,"std":0.003215},{"mean":0.426,"std":0.002646},{"mean":0.424,"std":0.001732},{"mean":0.422667,"std":0.002517},{"mean":0.414667,"std":0.007638},{"mean":0.399333,"std":0.008327},{"mean":0.373,"std":0.004359},{"mean":0.383,"std":0.006083}],"llm_call_rate":[{"mean":0.957091,"std":0.005827},{"mean":0.914545,"std":0.003186},{"mean":0.873212,"std":0.001857},{"mean":0.829455,"std":0.001753},{"mean":0.789697,"std":0.003135},{"mean":0.744485,"std":0.00401},{"mean":0.700121,"std":0.004131},{"mean":0.656909,"std":0.001636},{"mean":0.611758,"std":0.002649},{"mean":0.571333,"std":0.006015},{"mean":0.532667,"std":0.010413},{"mean":0.491758,"std":0.003728},{"mean":0.451576,"std":0.003093},{"mean":0.413273,"std":0.002678},{"mean":0.368485,"std":0.006881},{"mean":0.318485,"std":0.002783},{"mean":0.269091,"std":0.001311},{"mean":0.206485,"std":0.003374},{"mean":0.131515,"std":0.005954},{"mean":0.003697,"std":0.002003}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.00105},{"mean":0.000808,"std":0.0007},{"mean":0.000606,"std":0.000525},{"mean":0.001212,"std":0.00042},{"mean":0.001212,"std":0.000606},{"mean":0.001385,"std":0.0006},{"mean":0.001667,"std":0.000262},{"mean":0.002424,"std":0.000404},{"mean":0.002545,"std":0.000364},{"mean":0.002865,"std":0.000191},{"mean":0.002828,"std":0.000175},{"mean":0.003263,"std":0.000161},{"mean":0.004589,"std":0.000654},{"mean":0.006384,"std":0.000918},{"mean":0.010985,"std":0.000694},{"mean":0.017255,"std":0.000538},{"mean":0.031852,"std":0.002359},{"mean":0.055056,"std":0.001054},{"mean":0.088364,"std":0.000315}],"operating":{"tau":[{"mean":0.999996,"std":0.0},{"mean":0.999993,"std":0.0},{"mean":0.999989,"std":0.0},{"mean":0.999986,"std":1e-06},{"mean":0.999982,"std":1e-06},{"mean":0.999977,"std":1e-06},{"mean":0.999971,"std":2e-06},{"mean":0.999964,"std":2e-06},{"mean":0.999954,"std":3e-06},{"mean":0.999941,"std":5e-06},{"mean":0.999923,"std":7e-06},{"mean":0.999897,"std":8e-06},{"mean":0.999859,"std":1e-05},{"mean":0.999798,"std":1e-05},{"mean":0.999686,"std":2.5e-05},{"mean":0.999413,"std":4.5e-05},{"mean":0.998651,"std":0.000107},{"mean":0.995509,"std":0.000582},{"mean":0.958939,"std":0.008306},{"mean":0.307924,"std":0.04922}],"coverage":[{"mean":0.041455,"std":0.004643},{"mean":0.086182,"std":0.006254},{"mean":0.127879,"std":0.001766},{"mean":0.169091,"std":0.006052},{"mean":0.210182,"std":0.00365},{"mean":0.252788,"std":0.00631},{"mean":0.297212,"std":0.005649},{"mean":0.340848,"std":0.005144},{"mean":0.383515,"std":0.001525},{"mean":0.421333,"std":0.002092},{"mean":0.466606,"std":0.00164},{"mean":0.508788,"std":0.004357},{"mean":0.548545,"std":0.004098},{"mean":0.58903,"std":0.005976},{"mean":0.627818,"std":0.003483},{"mean":0.673818,"std":0.000793},{"mean":0.722909,"std":0.001967},{"mean":0.779576,"std":0.004006},{"mean":0.859879,"std":0.005321},{"mean":0.994606,"std":0.004746}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.000712,"std":0.001234},{"mean":0.000946,"std":0.00082},{"mean":0.000723,"std":0.000627},{"mean":0.00058,"std":0.000503},{"mean":0.001206,"std":0.000437},{"mean":0.001222,"std":0.0006},{"mean":0.001417,"std":0.000591},{"mean":0.00158,"std":0.000267},{"mean":0.002012,"std":0.000488},{"mean":0.002467,"std":0.000443},{"mean":0.002738,"std":0.000387},{"mean":0.002872,"std":0.000171},{"mean":0.002778,"std":2.8e-05},{"mean":0.002992,"std":0.00015},{"mean":0.003688,"std":0.000682},{"mean":0.005281,"std":0.00065},{"mean":0.008857,"std":0.001715},{"mean":0.020153,"std":0.001675},{"mean":0.083773,"std":0.003818}],"accuracy_8":[{"mean":0.822667,"std":0.000735},{"mean":0.826485,"std":0.001211},{"mean":0.830121,"std":0.00165},{"mean":0.833333,"std":0.002735},{"mean":0.837515,"std":0.002522},{"mean":0.842182,"std":0.002799},{"mean":0.846788,"std":0.002586},{"mean":0.852061,"std":0.00183},{"mean":0.856848,"std":0.001481},{"mean":0.861333,"std":0.002373},{"mean":0.866606,"std":0.001365},{"mean":0.871212,"std":0.001277},{"mean":0.875879,"std":0.000584},{"mean":0.880848,"std":0.001892},{"mean":0.886182,"std":0.001916},{"mean":0.892909,"std":0.00142},{"mean":0.899394,"std":0.002317},{"mean":0.907636,"std":0.002678},{"mean":0.918848,"std":0.001546},{"mean":0.914545,"std":0.001818}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.001155},{"mean":0.001333,"std":0.001155},{"mean":0.002,"std":0.001},{"mean":0.002667,"std":0.000577},{"mean":0.004,"std":0.001},{"mean":0.005,"std":0.001},{"mean":0.006333,"std":0.001528},{"mean":0.007,"std":0.001},{"mean":0.007333,"std":0.000577},{"mean":0.008333,"std":0.000577},{"mean":0.011333,"std":0.002082},{"mean":0.017,"std":0.002646},{"mean":0.032333,"std":0.005508},{"mean":0.080333,"std":0.005859},{"mean":0.405333,"std":0.019348}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.431333,"std":0.001155},{"mean":0.431333,"std":0.002082},{"mean":0.431333,"std":0.002082},{"mean":0.430667,"std":0.002309},{"mean":0.429667,"std":0.002517},{"mean":0.428667,"std":0.002309},{"mean":0.427333,"std":0.002887},{"mean":0.425333,"std":0.001528},{"mean":0.423,"std":0.001732},{"mean":0.414,"std":0.005292},{"mean":0.385333,"std":0.006658},{"mean":0.416667,"std":0.00611}],"llm_call_rate":[{"mean":0.958545,"std":0.004643},{"mean":0.913818,"std":0.006254},{"mean":0.872121,"std":0.001766},{"mean":0.830909,"std":0.006052},{"mean":0.789818,"std":0.00365},{"mean":0.747212,"std":0.00631},{"mean":0.702788,"std":0.005649},{"mean":0.659152,"std":0.005144},{"mean":0.616485,"std":0.001525},{"mean":0.578667,"std":0.002092},{"mean":0.533394,"std":0.00164},{"mean":0.491212,"std":0.004357},{"mean":0.451455,"std":0.004098},{"mean":0.41097,"std":0.005976},{"mean":0.372182,"std":0.003483},{"mean":0.326182,"std":0.000793},{"mean":0.277091,"std":0.001967},{"mean":0.220424,"std":0.004006},{"mean":0.140121,"std":0.005321},{"mean":0.005394,"std":0.004746}]}},"entropy":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.00105},{"mean":0.000808,"std":0.0007},{"mean":0.000606,"std":0.000525},{"mean":0.001212,"std":0.00042},{"mean":0.001212,"std":0.000606},{"mean":0.001385,"std":0.0006},{"mean":0.001667,"std":0.000262},{"mean":0.002155,"std":0.000617},{"mean":0.002667,"std":0.00021},{"mean":0.002865,"std":0.000191},{"mean":0.002929,"std":0.000175},{"mean":0.00317,"std":0.000161},{"mean":0.004416,"std":0.000519},{"mean":0.006141,"std":0.000741},{"mean":0.010985,"std":0.000694},{"mean":0.017469,"std":0.000445},{"mean":0.032256,"std":0.002122},{"mean":0.056332,"std":0.001436},{"mean":0.088364,"std":0.000315}],"operating":{"tau":[{"mean":-6.7e-05,"std":3e-06},{"mean":-0.000108,"std":6e-06},{"mean":-0.000148,"std":5e-06},{"mean":-0.000193,"std":1e-05},{"mean":-0.000241,"std":1.2e-05},{"mean":-0.000299,"std":1.6e-05},{"mean":-0.000372,"std":2.4e-05},{"mean":-0.000462,"std":2.4e-05},{"mean":-0.000574,"std":3.6e-05},{"mean":-0.000722,"std":5.6e-05},{"mean":-0.000922,"std":7.7e-05},{"mean":-0.0012,"std":9.5e-05},{"mean":-0.001594,"std":0.000103},{"mean":-0.002197,"std":0.000122},{"mean":-0.00329,"std":0.000282},{"mean":-0.005707,"std":0.000484},{"mean":-0.012091,"std":0.000966},{"mean":-0.033965,"std":0.003325},{"mean":-0.209168,"std":0.037608},{"mean":-1.741034,"std":0.074299}],"coverage":[{"mean":0.041818,"std":0.004166},{"mean":0.087091,"std":0.005598},{"mean":0.127818,"std":0.001791},{"mean":0.16903,"std":0.004914},{"mean":0.209939,"std":0.003611},{"mean":0.253152,"std":0.006419},{"mean":0.297455,"std":0.005372},{"mean":0.341515,"std":0.005522},{"mean":0.382606,"std":0.001111},{"mean":0.422061,"std":0.002107},{"mean":0.46697,"std":0.002729},{"mean":0.508727,"std":0.003483},{"mean":0.548424,"std":0.004462},{"mean":0.589515,"std":0.005471},{"mean":0.628545,"std":0.001818},{"mean":0.673212,"std":0.001747},{"mean":0.72303,"std":0.003789},{"mean":0.77897,"std":0.003901},{"mean":0.858485,"std":0.005825},{"mean":0.994364,"std":0.004468}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.000708,"std":0.001226},{"mean":0.000946,"std":0.00082},{"mean":0.000725,"std":0.000629},{"mean":0.000579,"std":0.000501},{"mean":0.001204,"std":0.000437},{"mean":0.001221,"std":0.0006},{"mean":0.001239,"std":0.000286},{"mean":0.001584,"std":0.00027},{"mean":0.001866,"std":0.000241},{"mean":0.002464,"std":0.000437},{"mean":0.00262,"std":0.000188},{"mean":0.002872,"std":0.000169},{"mean":0.002878,"std":0.000151},{"mean":0.002989,"std":0.000159},{"mean":0.003602,"std":0.000569},{"mean":0.005279,"std":0.00064},{"mean":0.008708,"std":0.001909},{"mean":0.01976,"std":0.001731},{"mean":0.083856,"std":0.003407}],"accuracy_8":[{"mean":0.822788,"std":0.000639},{"mean":0.826788,"std":0.001365},{"mean":0.830182,"std":0.001916},{"mean":0.833455,"std":0.002823},{"mean":0.837515,"std":0.00258},{"mean":0.842303,"std":0.002599},{"mean":0.846909,"std":0.002545},{"mean":0.852303,"std":0.001821},{"mean":0.856606,"std":0.00168},{"mean":0.861576,"std":0.002169},{"mean":0.866606,"std":0.001211},{"mean":0.871273,"std":0.001553},{"mean":0.875939,"std":0.000378},{"mean":0.881212,"std":0.001857},{"mean":0.886182,"std":0.002097},{"mean":0.892667,"std":0.001377},{"mean":0.899333,"std":0.002044},{"mean":0.907818,"std":0.002212},{"mean":0.918545,"std":0.001443},{"mean":0.914364,"std":0.002049}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.001155},{"mean":0.001333,"std":0.001155},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.003667,"std":0.000577},{"mean":0.005,"std":0.001},{"mean":0.006,"std":0.001},{"mean":0.007,"std":0.001},{"mean":0.007333,"std":0.000577},{"mean":0.008333,"std":0.000577},{"mean":0.011,"std":0.001732},{"mean":0.017,"std":0.002646},{"mean":0.032,"std":0.006083},{"mean":0.078,"std":0.006},{"mean":0.405667,"std":0.020033}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.431667,"std":0.000577},{"mean":0.431,"std":0.001732},{"mean":0.431333,"std":0.002082},{"mean":0.430667,"std":0.002309},{"mean":0.429667,"std":0.002517},{"mean":0.428667,"std":0.002309},{"mean":0.427667,"std":0.003215},{"mean":0.425333,"std":0.001528},{"mean":0.423667,"std":0.001528},{"mean":0.414333,"std":0.004041},{"mean":0.386,"std":0.005568},{"mean":0.417667,"std":0.010599}],"llm_call_rate":[{"mean":0.958182,"std":0.004166},{"mean":0.912909,"std":0.005598},{"mean":0.872182,"std":0.001791},{"mean":0.83097,"std":0.004914},{"mean":0.790061,"std":0.003611},{"mean":0.746848,"std":0.006419},{"mean":0.702545,"std":0.005372},{"mean":0.658485,"std":0.005522},{"mean":0.617394,"std":0.001111},{"mean":0.577939,"std":0.002107},{"mean":0.53303,"std":0.002729},{"mean":0.491273,"std":0.003483},{"mean":0.451576,"std":0.004462},{"mean":0.410485,"std":0.005471},{"mean":0.371455,"std":0.001818},{"mean":0.326788,"std":0.001747},{"mean":0.27697,"std":0.003789},{"mean":0.22103,"std":0.003901},{"mean":0.141515,"std":0.005825},{"mean":0.005636,"std":0.004468}]}},"margin":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.00105},{"mean":0.000808,"std":0.0007},{"mean":0.000606,"std":0.000525},{"mean":0.001212,"std":0.00042},{"mean":0.001414,"std":0.0007},{"mean":0.001732,"std":0.0006},{"mean":0.00197,"std":0.000262},{"mean":0.002155,"std":0.000233},{"mean":0.002303,"std":0.00021},{"mean":0.002534,"std":0.000505},{"mean":0.00303,"std":0.0},{"mean":0.003543,"std":0.000161},{"mean":0.004416,"std":0.000937},{"mean":0.00703,"std":0.001111},{"mean":0.011288,"std":0.000347},{"mean":0.018111,"std":0.001098},{"mean":0.033872,"std":0.001833},{"mean":0.058373,"std":0.000574},{"mean":0.088364,"std":0.000315}],"operating":{"tau":[{"mean":10.302077,"std":0.11805},{"mean":9.880064,"std":0.084241},{"mean":9.579717,"std":0.071777},{"mean":9.345912,"std":0.076486},{"mean":9.120394,"std":0.082707},{"mean":8.93576,"std":0.08089},{"mean":8.743272,"std":0.072682},{"mean":8.539599,"std":0.057863},{"mean":8.337033,"std":0.058034},{"mean":8.138294,"std":0.066164},{"mean":7.932826,"std":0.066669},{"mean":7.691033,"std":0.053882},{"mean":7.433314,"std":0.067645},{"mean":7.140606,"std":0.0412},{"mean":6.775006,"std":0.024856},{"mean":6.276716,"std":0.028691},{"mean":5.5991,"std":0.052672},{"mean":4.592648,"std":0.096831},{"mean":2.864016,"std":0.138764},{"mean":0.019183,"std":0.012443}],"coverage":[{"mean":0.040788,"std":0.005258},{"mean":0.086,"std":0.005155},{"mean":0.128364,"std":0.003577},{"mean":0.167879,"std":0.007301},{"mean":0.213697,"std":0.007052},{"mean":0.253636,"std":0.006174},{"mean":0.294485,"std":0.00883},{"mean":0.341515,"std":0.000584},{"mean":0.384727,"std":0.003186},{"mean":0.424061,"std":0.00021},{"mean":0.462485,"std":0.002508},{"mean":0.505818,"std":0.004639},{"mean":0.548848,"std":0.001315},{"mean":0.585879,"std":0.004565},{"mean":0.627212,"std":0.00134},{"mean":0.675212,"std":0.00147},{"mean":0.724121,"std":0.005108},{"mean":0.781212,"std":0.004604},{"mean":0.859515,"std":0.002612},{"mean":0.998424,"std":0.00129}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.00069,"std":0.001195},{"mean":0.000459,"std":0.000794},{"mean":0.000732,"std":0.000635},{"mean":0.001146,"std":0.000539},{"mean":0.001193,"std":0.000406},{"mean":0.001228,"std":0.000587},{"mean":0.001597,"std":0.000533},{"mean":0.001734,"std":0.000285},{"mean":0.002144,"std":1e-06},{"mean":0.002227,"std":0.000216},{"mean":0.002276,"std":0.000189},{"mean":0.002539,"std":0.000502},{"mean":0.003,"std":0.000176},{"mean":0.003189,"std":0.000295},{"mean":0.00395,"std":0.000319},{"mean":0.005692,"std":0.000729},{"mean":0.009385,"std":0.001114},{"mean":0.020658,"std":0.00112},{"mean":0.087349,"std":0.001075}],"accuracy_8":[{"mean":0.822545,"std":0.000481},{"mean":0.825091,"std":0.000481},{"mean":0.828303,"std":0.000933},{"mean":0.831818,"std":0.002212},{"mean":0.835697,"std":0.001944},{"mean":0.839818,"std":0.001585},{"mean":0.844121,"std":0.002548},{"mean":0.850545,"std":0.003186},{"mean":0.855697,"std":0.003379},{"mean":0.86,"std":0.001791},{"mean":0.86503,"std":0.000915},{"mean":0.869758,"std":0.000105},{"mean":0.874909,"std":0.000962},{"mean":0.879576,"std":0.001568},{"mean":0.885091,"std":0.001791},{"mean":0.892242,"std":0.001821},{"mean":0.898667,"std":0.002548},{"mean":0.907394,"std":0.002199},{"mean":0.917758,"std":0.001065},{"mean":0.912,"std":0.000656}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001,"std":0.001},{"mean":0.001333,"std":0.001155},{"mean":0.001333,"std":0.001155},{"mean":0.002333,"std":0.001155},{"mean":0.003,"std":0.001},{"mean":0.004333,"std":0.000577},{"mean":0.005,"std":0.001},{"mean":0.005667,"std":0.001155},{"mean":0.006333,"std":0.001528},{"mean":0.008,"std":0.001},{"mean":0.009333,"std":0.000577},{"mean":0.012333,"std":0.000577},{"mean":0.018667,"std":0.002082},{"mean":0.034667,"std":0.002887},{"mean":0.085667,"std":0.003786},{"mean":0.427,"std":0.009}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.431333,"std":0.001155},{"mean":0.431667,"std":0.002309},{"mean":0.431,"std":0.003464},{"mean":0.430667,"std":0.003215},{"mean":0.429667,"std":0.001528},{"mean":0.429333,"std":0.001155},{"mean":0.427333,"std":0.003055},{"mean":0.426,"std":0.002},{"mean":0.424,"std":0.003606},{"mean":0.414,"std":0.005568},{"mean":0.391,"std":0.004583},{"mean":0.431,"std":0.007211}],"llm_call_rate":[{"mean":0.959212,"std":0.005258},{"mean":0.914,"std":0.005155},{"mean":0.871636,"std":0.003577},{"mean":0.832121,"std":0.007301},{"mean":0.786303,"std":0.007052},{"mean":0.746364,"std":0.006174},{"mean":0.705515,"std":0.00883},{"mean":0.658485,"std":0.000584},{"mean":0.615273,"std":0.003186},{"mean":0.575939,"std":0.00021},{"mean":0.537515,"std":0.002508},{"mean":0.494182,"std":0.004639},{"mean":0.451152,"std":0.001315},{"mean":0.414121,"std":0.004565},{"mean":0.372788,"std":0.00134},{"mean":0.324788,"std":0.00147},{"mean":0.275879,"std":0.005108},{"mean":0.218788,"std":0.004604},{"mean":0.140485,"std":0.002612},{"mean":0.001576,"std":0.00129}]}},"msp_t":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.00105},{"mean":0.000808,"std":0.0007},{"mean":0.000606,"std":0.000525},{"mean":0.001212,"std":0.00042},{"mean":0.001212,"std":0.000606},{"mean":0.001212,"std":0.000794},{"mean":0.001667,"std":0.000262},{"mean":0.00229,"std":0.000617},{"mean":0.002545,"std":0.000364},{"mean":0.002865,"std":0.000191},{"mean":0.002929,"std":0.000175},{"mean":0.00317,"std":0.000161},{"mean":0.004502,"std":0.00015},{"mean":0.006222,"std":0.000741},{"mean":0.011136,"std":0.000394},{"mean":0.017683,"std":0.000688},{"mean":0.03367,"std":0.002234},{"mean":0.057608,"std":0.00134},{"mean":0.088364,"std":0.000315}],"operating":{"tau":[{"mean":0.999893,"std":1.3e-05},{"mean":0.999839,"std":1.7e-05},{"mean":0.999789,"std":1.6e-05},{"mean":0.999736,"std":1.8e-05},{"mean":0.999681,"std":1.9e-05},{"mean":0.999617,"std":2.2e-05},{"mean":0.99954,"std":2.9e-05},{"mean":0.999451,"std":3.3e-05},{"mean":0.999338,"std":2.7e-05},{"mean":0.999195,"std":3.9e-05},{"mean":0.99901,"std":4.3e-05},{"mean":0.998745,"std":6e-05},{"mean":0.9984,"std":5.3e-05},{"mean":0.997916,"std":5.1e-05},{"mean":0.997036,"std":6.2e-05},{"mean":0.995262,"std":0.000186},{"mean":0.991128,"std":0.000393},{"mean":0.97814,"std":0.001516},{"mean":0.893176,"std":0.017382},{"mean":0.249366,"std":0.027794}],"coverage":[{"mean":0.04097,"std":0.004273},{"mean":0.084788,"std":0.005845},{"mean":0.127576,"std":0.003354},{"mean":0.169515,"std":0.002169},{"mean":0.211333,"std":0.00258},{"mean":0.25303,"std":0.006031},{"mean":0.297091,"std":0.006988},{"mean":0.338545,"std":0.004923},{"mean":0.381333,"std":0.002302},{"mean":0.425273,"std":0.001311},{"mean":0.46697,"std":0.002259},{"mean":0.512,"std":0.004194},{"mean":0.551818,"std":0.003549},{"mean":0.590424,"std":0.003432},{"mean":0.629939,"std":0.002888},{"mean":0.674061,"std":0.002854},{"mean":0.722485,"std":0.004107},{"mean":0.778545,"std":0.002751},{"mean":0.855636,"std":0.006638},{"mean":0.995818,"std":0.003963}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.000736,"std":0.001275},{"mean":0.00094,"std":0.000814},{"mean":0.000712,"std":0.000617},{"mean":0.00086,"std":1.1e-05},{"mean":0.00096,"std":0.000421},{"mean":0.001223,"std":0.000601},{"mean":0.001251,"std":0.000808},{"mean":0.001747,"std":0.000268},{"mean":0.002279,"std":0.000645},{"mean":0.002465,"std":0.000439},{"mean":0.002485,"std":0.00035},{"mean":0.002855,"std":0.000175},{"mean":0.002976,"std":0.000166},{"mean":0.002886,"std":0.000277},{"mean":0.003868,"std":0.000575},{"mean":0.005452,"std":0.000515},{"mean":0.008715,"std":0.001523},{"mean":0.019611,"std":0.002},{"mean":0.085563,"std":0.00276}],"accuracy_8":[{"mean":0.822667,"std":0.000639},{"mean":0.826667,"std":0.001183},{"mean":0.829697,"std":0.001589},{"mean":0.833273,"std":0.002204},{"mean":0.837152,"std":0.002003},{"mean":0.842424,"std":0.002442},{"mean":0.84703,"std":0.002169},{"mean":0.852061,"std":0.001821},{"mean":0.856909,"std":0.002378},{"mean":0.861818,"std":0.001443},{"mean":0.86703,"std":0.001794},{"mean":0.872121,"std":0.001238},{"mean":0.876364,"std":0.000793},{"mean":0.881394,"std":0.001546},{"mean":0.886303,"std":0.001718},{"mean":0.892848,"std":0.00147},{"mean":0.89903,"std":0.001365},{"mean":0.907636,"std":0.002049},{"mean":0.917576,"std":0.001546},{"mean":0.913333,"std":0.001766}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.001155},{"mean":0.001667,"std":0.001528},{"mean":0.003,"std":0.0},{"mean":0.004333,"std":0.001155},{"mean":0.005,"std":0.001},{"mean":0.005667,"std":0.001155},{"mean":0.007,"std":0.001},{"mean":0.007667,"std":0.000577},{"mean":0.008,"std":0.001},{"mean":0.011333,"std":0.002082},{"mean":0.017667,"std":0.002082},{"mean":0.031667,"std":0.004726},{"mean":0.077333,"std":0.006506},{"mean":0.415333,"std":0.013429}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001732},{"mean":0.430667,"std":0.002309},{"mean":0.43,"std":0.002},{"mean":0.428667,"std":0.002309},{"mean":0.427667,"std":0.003215},{"mean":0.425333,"std":0.001528},{"mean":0.424333,"std":0.001155},{"mean":0.416,"std":0.005568},{"mean":0.391667,"std":0.006658},{"mean":0.423333,"std":0.007506}],"llm_call_rate":[{"mean":0.95903,"std":0.004273},{"mean":0.915212,"std":0.005845},{"mean":0.872424,"std":0.003354},{"mean":0.830485,"std":0.002169},{"mean":0.788667,"std":0.00258},{"mean":0.74697,"std":0.006031},{"mean":0.702909,"std":0.006988},{"mean":0.661455,"std":0.004923},{"mean":0.618667,"std":0.002302},{"mean":0.574727,"std":0.001311},{"mean":0.53303,"std":0.002259},{"mean":0.488,"std":0.004194},{"mean":0.448182,"std":0.003549},{"mean":0.409576,"std":0.003432},{"mean":0.370061,"std":0.002888},{"mean":0.325939,"std":0.002854},{"mean":0.277515,"std":0.004107},{"mean":0.221455,"std":0.002751},{"mean":0.144364,"std":0.006638},{"mean":0.004182,"std":0.003963}]}}}}},"modernbert/k25":{"model":"modernbert","k":25,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["ModernBERT-base-k25-seed42","ModernBERT-base-k25-seed43","ModernBERT-base-k25-seed44"],"logits_sha256":["da2c8b7a2fcefcda47211bec4d2248e7ca5288eda05cc2572ade0945abea36ff","ac640bffd502ea71d7dd5b5f9cc18d7068b6b8b1b950acbf3a2fd58a3c50f1b6","cd9b8473e1b5b0d4597feb76bdf99d27464910b85b536eb7169193676c94ce68"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.000606,"std":0.00105},{"mean":0.000404,"std":0.0007},{"mean":0.001515,"std":0.00105},{"mean":0.002424,"std":0.00168},{"mean":0.002424,"std":0.00105},{"mean":0.003117,"std":0.001558},{"mean":0.003333,"std":0.001312},{"mean":0.003771,"std":0.000233},{"mean":0.00497,"std":0.001111},{"mean":0.006501,"std":0.001664},{"mean":0.008788,"std":0.001987},{"mean":0.012587,"std":0.00239},{"mean":0.016364,"std":0.002249},{"mean":0.025455,"std":0.002753},{"mean":0.037652,"std":0.004124},{"mean":0.053262,"std":0.005147},{"mean":0.072929,"std":0.007982},{"mean":0.099777,"std":0.009231},{"mean":0.132364,"std":0.009291}],"operating":{"tau":[{"mean":0.999663,"std":9.9e-05},{"mean":0.999426,"std":0.000173},{"mean":0.999174,"std":0.000279},{"mean":0.998844,"std":0.000393},{"mean":0.998421,"std":0.000578},{"mean":0.997814,"std":0.000848},{"mean":0.997084,"std":0.001224},{"mean":0.996025,"std":0.00175},{"mean":0.994486,"std":0.002486},{"mean":0.992248,"std":0.003689},{"mean":0.988913,"std":0.00561},{"mean":0.982888,"std":0.009188},{"mean":0.97374,"std":0.014301},{"mean":0.957821,"std":0.022298},{"mean":0.93016,"std":0.037277},{"mean":0.876917,"std":0.050434},{"mean":0.79003,"std":0.070468},{"mean":0.626316,"std":0.075723},{"mean":0.425959,"std":0.049979},{"mean":0.094579,"std":0.022807}],"coverage":[{"mean":0.043273,"std":0.005844},{"mean":0.088303,"std":0.006529},{"mean":0.126545,"std":0.005724},{"mean":0.168667,"std":0.006549},{"mean":0.211152,"std":0.005876},{"mean":0.255091,"std":0.005068},{"mean":0.292848,"std":0.005853},{"mean":0.336,"std":0.004589},{"mean":0.379636,"std":0.004424},{"mean":0.423636,"std":0.003757},{"mean":0.466,"std":0.004158},{"mean":0.509333,"std":0.00646},{"mean":0.553939,"std":0.004777},{"mean":0.595273,"std":0.002926},{"mean":0.639394,"std":0.004195},{"mean":0.69497,"std":0.001183},{"mean":0.743818,"std":0.004413},{"mean":0.813212,"std":0.007033},{"mean":0.887333,"std":0.00646},{"mean":0.997576,"std":0.002854}],"selective_risk":[{"mean":0.001287,"std":0.002229},{"mean":0.00065,"std":0.001125},{"mean":0.000456,"std":0.00079},{"mean":0.001079,"std":4.1e-05},{"mean":0.001442,"std":0.001013},{"mean":0.002385,"std":0.001673},{"mean":0.002287,"std":0.001333},{"mean":0.002715,"std":0.00112},{"mean":0.003043,"std":0.001493},{"mean":0.003428,"std":0.001111},{"mean":0.004288,"std":0.000743},{"mean":0.005461,"std":0.00154},{"mean":0.006556,"std":0.00159},{"mean":0.008447,"std":0.002587},{"mean":0.011837,"std":0.002651},{"mean":0.015695,"std":0.002254},{"mean":0.02435,"std":0.003309},{"mean":0.041473,"std":0.006745},{"mean":0.067242,"std":0.010263},{"mean":0.130479,"std":0.011068}],"accuracy_8":[{"mean":0.824848,"std":0.001589},{"mean":0.829394,"std":0.001546},{"mean":0.833818,"std":0.000481},{"mean":0.838182,"std":0.000315},{"mean":0.842727,"std":0.000481},{"mean":0.846545,"std":0.000909},{"mean":0.850121,"std":0.00134},{"mean":0.853939,"std":0.001944},{"mean":0.858727,"std":0.001273},{"mean":0.863818,"std":0.000315},{"mean":0.868182,"std":0.000545},{"mean":0.872727,"std":0.000793},{"mean":0.877576,"std":0.00021},{"mean":0.882182,"std":0.001575},{"mean":0.886788,"std":0.001377},{"mean":0.892424,"std":0.001568},{"mean":0.895697,"std":0.002099},{"mean":0.896667,"std":0.002854},{"mean":0.892788,"std":0.005214},{"mean":0.868424,"std":0.009919}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.001155},{"mean":0.001667,"std":0.002082},{"mean":0.002,"std":0.001732},{"mean":0.003,"std":0.001732},{"mean":0.004333,"std":0.002887},{"mean":0.006,"std":0.003},{"mean":0.008333,"std":0.002517},{"mean":0.011667,"std":0.005033},{"mean":0.016,"std":0.005},{"mean":0.022,"std":0.006557},{"mean":0.034,"std":0.008544},{"mean":0.049,"std":0.00755},{"mean":0.079667,"std":0.007024},{"mean":0.149,"std":0.018248},{"mean":0.262,"std":0.043405},{"mean":0.59,"std":0.059858}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.432333,"std":0.002517},{"mean":0.432667,"std":0.002517},{"mean":0.433,"std":0.002},{"mean":0.433667,"std":0.002309},{"mean":0.434667,"std":0.002309},{"mean":0.435667,"std":0.001528},{"mean":0.436333,"std":0.002082},{"mean":0.435333,"std":0.003215},{"mean":0.44,"std":0.0},{"mean":0.456333,"std":0.008963},{"mean":0.482,"std":0.022605},{"mean":0.597,"std":0.052307}],"llm_call_rate":[{"mean":0.956727,"std":0.005844},{"mean":0.911697,"std":0.006529},{"mean":0.873455,"std":0.005724},{"mean":0.831333,"std":0.006549},{"mean":0.788848,"std":0.005876},{"mean":0.744909,"std":0.005068},{"mean":0.707152,"std":0.005853},{"mean":0.664,"std":0.004589},{"mean":0.620364,"std":0.004424},{"mean":0.576364,"std":0.003757},{"mean":0.534,"std":0.004158},{"mean":0.490667,"std":0.00646},{"mean":0.446061,"std":0.004777},{"mean":0.404727,"std":0.002926},{"mean":0.360606,"std":0.004195},{"mean":0.30503,"std":0.001183},{"mean":0.256182,"std":0.004413},{"mean":0.186788,"std":0.007033},{"mean":0.112667,"std":0.00646},{"mean":0.002424,"std":0.002854}]}},"entropy":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.000606,"std":0.00105},{"mean":0.000404,"std":0.0007},{"mean":0.001515,"std":0.00105},{"mean":0.002424,"std":0.00168},{"mean":0.002222,"std":0.001262},{"mean":0.002944,"std":0.0012},{"mean":0.00303,"std":0.001461},{"mean":0.003636,"std":0.000404},{"mean":0.005091,"std":0.001455},{"mean":0.00595,"std":0.001515},{"mean":0.008182,"std":0.002589},{"mean":0.011655,"std":0.002379},{"mean":0.015671,"std":0.001522},{"mean":0.023192,"std":0.0022},{"mean":0.034545,"std":0.003943},{"mean":0.051266,"std":0.005669},{"mean":0.072929,"std":0.008349},{"mean":0.099904,"std":0.008689},{"mean":0.132364,"std":0.009291}],"operating":{"tau":[{"mean":-0.004034,"std":0.001093},{"mean":-0.006585,"std":0.001791},{"mean":-0.009134,"std":0.002738},{"mean":-0.012335,"std":0.00379},{"mean":-0.016235,"std":0.005192},{"mean":-0.021758,"std":0.007499},{"mean":-0.027706,"std":0.010173},{"mean":-0.036424,"std":0.013926},{"mean":-0.047697,"std":0.018796},{"mean":-0.065596,"std":0.027147},{"mean":-0.089165,"std":0.037877},{"mean":-0.126815,"std":0.05592},{"mean":-0.176843,"std":0.071399},{"mean":-0.260132,"std":0.116988},{"mean":-0.39241,"std":0.154989},{"mean":-0.594956,"std":0.174638},{"mean":-0.876524,"std":0.184793},{"mean":-1.344435,"std":0.230211},{"mean":-2.182801,"std":0.204968},{"mean":-3.86443,"std":0.127446}],"coverage":[{"mean":0.042182,"std":0.005852},{"mean":0.087515,"std":0.007028},{"mean":0.127091,"std":0.005645},{"mean":0.169879,"std":0.005605},{"mean":0.211212,"std":0.007968},{"mean":0.256364,"std":0.005677},{"mean":0.293939,"std":0.006015},{"mean":0.336364,"std":0.002909},{"mean":0.378424,"std":0.002956},{"mean":0.426303,"std":0.00447},{"mean":0.469212,"std":0.003714},{"mean":0.510909,"std":0.006487},{"mean":0.553273,"std":0.004442},{"mean":0.59297,"std":0.004395},{"mean":0.637455,"std":0.003026},{"mean":0.690848,"std":0.002842},{"mean":0.741879,"std":0.00129},{"mean":0.797576,"std":0.006708},{"mean":0.876,"std":0.007086},{"mean":0.998606,"std":0.001568}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.000656,"std":0.001137},{"mean":0.000457,"std":0.000792},{"mean":0.001071,"std":3.5e-05},{"mean":0.001446,"std":0.001025},{"mean":0.002371,"std":0.001658},{"mean":0.002281,"std":0.001335},{"mean":0.002709,"std":0.001105},{"mean":0.00305,"std":0.001488},{"mean":0.003407,"std":0.001108},{"mean":0.004005,"std":0.000606},{"mean":0.005326,"std":0.001566},{"mean":0.006025,"std":0.001486},{"mean":0.007755,"std":0.002549},{"mean":0.010741,"std":0.001805},{"mean":0.014737,"std":0.001624},{"mean":0.02124,"std":0.002421},{"mean":0.033949,"std":0.004496},{"mean":0.062091,"std":0.008242},{"mean":0.131324,"std":0.010378}],"accuracy_8":[{"mean":0.824909,"std":0.001443},{"mean":0.829394,"std":0.001238},{"mean":0.833515,"std":0.00084},{"mean":0.837758,"std":0.000859},{"mean":0.842667,"std":0.000378},{"mean":0.846485,"std":0.000933},{"mean":0.850182,"std":0.000962},{"mean":0.853758,"std":0.00165},{"mean":0.858061,"std":0.000933},{"mean":0.863879,"std":0.00084},{"mean":0.868061,"std":0.000278},{"mean":0.872667,"std":0.000278},{"mean":0.877273,"std":0.000833},{"mean":0.882424,"std":0.00147},{"mean":0.886303,"std":0.00183},{"mean":0.891273,"std":0.000833},{"mean":0.895636,"std":0.001845},{"mean":0.897394,"std":0.001766},{"mean":0.89297,"std":0.0033},{"mean":0.868364,"std":0.009899}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.001155},{"mean":0.001667,"std":0.002082},{"mean":0.002,"std":0.001732},{"mean":0.003,"std":0.001732},{"mean":0.004333,"std":0.002887},{"mean":0.006,"std":0.003},{"mean":0.007667,"std":0.001528},{"mean":0.011333,"std":0.005132},{"mean":0.014333,"std":0.004509},{"mean":0.019667,"std":0.006506},{"mean":0.03,"std":0.005568},{"mean":0.044333,"std":0.003055},{"mean":0.066667,"std":0.004509},{"mean":0.115,"std":0.011269},{"mean":0.228667,"std":0.036116},{"mean":0.595333,"std":0.053873}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.432667,"std":0.002082},{"mean":0.433,"std":0.002646},{"mean":0.433,"std":0.002},{"mean":0.434,"std":0.002646},{"mean":0.434333,"std":0.002082},{"mean":0.434667,"std":0.002517},{"mean":0.435667,"std":0.001528},{"mean":0.436333,"std":0.003055},{"mean":0.441,"std":0.003606},{"mean":0.450667,"std":0.004041},{"mean":0.478,"std":0.016462},{"mean":0.598,"std":0.05009}],"llm_call_rate":[{"mean":0.957818,"std":0.005852},{"mean":0.912485,"std":0.007028},{"mean":0.872909,"std":0.005645},{"mean":0.830121,"std":0.005605},{"mean":0.788788,"std":0.007968},{"mean":0.743636,"std":0.005677},{"mean":0.706061,"std":0.006015},{"mean":0.663636,"std":0.002909},{"mean":0.621576,"std":0.002956},{"mean":0.573697,"std":0.00447},{"mean":0.530788,"std":0.003714},{"mean":0.489091,"std":0.006487},{"mean":0.446727,"std":0.004442},{"mean":0.40703,"std":0.004395},{"mean":0.362545,"std":0.003026},{"mean":0.309152,"std":0.002842},{"mean":0.258121,"std":0.00129},{"mean":0.202424,"std":0.006708},{"mean":0.124,"std":0.007086},{"mean":0.001394,"std":0.001568}]}},"margin":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.000606,"std":0.00105},{"mean":0.001616,"std":0.0007},{"mean":0.001515,"std":0.000525},{"mean":0.002182,"std":0.001924},{"mean":0.003232,"std":0.001948},{"mean":0.003983,"std":0.0012},{"mean":0.004394,"std":0.000946},{"mean":0.005253,"std":0.001212},{"mean":0.006545,"std":0.001666},{"mean":0.008815,"std":0.001993},{"mean":0.012121,"std":0.001687},{"mean":0.015664,"std":0.001747},{"mean":0.021991,"std":0.004721},{"mean":0.031515,"std":0.005399},{"mean":0.045,"std":0.007737},{"mean":0.06303,"std":0.009201},{"mean":0.083973,"std":0.010008},{"mean":0.107368,"std":0.009952},{"mean":0.132364,"std":0.009291}],"operating":{"tau":[{"mean":10.016482,"std":0.370716},{"mean":9.396996,"std":0.357154},{"mean":8.925707,"std":0.365196},{"mean":8.52106,"std":0.366598},{"mean":8.157116,"std":0.358799},{"mean":7.789321,"std":0.372978},{"mean":7.394092,"std":0.407697},{"mean":7.001579,"std":0.378804},{"mean":6.64154,"std":0.400015},{"mean":6.229995,"std":0.468092},{"mean":5.831081,"std":0.495126},{"mean":5.380926,"std":0.503521},{"mean":4.878037,"std":0.512826},{"mean":4.266637,"std":0.500192},{"mean":3.662983,"std":0.533591},{"mean":2.962018,"std":0.537373},{"mean":2.229522,"std":0.480283},{"mean":1.418098,"std":0.264465},{"mean":0.608773,"std":0.106832},{"mean":0.004644,"std":0.004579}],"coverage":[{"mean":0.043636,"std":0.006793},{"mean":0.084667,"std":0.006686},{"mean":0.12503,"std":0.007391},{"mean":0.16703,"std":0.006429},{"mean":0.20297,"std":0.008623},{"mean":0.246182,"std":0.005101},{"mean":0.289394,"std":0.003564},{"mean":0.336485,"std":0.004798},{"mean":0.377394,"std":0.004399},{"mean":0.423455,"std":0.005292},{"mean":0.466061,"std":0.009026},{"mean":0.508364,"std":0.006142},{"mean":0.55303,"std":0.005195},{"mean":0.602303,"std":0.001365},{"mean":0.651758,"std":0.004131},{"mean":0.704545,"std":0.011095},{"mean":0.762182,"std":0.012923},{"mean":0.828364,"std":0.006598},{"mean":0.913091,"std":0.006973},{"mean":0.999333,"std":0.000639}],"selective_risk":[{"mean":0.00119,"std":0.002062},{"mean":0.000659,"std":0.001141},{"mean":0.001448,"std":0.001492},{"mean":0.001458,"std":0.000654},{"mean":0.001509,"std":0.000566},{"mean":0.002241,"std":0.002011},{"mean":0.003365,"std":0.002069},{"mean":0.003432,"std":0.001392},{"mean":0.004339,"std":0.000518},{"mean":0.004574,"std":0.000846},{"mean":0.005836,"std":0.001306},{"mean":0.006661,"std":0.001891},{"mean":0.008866,"std":0.002243},{"mean":0.012476,"std":0.001952},{"mean":0.015986,"std":0.002279},{"mean":0.02291,"std":0.006204},{"mean":0.035051,"std":0.008715},{"mean":0.055251,"std":0.011672},{"mean":0.08947,"std":0.01363},{"mean":0.132023,"std":0.009673}],"accuracy_8":[{"mean":0.82503,"std":0.00082},{"mean":0.828727,"std":0.000962},{"mean":0.833273,"std":0.000656},{"mean":0.838424,"std":0.001001},{"mean":0.842485,"std":0.000584},{"mean":0.846364,"std":0.000182},{"mean":0.850242,"std":0.000555},{"mean":0.855515,"std":0.000933},{"mean":0.860182,"std":0.001898},{"mean":0.864667,"std":0.00164},{"mean":0.86897,"std":0.000915},{"mean":0.874182,"std":0.001106},{"mean":0.878788,"std":0.00147},{"mean":0.883273,"std":0.002321},{"mean":0.888182,"std":0.001488},{"mean":0.892182,"std":0.002715},{"mean":0.894848,"std":0.00438},{"mean":0.894848,"std":0.006566},{"mean":0.885636,"std":0.008929},{"mean":0.867818,"std":0.00947}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.001155},{"mean":0.000667,"std":0.001155},{"mean":0.001,"std":0.001},{"mean":0.001667,"std":0.002082},{"mean":0.003,"std":0.002646},{"mean":0.004,"std":0.002},{"mean":0.006667,"std":0.001155},{"mean":0.008333,"std":0.002517},{"mean":0.011667,"std":0.004163},{"mean":0.015,"std":0.005568},{"mean":0.021667,"std":0.006506},{"mean":0.034667,"std":0.006506},{"mean":0.047333,"std":0.007506},{"mean":0.075,"std":0.018358},{"mean":0.123333,"std":0.026083},{"mean":0.212,"std":0.039038},{"mean":0.371,"std":0.057026},{"mean":0.6,"std":0.048218}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.002},{"mean":0.432333,"std":0.002517},{"mean":0.432333,"std":0.002517},{"mean":0.432667,"std":0.002082},{"mean":0.432667,"std":0.002517},{"mean":0.433333,"std":0.002082},{"mean":0.433667,"std":0.002082},{"mean":0.433667,"std":0.000577},{"mean":0.436,"std":0.002646},{"mean":0.434667,"std":0.002517},{"mean":0.436333,"std":0.005508},{"mean":0.444667,"std":0.011676},{"mean":0.463333,"std":0.017156},{"mean":0.512,"std":0.040731},{"mean":0.601,"std":0.047286}],"llm_call_rate":[{"mean":0.956364,"std":0.006793},{"mean":0.915333,"std":0.006686},{"mean":0.87497,"std":0.007391},{"mean":0.83297,"std":0.006429},{"mean":0.79703,"std":0.008623},{"mean":0.753818,"std":0.005101},{"mean":0.710606,"std":0.003564},{"mean":0.663515,"std":0.004798},{"mean":0.622606,"std":0.004399},{"mean":0.576545,"std":0.005292},{"mean":0.533939,"std":0.009026},{"mean":0.491636,"std":0.006142},{"mean":0.44697,"std":0.005195},{"mean":0.397697,"std":0.001365},{"mean":0.348242,"std":0.004131},{"mean":0.295455,"std":0.011095},{"mean":0.237818,"std":0.012923},{"mean":0.171636,"std":0.006598},{"mean":0.086909,"std":0.006973},{"mean":0.000667,"std":0.000639}]}},"msp_t":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.000606,"std":0.00105},{"mean":0.001212,"std":0.0},{"mean":0.001515,"std":0.00105},{"mean":0.002424,"std":0.00168},{"mean":0.002424,"std":0.00105},{"mean":0.003117,"std":0.001558},{"mean":0.003333,"std":0.001312},{"mean":0.003906,"std":0.000467},{"mean":0.00497,"std":0.001111},{"mean":0.006501,"std":0.001491},{"mean":0.008889,"std":0.001852},{"mean":0.012774,"std":0.002136},{"mean":0.016797,"std":0.002328},{"mean":0.025455,"std":0.002753},{"mean":0.038333,"std":0.004205},{"mean":0.054545,"std":0.005462},{"mean":0.073333,"std":0.008492},{"mean":0.099713,"std":0.009212},{"mean":0.132364,"std":0.009291}],"operating":{"tau":[{"mean":0.999781,"std":7.2e-05},{"mean":0.999618,"std":0.000122},{"mean":0.999448,"std":0.000175},{"mean":0.999217,"std":0.000249},{"mean":0.998913,"std":0.00034},{"mean":0.99849,"std":0.00048},{"mean":0.99796,"std":0.000714},{"mean":0.997165,"std":0.001035},{"mean":0.996012,"std":0.001498},{"mean":0.994325,"std":0.002327},{"mean":0.991674,"std":0.003604},{"mean":0.98701,"std":0.006122},{"mean":0.979547,"std":0.010109},{"mean":0.965672,"std":0.017047},{"mean":0.942523,"std":0.028598},{"mean":0.892768,"std":0.04222},{"mean":0.809839,"std":0.064578},{"mean":0.648458,"std":0.069707},{"mean":0.446051,"std":0.039643},{"mean":0.099455,"std":0.021722}],"coverage":[{"mean":0.043333,"std":0.005808},{"mean":0.088485,"std":0.006156},{"mean":0.126121,"std":0.006455},{"mean":0.167636,"std":0.007613},{"mean":0.211091,"std":0.006011},{"mean":0.254788,"std":0.005364},{"mean":0.292242,"std":0.005932},{"mean":0.336303,"std":0.004034},{"mean":0.380061,"std":0.004462},{"mean":0.424,"std":0.003511},{"mean":0.466424,"std":0.003334},{"mean":0.508545,"std":0.006254},{"mean":0.553455,"std":0.004848},{"mean":0.596242,"std":0.002482},{"mean":0.64,"std":0.004021},{"mean":0.695939,"std":0.002649},{"mean":0.745697,"std":0.006876},{"mean":0.814909,"std":0.009113},{"mean":0.888182,"std":0.006638},{"mean":0.997576,"std":0.002854}],"selective_risk":[{"mean":0.001272,"std":0.002204},{"mean":0.000652,"std":0.00113},{"mean":0.000457,"std":0.000792},{"mean":0.001455,"std":0.000659},{"mean":0.001439,"std":0.001005},{"mean":0.002388,"std":0.001673},{"mean":0.0025,"std":0.001126},{"mean":0.002712,"std":0.001115},{"mean":0.003197,"std":0.00156},{"mean":0.003425,"std":0.001113},{"mean":0.004285,"std":0.000755},{"mean":0.00547,"std":0.001549},{"mean":0.00667,"std":0.001756},{"mean":0.008638,"std":0.001977},{"mean":0.01192,"std":0.002803},{"mean":0.016104,"std":0.002728},{"mean":0.025008,"std":0.004058},{"mean":0.042779,"std":0.008014},{"mean":0.068062,"std":0.010307},{"mean":0.130479,"std":0.011068}],"accuracy_8":[{"mean":0.824848,"std":0.001589},{"mean":0.829394,"std":0.001546},{"mean":0.833758,"std":0.000639},{"mean":0.838121,"std":0.000278},{"mean":0.842667,"std":0.000555},{"mean":0.846485,"std":0.000915},{"mean":0.850182,"std":0.001273},{"mean":0.854242,"std":0.002003},{"mean":0.858667,"std":0.001169},{"mean":0.863758,"std":0.000584},{"mean":0.868061,"std":0.000555},{"mean":0.872727,"std":0.000962},{"mean":0.877515,"std":0.000278},{"mean":0.882485,"std":0.001315},{"mean":0.88697,"std":0.001211},{"mean":0.892424,"std":0.001689},{"mean":0.895758,"std":0.002003},{"mean":0.896364,"std":0.003677},{"mean":0.892485,"std":0.005299},{"mean":0.868424,"std":0.009919}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000667,"std":0.001155},{"mean":0.000667,"std":0.001155},{"mean":0.001667,"std":0.002082},{"mean":0.002333,"std":0.001528},{"mean":0.003,"std":0.001732},{"mean":0.004667,"std":0.003215},{"mean":0.006,"std":0.003},{"mean":0.008333,"std":0.002517},{"mean":0.011667,"std":0.005033},{"mean":0.016333,"std":0.005508},{"mean":0.022333,"std":0.005033},{"mean":0.034,"std":0.008544},{"mean":0.050333,"std":0.008737},{"mean":0.083,"std":0.009539},{"mean":0.154333,"std":0.022745},{"mean":0.266,"std":0.04417},{"mean":0.59,"std":0.059858}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.432667,"std":0.002517},{"mean":0.432667,"std":0.002517},{"mean":0.433,"std":0.002},{"mean":0.433667,"std":0.002309},{"mean":0.435,"std":0.002646},{"mean":0.435333,"std":0.001155},{"mean":0.435667,"std":0.002082},{"mean":0.435667,"std":0.003786},{"mean":0.439667,"std":0.001155},{"mean":0.458,"std":0.01179},{"mean":0.483,"std":0.022539},{"mean":0.597,"std":0.052307}],"llm_call_rate":[{"mean":0.956667,"std":0.005808},{"mean":0.911515,"std":0.006156},{"mean":0.873879,"std":0.006455},{"mean":0.832364,"std":0.007613},{"mean":0.788909,"std":0.006011},{"mean":0.745212,"std":0.005364},{"mean":0.707758,"std":0.005932},{"mean":0.663697,"std":0.004034},{"mean":0.619939,"std":0.004462},{"mean":0.576,"std":0.003511},{"mean":0.533576,"std":0.003334},{"mean":0.491455,"std":0.006254},{"mean":0.446545,"std":0.004848},{"mean":0.403758,"std":0.002482},{"mean":0.36,"std":0.004021},{"mean":0.304061,"std":0.002649},{"mean":0.254303,"std":0.006876},{"mean":0.185091,"std":0.009113},{"mean":0.111818,"std":0.006638},{"mean":0.002424,"std":0.002854}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000808,"std":0.0014},{"mean":0.000909,"std":0.000909},{"mean":0.002182,"std":0.00126},{"mean":0.002828,"std":0.0014},{"mean":0.002944,"std":0.000794},{"mean":0.003333,"std":0.001144},{"mean":0.003771,"std":0.000617},{"mean":0.004242,"std":0.000555},{"mean":0.005179,"std":0.001252},{"mean":0.006566,"std":0.001852},{"mean":0.008019,"std":0.001435},{"mean":0.012468,"std":0.00119},{"mean":0.018909,"std":0.001924},{"mean":0.031136,"std":0.003499},{"mean":0.05041,"std":0.007936},{"mean":0.075623,"std":0.008394},{"mean":0.106348,"std":0.009684},{"mean":0.140303,"std":0.009282}],"operating":{"tau":[{"mean":0.999865,"std":4e-05},{"mean":0.99975,"std":8.3e-05},{"mean":0.999632,"std":0.000114},{"mean":0.999501,"std":0.000156},{"mean":0.999341,"std":0.000212},{"mean":0.999123,"std":0.000272},{"mean":0.998823,"std":0.000372},{"mean":0.998429,"std":0.000571},{"mean":0.997895,"std":0.000804},{"mean":0.997214,"std":0.001053},{"mean":0.996126,"std":0.00142},{"mean":0.99451,"std":0.002259},{"mean":0.991742,"std":0.003805},{"mean":0.987658,"std":0.005545},{"mean":0.979646,"std":0.008965},{"mean":0.964769,"std":0.014112},{"mean":0.932701,"std":0.019427},{"mean":0.85209,"std":0.044865},{"mean":0.633404,"std":0.061157},{"mean":0.207533,"std":0.021759}],"coverage":[{"mean":0.038727,"std":0.00317},{"mean":0.089515,"std":0.001365},{"mean":0.134303,"std":0.00371},{"mean":0.173455,"std":0.006142},{"mean":0.211939,"std":0.002068},{"mean":0.252727,"std":0.001373},{"mean":0.294303,"std":0.004859},{"mean":0.336121,"std":0.001238},{"mean":0.378606,"std":0.00082},{"mean":0.418667,"std":0.002759},{"mean":0.461576,"std":0.005728},{"mean":0.505636,"std":0.00411},{"mean":0.548424,"std":0.005634},{"mean":0.589091,"std":0.008405},{"mean":0.634545,"std":0.004999},{"mean":0.681636,"std":0.003367},{"mean":0.730485,"std":0.005739},{"mean":0.787636,"std":0.008092},{"mean":0.868848,"std":0.007546},{"mean":0.99897,"std":0.001034}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000438,"std":0.000759},{"mean":0.000671,"std":0.001163},{"mean":0.000856,"std":0.000851},{"mean":0.002159,"std":0.001248},{"mean":0.00247,"std":0.00122},{"mean":0.002886,"std":0.001129},{"mean":0.002881,"std":0.000956},{"mean":0.003622,"std":0.000686},{"mean":0.004069,"std":0.000432},{"mean":0.00467,"std":0.00091},{"mean":0.005295,"std":0.001384},{"mean":0.006259,"std":0.001918},{"mean":0.007252,"std":0.001819},{"mean":0.010309,"std":0.001582},{"mean":0.016679,"std":0.000604},{"mean":0.027144,"std":0.003566},{"mean":0.059082,"std":0.012182},{"mean":0.139596,"std":0.009668}],"accuracy_8":[{"mean":0.822788,"std":0.000555},{"mean":0.826364,"std":0.001012},{"mean":0.830364,"std":0.001106},{"mean":0.833939,"std":0.001857},{"mean":0.837697,"std":0.001481},{"mean":0.841333,"std":0.001111},{"mean":0.846121,"std":0.000639},{"mean":0.849697,"std":0.001096},{"mean":0.853394,"std":0.000933},{"mean":0.857758,"std":0.001155},{"mean":0.862242,"std":0.000933},{"mean":0.866424,"std":0.00134},{"mean":0.872788,"std":0.001034},{"mean":0.877333,"std":0.001589},{"mean":0.883333,"std":0.001927},{"mean":0.888182,"std":0.001311},{"mean":0.893697,"std":0.000525},{"mean":0.898121,"std":0.002373},{"mean":0.895091,"std":0.006027},{"mean":0.860424,"std":0.009693}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.002,"std":0.001732},{"mean":0.002667,"std":0.001528},{"mean":0.004,"std":0.002},{"mean":0.004333,"std":0.002517},{"mean":0.006333,"std":0.002517},{"mean":0.008333,"std":0.001528},{"mean":0.010667,"std":0.002887},{"mean":0.013667,"std":0.004726},{"mean":0.016,"std":0.006},{"mean":0.020333,"std":0.005508},{"mean":0.031667,"std":0.005132},{"mean":0.056667,"std":0.001528},{"mean":0.1,"std":0.012124},{"mean":0.235333,"std":0.044004},{"mean":0.65,"std":0.043555}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.001155},{"mean":0.433,"std":0.002},{"mean":0.433667,"std":0.002309},{"mean":0.433,"std":0.002646},{"mean":0.434,"std":0.001732},{"mean":0.435,"std":0.002646},{"mean":0.437,"std":0.001},{"mean":0.440333,"std":0.001528},{"mean":0.445667,"std":0.004163},{"mean":0.481333,"std":0.020008},{"mean":0.650333,"std":0.04365}],"llm_call_rate":[{"mean":0.961273,"std":0.00317},{"mean":0.910485,"std":0.001365},{"mean":0.865697,"std":0.00371},{"mean":0.826545,"std":0.006142},{"mean":0.788061,"std":0.002068},{"mean":0.747273,"std":0.001373},{"mean":0.705697,"std":0.004859},{"mean":0.663879,"std":0.001238},{"mean":0.621394,"std":0.00082},{"mean":0.581333,"std":0.002759},{"mean":0.538424,"std":0.005728},{"mean":0.494364,"std":0.00411},{"mean":0.451576,"std":0.005634},{"mean":0.410909,"std":0.008405},{"mean":0.365455,"std":0.004999},{"mean":0.318364,"std":0.003367},{"mean":0.269515,"std":0.005739},{"mean":0.212364,"std":0.008092},{"mean":0.131152,"std":0.007546},{"mean":0.00103,"std":0.001034}]}},"entropy":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000808,"std":0.0014},{"mean":0.000909,"std":0.000909},{"mean":0.002182,"std":0.00126},{"mean":0.002626,"std":0.001262},{"mean":0.002944,"std":0.000794},{"mean":0.003182,"std":0.000909},{"mean":0.003636,"std":0.000404},{"mean":0.004242,"std":0.000555},{"mean":0.005179,"std":0.001252},{"mean":0.006364,"std":0.001987},{"mean":0.007925,"std":0.001591},{"mean":0.012294,"std":0.001227},{"mean":0.018586,"std":0.001614},{"mean":0.030985,"std":0.003494},{"mean":0.050196,"std":0.008433},{"mean":0.075556,"std":0.007306},{"mean":0.10673,"std":0.009187},{"mean":0.140303,"std":0.009282}],"operating":{"tau":[{"mean":-0.001556,"std":0.000421},{"mean":-0.002706,"std":0.000803},{"mean":-0.00384,"std":0.001089},{"mean":-0.005039,"std":0.00141},{"mean":-0.006491,"std":0.001898},{"mean":-0.008342,"std":0.002302},{"mean":-0.010906,"std":0.00306},{"mean":-0.014045,"std":0.004518},{"mean":-0.018098,"std":0.006159},{"mean":-0.023167,"std":0.007799},{"mean":-0.031052,"std":0.009991},{"mean":-0.041849,"std":0.015088},{"mean":-0.05935,"std":0.023384},{"mean":-0.082254,"std":0.031132},{"mean":-0.12617,"std":0.045754},{"mean":-0.198409,"std":0.063915},{"mean":-0.330057,"std":0.074651},{"mean":-0.593039,"std":0.125425},{"mean":-1.091996,"std":0.145741},{"mean":-1.961397,"std":0.030848}],"coverage":[{"mean":0.038667,"std":0.003364},{"mean":0.089636,"std":0.002212},{"mean":0.134848,"std":0.002169},{"mean":0.173697,"std":0.006088},{"mean":0.212727,"std":0.002446},{"mean":0.252242,"std":0.001718},{"mean":0.295212,"std":0.004547},{"mean":0.33697,"std":0.00021},{"mean":0.37903,"std":0.001412},{"mean":0.418606,"std":0.002795},{"mean":0.463091,"std":0.003338},{"mean":0.506182,"std":0.004983},{"mean":0.548788,"std":0.006091},{"mean":0.587212,"std":0.008325},{"mean":0.634545,"std":0.003632},{"mean":0.681212,"std":0.00258},{"mean":0.730121,"std":0.005845},{"mean":0.786121,"std":0.00714},{"mean":0.864606,"std":0.010511},{"mean":0.997939,"std":0.000915}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000442,"std":0.000765},{"mean":0.000671,"std":0.001162},{"mean":0.000853,"std":0.000847},{"mean":0.002164,"std":0.001254},{"mean":0.002462,"std":0.001217},{"mean":0.003058,"std":0.000824},{"mean":0.002878,"std":0.000955},{"mean":0.003476,"std":0.000763},{"mean":0.004058,"std":0.000246},{"mean":0.004545,"std":0.0007},{"mean":0.00518,"std":0.001463},{"mean":0.006383,"std":0.001929},{"mean":0.007159,"std":0.001688},{"mean":0.010138,"std":0.001714},{"mean":0.016022,"std":0.000355},{"mean":0.026811,"std":0.004005},{"mean":0.056681,"std":0.013076},{"mean":0.138947,"std":0.010083}],"accuracy_8":[{"mean":0.822909,"std":0.000545},{"mean":0.826545,"std":0.000962},{"mean":0.830424,"std":0.000859},{"mean":0.834364,"std":0.001616},{"mean":0.838242,"std":0.001514},{"mean":0.841333,"std":0.001111},{"mean":0.846242,"std":0.000458},{"mean":0.849697,"std":0.000933},{"mean":0.853879,"std":0.000555},{"mean":0.857697,"std":0.001065},{"mean":0.862545,"std":0.000793},{"mean":0.866788,"std":0.001001},{"mean":0.87303,"std":0.001001},{"mean":0.877212,"std":0.001514},{"mean":0.883636,"std":0.001734},{"mean":0.888424,"std":0.001568},{"mean":0.894121,"std":0.00042},{"mean":0.897879,"std":0.002699},{"mean":0.895818,"std":0.006082},{"mean":0.86097,"std":0.009913}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.002,"std":0.001732},{"mean":0.002667,"std":0.001528},{"mean":0.004,"std":0.002},{"mean":0.004333,"std":0.002517},{"mean":0.006,"std":0.002646},{"mean":0.008333,"std":0.001528},{"mean":0.010667,"std":0.002887},{"mean":0.013333,"std":0.004933},{"mean":0.016333,"std":0.006028},{"mean":0.02,"std":0.005},{"mean":0.030667,"std":0.00611},{"mean":0.053667,"std":0.000577},{"mean":0.093,"std":0.012166},{"mean":0.214,"std":0.048877},{"mean":0.645667,"std":0.047427}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.002},{"mean":0.433667,"std":0.002309},{"mean":0.433,"std":0.002646},{"mean":0.434,"std":0.001732},{"mean":0.435333,"std":0.002887},{"mean":0.436667,"std":0.001528},{"mean":0.439,"std":0.002},{"mean":0.443667,"std":0.004041},{"mean":0.475,"std":0.017},{"mean":0.647333,"std":0.046047}],"llm_call_rate":[{"mean":0.961333,"std":0.003364},{"mean":0.910364,"std":0.002212},{"mean":0.865152,"std":0.002169},{"mean":0.826303,"std":0.006088},{"mean":0.787273,"std":0.002446},{"mean":0.747758,"std":0.001718},{"mean":0.704788,"std":0.004547},{"mean":0.66303,"std":0.00021},{"mean":0.62097,"std":0.001412},{"mean":0.581394,"std":0.002795},{"mean":0.536909,"std":0.003338},{"mean":0.493818,"std":0.004983},{"mean":0.451212,"std":0.006091},{"mean":0.412788,"std":0.008325},{"mean":0.365455,"std":0.003632},{"mean":0.318788,"std":0.00258},{"mean":0.269879,"std":0.005845},{"mean":0.213879,"std":0.00714},{"mean":0.135394,"std":0.010511},{"mean":0.002061,"std":0.000915}]}},"margin":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.000606,"std":0.00105},{"mean":0.000808,"std":0.0014},{"mean":0.001212,"std":0.000525},{"mean":0.002424,"std":0.00084},{"mean":0.003232,"std":0.001525},{"mean":0.003117,"std":0.0009},{"mean":0.003788,"std":0.000694},{"mean":0.004175,"std":0.001017},{"mean":0.004848,"std":0.001169},{"mean":0.00562,"std":0.001653},{"mean":0.006667,"std":0.001987},{"mean":0.009138,"std":0.001639},{"mean":0.014113,"std":0.000912},{"mean":0.021333,"std":0.002388},{"mean":0.032955,"std":0.004837},{"mean":0.051266,"std":0.00801},{"mean":0.078249,"std":0.009459},{"mean":0.109219,"std":0.00833},{"mean":0.140303,"std":0.009282}],"operating":{"tau":[{"mean":10.436617,"std":0.311936},{"mean":9.802001,"std":0.300368},{"mean":9.359043,"std":0.278511},{"mean":8.994682,"std":0.256285},{"mean":8.656672,"std":0.247633},{"mean":8.354678,"std":0.246157},{"mean":8.072577,"std":0.253884},{"mean":7.771729,"std":0.249671},{"mean":7.446468,"std":0.275108},{"mean":7.100826,"std":0.28172},{"mean":6.763819,"std":0.302634},{"mean":6.386631,"std":0.304243},{"mean":5.939818,"std":0.35726},{"mean":5.492457,"std":0.324585},{"mean":4.980332,"std":0.36373},{"mean":4.321137,"std":0.360631},{"mean":3.551624,"std":0.319477},{"mean":2.533772,"std":0.355512},{"mean":1.2438,"std":0.172861},{"mean":0.004514,"std":0.002256}],"coverage":[{"mean":0.042061,"std":0.004022},{"mean":0.089091,"std":0.000945},{"mean":0.133333,"std":0.005546},{"mean":0.175152,"std":0.005768},{"mean":0.215697,"std":0.002169},{"mean":0.253091,"std":0.000481},{"mean":0.294242,"std":0.000458},{"mean":0.335515,"std":0.001857},{"mean":0.377091,"std":0.002049},{"mean":0.419758,"std":0.004001},{"mean":0.461515,"std":0.0029},{"mean":0.503576,"std":0.002309},{"mean":0.550182,"std":0.00756},{"mean":0.591152,"std":0.001412},{"mean":0.635333,"std":0.005157},{"mean":0.681939,"std":0.006518},{"mean":0.733939,"std":0.001944},{"mean":0.796788,"std":0.010263},{"mean":0.879879,"std":0.006289},{"mean":0.999758,"std":0.000105}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000436,"std":0.000756},{"mean":0.000669,"std":0.001158},{"mean":0.001686,"std":1.7e-05},{"mean":0.002394,"std":0.000828},{"mean":0.002885,"std":0.000948},{"mean":0.00289,"std":0.001367},{"mean":0.003372,"std":0.000814},{"mean":0.004042,"std":0.000653},{"mean":0.004459,"std":0.001354},{"mean":0.005172,"std":0.001487},{"mean":0.005929,"std":0.002081},{"mean":0.006558,"std":0.001985},{"mean":0.0081,"std":0.001798},{"mean":0.012076,"std":0.001775},{"mean":0.018579,"std":0.002565},{"mean":0.032514,"std":0.006105},{"mean":0.066694,"std":0.012497},{"mean":0.140216,"std":0.009294}],"accuracy_8":[{"mean":0.822667,"std":0.001096},{"mean":0.825818,"std":0.001106},{"mean":0.829273,"std":0.001443},{"mean":0.832303,"std":0.001412},{"mean":0.835758,"std":0.002169},{"mean":0.839939,"std":0.001689},{"mean":0.844,"std":0.000945},{"mean":0.848364,"std":0.000481},{"mean":0.851939,"std":0.000933},{"mean":0.856303,"std":0.001365},{"mean":0.860485,"std":0.001155},{"mean":0.865758,"std":0.001481},{"mean":0.871939,"std":0.000933},{"mean":0.877212,"std":0.001111},{"mean":0.882909,"std":0.001585},{"mean":0.887939,"std":0.001892},{"mean":0.892909,"std":0.002097},{"mean":0.896788,"std":0.003305},{"mean":0.892788,"std":0.007057},{"mean":0.859818,"std":0.009278}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.001528},{"mean":0.003667,"std":0.001528},{"mean":0.004333,"std":0.002517},{"mean":0.005667,"std":0.002082},{"mean":0.007333,"std":0.002309},{"mean":0.009,"std":0.003606},{"mean":0.012,"std":0.004583},{"mean":0.015333,"std":0.007095},{"mean":0.018,"std":0.007},{"mean":0.023333,"std":0.006658},{"mean":0.038667,"std":0.006028},{"mean":0.064,"std":0.00781},{"mean":0.125,"std":0.022338},{"mean":0.277,"std":0.052144},{"mean":0.653333,"std":0.042442}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.433,"std":0.002},{"mean":0.433,"std":0.002646},{"mean":0.433333,"std":0.002887},{"mean":0.434333,"std":0.002082},{"mean":0.435,"std":0.001732},{"mean":0.438667,"std":0.001155},{"mean":0.440667,"std":0.004726},{"mean":0.451333,"std":0.009018},{"mean":0.495333,"std":0.02829},{"mean":0.653333,"std":0.042442}],"llm_call_rate":[{"mean":0.957939,"std":0.004022},{"mean":0.910909,"std":0.000945},{"mean":0.866667,"std":0.005546},{"mean":0.824848,"std":0.005768},{"mean":0.784303,"std":0.002169},{"mean":0.746909,"std":0.000481},{"mean":0.705758,"std":0.000458},{"mean":0.664485,"std":0.001857},{"mean":0.622909,"std":0.002049},{"mean":0.580242,"std":0.004001},{"mean":0.538485,"std":0.0029},{"mean":0.496424,"std":0.002309},{"mean":0.449818,"std":0.00756},{"mean":0.408848,"std":0.001412},{"mean":0.364667,"std":0.005157},{"mean":0.318061,"std":0.006518},{"mean":0.266061,"std":0.001944},{"mean":0.203212,"std":0.010263},{"mean":0.120121,"std":0.006289},{"mean":0.000242,"std":0.000105}]}},"msp_t":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000808,"std":0.0014},{"mean":0.000909,"std":0.000909},{"mean":0.002182,"std":0.00126},{"mean":0.002828,"std":0.0014},{"mean":0.003117,"std":0.001039},{"mean":0.003333,"std":0.001144},{"mean":0.003906,"std":0.000233},{"mean":0.004121,"std":0.000555},{"mean":0.005179,"std":0.001252},{"mean":0.006667,"std":0.001987},{"mean":0.008112,"std":0.001557},{"mean":0.012641,"std":0.001171},{"mean":0.018909,"std":0.002155},{"mean":0.031136,"std":0.003278},{"mean":0.049982,"std":0.007799},{"mean":0.075758,"std":0.008687},{"mean":0.105582,"std":0.00941},{"mean":0.140303,"std":0.009282}],"operating":{"tau":[{"mean":0.999916,"std":3e-05},{"mean":0.999843,"std":4.7e-05},{"mean":0.999765,"std":6.4e-05},{"mean":0.999678,"std":8.7e-05},{"mean":0.999568,"std":0.000111},{"mean":0.999418,"std":0.000142},{"mean":0.999212,"std":0.000186},{"mean":0.998943,"std":0.000292},{"mean":0.998567,"std":0.000402},{"mean":0.998057,"std":0.000541},{"mean":0.997273,"std":0.000736},{"mean":0.996092,"std":0.001279},{"mean":0.993989,"std":0.002232},{"mean":0.990831,"std":0.003453},{"mean":0.984551,"std":0.005675},{"mean":0.971926,"std":0.009917},{"mean":0.944243,"std":0.013482},{"mean":0.869681,"std":0.034995},{"mean":0.654029,"std":0.05047},{"mean":0.209907,"std":0.026441}],"coverage":[{"mean":0.039212,"std":0.00305},{"mean":0.088727,"std":0.000833},{"mean":0.133636,"std":0.004639},{"mean":0.173333,"std":0.005912},{"mean":0.213212,"std":0.002723},{"mean":0.25297,"std":0.001525},{"mean":0.295091,"std":0.003963},{"mean":0.335515,"std":0.001155},{"mean":0.378909,"std":0.000182},{"mean":0.420424,"std":0.003135},{"mean":0.461939,"std":0.006053},{"mean":0.506,"std":0.005068},{"mean":0.548727,"std":0.005671},{"mean":0.589394,"std":0.00856},{"mean":0.635091,"std":0.004465},{"mean":0.681879,"std":0.003611},{"mean":0.730788,"std":0.004846},{"mean":0.788121,"std":0.008939},{"mean":0.869394,"std":0.007264},{"mean":0.998909,"std":0.001311}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000439,"std":0.00076},{"mean":0.000673,"std":0.001166},{"mean":0.000851,"std":0.000844},{"mean":0.002159,"std":0.001254},{"mean":0.002675,"std":0.001288},{"mean":0.002891,"std":0.001129},{"mean":0.002879,"std":0.000958},{"mean":0.00375,"std":0.000523},{"mean":0.004065,"std":0.000429},{"mean":0.004902,"std":0.001304},{"mean":0.005292,"std":0.001382},{"mean":0.006256,"std":0.001917},{"mean":0.007247,"std":0.001821},{"mean":0.010484,"std":0.00129},{"mean":0.016589,"std":0.000708},{"mean":0.027508,"std":0.003866},{"mean":0.059331,"std":0.011574},{"mean":0.139421,"std":0.00969}],"accuracy_8":[{"mean":0.822848,"std":0.000458},{"mean":0.826364,"std":0.001012},{"mean":0.830303,"std":0.001377},{"mean":0.833758,"std":0.001839},{"mean":0.837758,"std":0.001211},{"mean":0.841394,"std":0.000859},{"mean":0.846242,"std":0.000555},{"mean":0.849455,"std":0.001012},{"mean":0.853455,"std":0.001273},{"mean":0.858,"std":0.001106},{"mean":0.862364,"std":0.000833},{"mean":0.866545,"std":0.00142},{"mean":0.872667,"std":0.001377},{"mean":0.877455,"std":0.001924},{"mean":0.883515,"std":0.001866},{"mean":0.888242,"std":0.00134},{"mean":0.893697,"std":0.000555},{"mean":0.898121,"std":0.002273},{"mean":0.895091,"std":0.005953},{"mean":0.860545,"std":0.009718}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.0,"std":0.0},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.002,"std":0.001732},{"mean":0.003,"std":0.001732},{"mean":0.004,"std":0.002},{"mean":0.004333,"std":0.002517},{"mean":0.006667,"std":0.002082},{"mean":0.008333,"std":0.001528},{"mean":0.011333,"std":0.004041},{"mean":0.013667,"std":0.004726},{"mean":0.016,"std":0.006},{"mean":0.020333,"std":0.005508},{"mean":0.032333,"std":0.004041},{"mean":0.056333,"std":0.002082},{"mean":0.101667,"std":0.013577},{"mean":0.237,"std":0.04293},{"mean":0.649,"std":0.043313}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432667,"std":0.001528},{"mean":0.433,"std":0.002},{"mean":0.433333,"std":0.002887},{"mean":0.433,"std":0.002646},{"mean":0.434,"std":0.001732},{"mean":0.435,"std":0.002646},{"mean":0.437,"std":0.001},{"mean":0.44,"std":0.001},{"mean":0.445667,"std":0.004041},{"mean":0.482,"std":0.020664},{"mean":0.649667,"std":0.043466}],"llm_call_rate":[{"mean":0.960788,"std":0.00305},{"mean":0.911273,"std":0.000833},{"mean":0.866364,"std":0.004639},{"mean":0.826667,"std":0.005912},{"mean":0.786788,"std":0.002723},{"mean":0.74703,"std":0.001525},{"mean":0.704909,"std":0.003963},{"mean":0.664485,"std":0.001155},{"mean":0.621091,"std":0.000182},{"mean":0.579576,"std":0.003135},{"mean":0.538061,"std":0.006053},{"mean":0.494,"std":0.005068},{"mean":0.451273,"std":0.005671},{"mean":0.410606,"std":0.00856},{"mean":0.364909,"std":0.004465},{"mean":0.318121,"std":0.003611},{"mean":0.269212,"std":0.004846},{"mean":0.211879,"std":0.008939},{"mean":0.130606,"std":0.007264},{"mean":0.001091,"std":0.001311}]}}}}},"modernbert/k5":{"model":"modernbert","k":5,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["ModernBERT-base-k5-seed42","ModernBERT-base-k5-seed43","ModernBERT-base-k5-seed44"],"logits_sha256":["8ffbbf00d37abfc1fc2e3601e96ed26c71c6cb2452c8c88d32375674f22996c1","561ef2079fd97de9a221e029b2589c50ccd17a579423fa9503ef95fe2944730c","9c6b23ca81db2a58e2f444a13533edcf9687f6d4020ba6ec6ecc52915f26e784"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.007273,"std":0.006298},{"mean":0.013333,"std":0.006298},{"mean":0.019091,"std":0.007441},{"mean":0.024485,"std":0.014295},{"mean":0.032525,"std":0.018155},{"mean":0.043636,"std":0.020838},{"mean":0.054394,"std":0.022074},{"mean":0.067609,"std":0.026416},{"mean":0.082303,"std":0.030925},{"mean":0.101047,"std":0.032599},{"mean":0.119091,"std":0.036122},{"mean":0.13669,"std":0.03683},{"mean":0.157835,"std":0.041628},{"mean":0.181414,"std":0.044874},{"mean":0.204545,"std":0.045238},{"mean":0.22795,"std":0.043413},{"mean":0.251515,"std":0.04142},{"mean":0.275662,"std":0.041054},{"mean":0.300545,"std":0.041032}],"operating":{"tau":[{"mean":0.992054,"std":0.003359},{"mean":0.978434,"std":0.009836},{"mean":0.9581,"std":0.017646},{"mean":0.924451,"std":0.031772},{"mean":0.878824,"std":0.051397},{"mean":0.823344,"std":0.070193},{"mean":0.764557,"std":0.079302},{"mean":0.69754,"std":0.090735},{"mean":0.630925,"std":0.089267},{"mean":0.568431,"std":0.0831},{"mean":0.499097,"std":0.082568},{"mean":0.43943,"std":0.075213},{"mean":0.386689,"std":0.068373},{"mean":0.340915,"std":0.060505},{"mean":0.293787,"std":0.050125},{"mean":0.247868,"std":0.039064},{"mean":0.207198,"std":0.027485},{"mean":0.172273,"std":0.020891},{"mean":0.134004,"std":0.013426},{"mean":0.05089,"std":0.006766}],"coverage":[{"mean":0.045697,"std":0.005932},{"mean":0.086788,"std":0.011305},{"mean":0.126121,"std":0.012723},{"mean":0.167394,"std":0.010934},{"mean":0.210545,"std":0.010792},{"mean":0.256909,"std":0.008456},{"mean":0.299939,"std":0.010919},{"mean":0.344909,"std":0.007086},{"mean":0.39097,"std":0.004338},{"mean":0.434424,"std":0.001277},{"mean":0.48697,"std":0.009621},{"mean":0.536121,"std":0.012461},{"mean":0.582667,"std":0.014364},{"mean":0.629515,"std":0.016743},{"mean":0.680303,"std":0.018483},{"mean":0.737939,"std":0.013614},{"mean":0.797515,"std":0.0157},{"mean":0.856303,"std":0.015759},{"mean":0.922788,"std":0.009346},{"mean":0.999818,"std":0.000182}],"selective_risk":[{"mean":0.001431,"std":0.002478},{"mean":0.00416,"std":0.004437},{"mean":0.009489,"std":0.006784},{"mean":0.015703,"std":0.004602},{"mean":0.019704,"std":0.008102},{"mean":0.025825,"std":0.014147},{"mean":0.03168,"std":0.015362},{"mean":0.043186,"std":0.02126},{"mean":0.052753,"std":0.021316},{"mean":0.064125,"std":0.025665},{"mean":0.078595,"std":0.032993},{"mean":0.096541,"std":0.036612},{"mean":0.112829,"std":0.040455},{"mean":0.129668,"std":0.04094},{"mean":0.150631,"std":0.047923},{"mean":0.176371,"std":0.050895},{"mean":0.202569,"std":0.051585},{"mean":0.23094,"std":0.047704},{"mean":0.262475,"std":0.043694},{"mean":0.300413,"std":0.041164}],"accuracy_8":[{"mean":0.82303,"std":0.001211},{"mean":0.826,"std":0.001924},{"mean":0.828242,"std":0.001728},{"mean":0.83103,"std":0.002989},{"mean":0.835333,"std":0.004818},{"mean":0.838364,"std":0.006909},{"mean":0.840909,"std":0.007002},{"mean":0.841394,"std":0.008301},{"mean":0.841758,"std":0.00913},{"mean":0.841576,"std":0.011708},{"mean":0.841333,"std":0.015839},{"mean":0.836606,"std":0.018592},{"mean":0.830606,"std":0.022338},{"mean":0.823758,"std":0.025631},{"mean":0.813879,"std":0.031684},{"mean":0.798242,"std":0.03785},{"mean":0.779515,"std":0.040624},{"mean":0.759636,"std":0.039364},{"mean":0.734909,"std":0.038982},{"mean":0.699515,"std":0.041114}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.001,"std":0.001},{"mean":0.003,"std":0.001},{"mean":0.006333,"std":0.001528},{"mean":0.010333,"std":0.004041},{"mean":0.014667,"std":0.008021},{"mean":0.02,"std":0.009165},{"mean":0.032667,"std":0.01222},{"mean":0.047,"std":0.016093},{"mean":0.063667,"std":0.02203},{"mean":0.087,"std":0.03061},{"mean":0.118333,"std":0.038553},{"mean":0.152333,"std":0.046004},{"mean":0.191667,"std":0.04817},{"mean":0.248667,"std":0.064686},{"mean":0.321,"std":0.071084},{"mean":0.411333,"std":0.077835},{"mean":0.522,"std":0.071687},{"mean":0.659,"std":0.054836},{"mean":0.851667,"std":0.043501}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.001155},{"mean":0.433,"std":0.003},{"mean":0.434,"std":0.004583},{"mean":0.435333,"std":0.004041},{"mean":0.441,"std":0.004583},{"mean":0.444667,"std":0.007638},{"mean":0.452333,"std":0.007506},{"mean":0.458667,"std":0.015044},{"mean":0.471,"std":0.017349},{"mean":0.489,"std":0.022539},{"mean":0.504,"std":0.025357},{"mean":0.531,"std":0.04015},{"mean":0.569,"std":0.047571},{"mean":0.620667,"std":0.054519},{"mean":0.678333,"std":0.050639},{"mean":0.746667,"std":0.043317},{"mean":0.852333,"std":0.043004}],"llm_call_rate":[{"mean":0.954303,"std":0.005932},{"mean":0.913212,"std":0.011305},{"mean":0.873879,"std":0.012723},{"mean":0.832606,"std":0.010934},{"mean":0.789455,"std":0.010792},{"mean":0.743091,"std":0.008456},{"mean":0.700061,"std":0.010919},{"mean":0.655091,"std":0.007086},{"mean":0.60903,"std":0.004338},{"mean":0.565576,"std":0.001277},{"mean":0.51303,"std":0.009621},{"mean":0.463879,"std":0.012461},{"mean":0.417333,"std":0.014364},{"mean":0.370485,"std":0.016743},{"mean":0.319697,"std":0.018483},{"mean":0.262061,"std":0.013614},{"mean":0.202485,"std":0.0157},{"mean":0.143697,"std":0.015759},{"mean":0.077212,"std":0.009346},{"mean":0.000182,"std":0.000182}]}},"entropy":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.007273,"std":0.006298},{"mean":0.012929,"std":0.00622},{"mean":0.018485,"std":0.006188},{"mean":0.023273,"std":0.013469},{"mean":0.032323,"std":0.015408},{"mean":0.042424,"std":0.018947},{"mean":0.054091,"std":0.020696},{"mean":0.066128,"std":0.023477},{"mean":0.079394,"std":0.028541},{"mean":0.097521,"std":0.035951},{"mean":0.115556,"std":0.03797},{"mean":0.134266,"std":0.040072},{"mean":0.155758,"std":0.04062},{"mean":0.177374,"std":0.042363},{"mean":0.202879,"std":0.042202},{"mean":0.22574,"std":0.042291},{"mean":0.250505,"std":0.043235},{"mean":0.274577,"std":0.041956},{"mean":0.300545,"std":0.041032}],"operating":{"tau":[{"mean":-0.072383,"std":0.027793},{"mean":-0.166459,"std":0.064976},{"mean":-0.30519,"std":0.108948},{"mean":-0.498287,"std":0.167083},{"mean":-0.719441,"std":0.251271},{"mean":-0.968608,"std":0.298663},{"mean":-1.219861,"std":0.333972},{"mean":-1.454001,"std":0.342396},{"mean":-1.710167,"std":0.356187},{"mean":-1.955836,"std":0.344669},{"mean":-2.169875,"std":0.334869},{"mean":-2.40032,"std":0.320818},{"mean":-2.608627,"std":0.279904},{"mean":-2.820878,"std":0.235573},{"mean":-2.99992,"std":0.2039},{"mean":-3.160972,"std":0.165352},{"mean":-3.337423,"std":0.120543},{"mean":-3.500273,"std":0.095649},{"mean":-3.681308,"std":0.078646},{"mean":-4.249237,"std":0.100272}],"coverage":[{"mean":0.045515,"std":0.005623},{"mean":0.085697,"std":0.010372},{"mean":0.127152,"std":0.011779},{"mean":0.167273,"std":0.011818},{"mean":0.210182,"std":0.008178},{"mean":0.257333,"std":0.008498},{"mean":0.302061,"std":0.007598},{"mean":0.342667,"std":0.008887},{"mean":0.390788,"std":0.006686},{"mean":0.437636,"std":0.004727},{"mean":0.480788,"std":0.005912},{"mean":0.528,"std":0.009096},{"mean":0.575879,"std":0.012313},{"mean":0.629515,"std":0.011897},{"mean":0.67903,"std":0.014275},{"mean":0.728545,"std":0.011653},{"mean":0.791394,"std":0.009819},{"mean":0.848485,"std":0.012676},{"mean":0.919758,"std":0.012385},{"mean":0.999697,"std":0.000525}],"selective_risk":[{"mean":0.001425,"std":0.002467},{"mean":0.004251,"std":0.004538},{"mean":0.009874,"std":0.007467},{"mean":0.016151,"std":0.004508},{"mean":0.019399,"std":0.007435},{"mean":0.025089,"std":0.012337},{"mean":0.032985,"std":0.014311},{"mean":0.040463,"std":0.017483},{"mean":0.052598,"std":0.020928},{"mean":0.062612,"std":0.022203},{"mean":0.07402,"std":0.027771},{"mean":0.089908,"std":0.037619},{"mean":0.106339,"std":0.041131},{"mean":0.126702,"std":0.043145},{"mean":0.147501,"std":0.047162},{"mean":0.169163,"std":0.046071},{"mean":0.197602,"std":0.04709},{"mean":0.225373,"std":0.045796},{"mean":0.259918,"std":0.046872},{"mean":0.300322,"std":0.041334}],"accuracy_8":[{"mean":0.82297,"std":0.001155},{"mean":0.825818,"std":0.001616},{"mean":0.828364,"std":0.001734},{"mean":0.831273,"std":0.002678},{"mean":0.83503,"std":0.004455},{"mean":0.838727,"std":0.00629},{"mean":0.840485,"std":0.006767},{"mean":0.842,"std":0.007825},{"mean":0.842,"std":0.009838},{"mean":0.842545,"std":0.009764},{"mean":0.840848,"std":0.012361},{"mean":0.837152,"std":0.019603},{"mean":0.832485,"std":0.022794},{"mean":0.823636,"std":0.024857},{"mean":0.813939,"std":0.029455},{"mean":0.802364,"std":0.031907},{"mean":0.78303,"std":0.036185},{"mean":0.763818,"std":0.036888},{"mean":0.736121,"std":0.042133},{"mean":0.699636,"std":0.041278}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.001,"std":0.001},{"mean":0.003,"std":0.001},{"mean":0.006333,"std":0.000577},{"mean":0.008667,"std":0.002309},{"mean":0.014,"std":0.007},{"mean":0.017667,"std":0.008505},{"mean":0.026,"std":0.011136},{"mean":0.038,"std":0.012124},{"mean":0.055,"std":0.019313},{"mean":0.071333,"std":0.024583},{"mean":0.097,"std":0.035763},{"mean":0.125667,"std":0.0506},{"mean":0.172333,"std":0.053594},{"mean":0.220667,"std":0.062581},{"mean":0.282,"std":0.054562},{"mean":0.377667,"std":0.065126},{"mean":0.482333,"std":0.06133},{"mean":0.635667,"std":0.065577},{"mean":0.851333,"std":0.044523}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.001155},{"mean":0.433,"std":0.002},{"mean":0.433,"std":0.003},{"mean":0.434667,"std":0.004041},{"mean":0.437333,"std":0.004163},{"mean":0.440333,"std":0.003786},{"mean":0.446667,"std":0.006429},{"mean":0.452,"std":0.007937},{"mean":0.461,"std":0.015716},{"mean":0.472,"std":0.023516},{"mean":0.494667,"std":0.023756},{"mean":0.518333,"std":0.032808},{"mean":0.546667,"std":0.033232},{"mean":0.595667,"std":0.045797},{"mean":0.649,"std":0.042509},{"mean":0.729667,"std":0.048003},{"mean":0.852,"std":0.043509}],"llm_call_rate":[{"mean":0.954485,"std":0.005623},{"mean":0.914303,"std":0.010372},{"mean":0.872848,"std":0.011779},{"mean":0.832727,"std":0.011818},{"mean":0.789818,"std":0.008178},{"mean":0.742667,"std":0.008498},{"mean":0.697939,"std":0.007598},{"mean":0.657333,"std":0.008887},{"mean":0.609212,"std":0.006686},{"mean":0.562364,"std":0.004727},{"mean":0.519212,"std":0.005912},{"mean":0.472,"std":0.009096},{"mean":0.424121,"std":0.012313},{"mean":0.370485,"std":0.011897},{"mean":0.32097,"std":0.014275},{"mean":0.271455,"std":0.011653},{"mean":0.208606,"std":0.009819},{"mean":0.151515,"std":0.012676},{"mean":0.080242,"std":0.012385},{"mean":0.000303,"std":0.000525}]}},"margin":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.008485,"std":0.006884},{"mean":0.014949,"std":0.007102},{"mean":0.022121,"std":0.014947},{"mean":0.030788,"std":0.01746},{"mean":0.04101,"std":0.022105},{"mean":0.051602,"std":0.026244},{"mean":0.07197,"std":0.031679},{"mean":0.088485,"std":0.034043},{"mean":0.108,"std":0.038265},{"mean":0.129697,"std":0.042011},{"mean":0.152626,"std":0.04309},{"mean":0.173613,"std":0.043662},{"mean":0.194978,"std":0.047495},{"mean":0.214545,"std":0.045668},{"mean":0.233333,"std":0.044138},{"mean":0.251979,"std":0.044812},{"mean":0.267205,"std":0.043686},{"mean":0.284466,"std":0.042281},{"mean":0.300545,"std":0.041032}],"operating":{"tau":[{"mean":7.214479,"std":0.712888},{"mean":5.998887,"std":0.665602},{"mean":5.15003,"std":0.647515},{"mean":4.477897,"std":0.619416},{"mean":3.877443,"std":0.595376},{"mean":3.383128,"std":0.590394},{"mean":2.933454,"std":0.526573},{"mean":2.536103,"std":0.458694},{"mean":2.174813,"std":0.398096},{"mean":1.848228,"std":0.351823},{"mean":1.570818,"std":0.35613},{"mean":1.331647,"std":0.294686},{"mean":1.078254,"std":0.239175},{"mean":0.879064,"std":0.193745},{"mean":0.704645,"std":0.153989},{"mean":0.551991,"std":0.130239},{"mean":0.394323,"std":0.089645},{"mean":0.247306,"std":0.052669},{"mean":0.116959,"std":0.027347},{"mean":0.000497,"std":0.000427}],"coverage":[{"mean":0.044667,"std":0.009028},{"mean":0.086667,"std":0.009966},{"mean":0.12903,"std":0.009987},{"mean":0.168061,"std":0.0123},{"mean":0.211758,"std":0.01158},{"mean":0.255152,"std":0.00816},{"mean":0.300909,"std":0.007428},{"mean":0.346909,"std":0.003779},{"mean":0.395091,"std":0.006195},{"mean":0.440545,"std":0.001753},{"mean":0.489576,"std":0.015065},{"mean":0.541152,"std":0.011847},{"mean":0.601576,"std":0.014768},{"mean":0.651515,"std":0.018742},{"mean":0.702061,"std":0.017974},{"mean":0.755697,"std":0.022072},{"mean":0.815394,"std":0.014434},{"mean":0.877576,"std":0.005912},{"mean":0.936303,"std":0.007},{"mean":0.999515,"std":0.000378}],"selective_risk":[{"mean":0.001536,"std":0.002661},{"mean":0.006429,"std":0.003973},{"mean":0.011387,"std":0.004253},{"mean":0.01717,"std":0.008553},{"mean":0.022967,"std":0.014141},{"mean":0.031248,"std":0.017528},{"mean":0.041338,"std":0.021284},{"mean":0.051406,"std":0.026011},{"mean":0.070239,"std":0.031294},{"mean":0.085095,"std":0.033493},{"mean":0.104426,"std":0.043089},{"mean":0.126355,"std":0.046712},{"mean":0.152543,"std":0.048213},{"mean":0.17575,"std":0.05167},{"mean":0.195901,"std":0.052061},{"mean":0.216001,"std":0.051794},{"mean":0.240222,"std":0.048559},{"mean":0.260339,"std":0.045375},{"mean":0.280196,"std":0.044179},{"mean":0.300389,"std":0.040972}],"accuracy_8":[{"mean":0.822848,"std":0.000933},{"mean":0.825636,"std":0.002204},{"mean":0.828667,"std":0.001927},{"mean":0.831818,"std":0.004252},{"mean":0.834788,"std":0.005853},{"mean":0.836788,"std":0.006632},{"mean":0.838788,"std":0.008429},{"mean":0.839818,"std":0.010982},{"mean":0.837818,"std":0.014059},{"mean":0.836424,"std":0.015439},{"mean":0.831818,"std":0.021684},{"mean":0.824303,"std":0.024844},{"mean":0.813697,"std":0.028247},{"mean":0.802909,"std":0.032377},{"mean":0.790545,"std":0.035546},{"mean":0.776424,"std":0.038862},{"mean":0.758,"std":0.038339},{"mean":0.740727,"std":0.039273},{"mean":0.721212,"std":0.041},{"mean":0.699576,"std":0.041064}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.001333,"std":0.001155},{"mean":0.004333,"std":0.001155},{"mean":0.008,"std":0.002646},{"mean":0.013333,"std":0.006506},{"mean":0.020667,"std":0.008963},{"mean":0.032333,"std":0.013051},{"mean":0.046333,"std":0.017243},{"mean":0.071,"std":0.019698},{"mean":0.094333,"std":0.027683},{"mean":0.136,"std":0.045299},{"mean":0.187,"std":0.058103},{"mean":0.255,"std":0.069022},{"mean":0.319667,"std":0.075659},{"mean":0.387667,"std":0.070607},{"mean":0.455333,"std":0.068973},{"mean":0.551333,"std":0.066161},{"mean":0.64,"std":0.058949},{"mean":0.74,"std":0.056},{"mean":0.851667,"std":0.041501}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.433667,"std":0.002517},{"mean":0.434667,"std":0.003512},{"mean":0.438,"std":0.003606},{"mean":0.443,"std":0.005568},{"mean":0.446667,"std":0.008622},{"mean":0.456667,"std":0.009292},{"mean":0.466,"std":0.013115},{"mean":0.484667,"std":0.02442},{"mean":0.513333,"std":0.03402},{"mean":0.545667,"std":0.040526},{"mean":0.573333,"std":0.047816},{"mean":0.612,"std":0.048135},{"mean":0.645,"std":0.049275},{"mean":0.698667,"std":0.044658},{"mean":0.741,"std":0.05003},{"mean":0.794333,"std":0.047035},{"mean":0.852333,"std":0.042501}],"llm_call_rate":[{"mean":0.955333,"std":0.009028},{"mean":0.913333,"std":0.009966},{"mean":0.87097,"std":0.009987},{"mean":0.831939,"std":0.0123},{"mean":0.788242,"std":0.01158},{"mean":0.744848,"std":0.00816},{"mean":0.699091,"std":0.007428},{"mean":0.653091,"std":0.003779},{"mean":0.604909,"std":0.006195},{"mean":0.559455,"std":0.001753},{"mean":0.510424,"std":0.015065},{"mean":0.458848,"std":0.011847},{"mean":0.398424,"std":0.014768},{"mean":0.348485,"std":0.018742},{"mean":0.297939,"std":0.017974},{"mean":0.244303,"std":0.022072},{"mean":0.184606,"std":0.014434},{"mean":0.122424,"std":0.005912},{"mean":0.063697,"std":0.007},{"mean":0.000485,"std":0.000378}]}},"msp_t":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.006667,"std":0.005249},{"mean":0.013333,"std":0.006298},{"mean":0.018182,"std":0.007441},{"mean":0.02497,"std":0.01457},{"mean":0.032525,"std":0.018002},{"mean":0.044156,"std":0.020616},{"mean":0.056364,"std":0.021494},{"mean":0.068283,"std":0.026853},{"mean":0.084848,"std":0.030472},{"mean":0.1027,"std":0.032636},{"mean":0.120404,"std":0.035624},{"mean":0.138462,"std":0.036367},{"mean":0.158874,"std":0.041604},{"mean":0.182788,"std":0.04388},{"mean":0.204621,"std":0.044805},{"mean":0.228235,"std":0.043191},{"mean":0.251313,"std":0.041648},{"mean":0.275726,"std":0.040966},{"mean":0.300545,"std":0.041032}],"operating":{"tau":[{"mean":0.995706,"std":0.003586},{"mean":0.987373,"std":0.009977},{"mean":0.973926,"std":0.018869},{"mean":0.949638,"std":0.035401},{"mean":0.914597,"std":0.057341},{"mean":0.870329,"std":0.082377},{"mean":0.82198,"std":0.094471},{"mean":0.762286,"std":0.111569},{"mean":0.699601,"std":0.113909},{"mean":0.635196,"std":0.111133},{"mean":0.565536,"std":0.112008},{"mean":0.500566,"std":0.102722},{"mean":0.445042,"std":0.097224},{"mean":0.393977,"std":0.085969},{"mean":0.340579,"std":0.07431},{"mean":0.289517,"std":0.06138},{"mean":0.242881,"std":0.047352},{"mean":0.201282,"std":0.036721},{"mean":0.155756,"std":0.02568},{"mean":0.056662,"std":0.006801}],"coverage":[{"mean":0.045212,"std":0.005611},{"mean":0.086242,"std":0.010722},{"mean":0.125697,"std":0.012435},{"mean":0.167758,"std":0.010114},{"mean":0.211394,"std":0.010742},{"mean":0.256242,"std":0.005819},{"mean":0.29897,"std":0.011467},{"mean":0.344485,"std":0.008293},{"mean":0.389939,"std":0.002123},{"mean":0.433879,"std":0.002169},{"mean":0.487091,"std":0.009585},{"mean":0.536364,"std":0.010982},{"mean":0.582545,"std":0.014491},{"mean":0.630061,"std":0.016001},{"mean":0.682909,"std":0.017163},{"mean":0.738061,"std":0.015301},{"mean":0.797939,"std":0.015327},{"mean":0.856606,"std":0.013739},{"mean":0.922606,"std":0.008637},{"mean":0.999818,"std":0.000182}],"selective_risk":[{"mean":0.001418,"std":0.002457},{"mean":0.004953,"std":0.003427},{"mean":0.010373,"std":0.005789},{"mean":0.016033,"std":0.004195},{"mean":0.019325,"std":0.007918},{"mean":0.025934,"std":0.013792},{"mean":0.032403,"std":0.015136},{"mean":0.043015,"std":0.020714},{"mean":0.053739,"std":0.021206},{"mean":0.064759,"std":0.026424},{"mean":0.080317,"std":0.033141},{"mean":0.098497,"std":0.035658},{"mean":0.114243,"std":0.039647},{"mean":0.13004,"std":0.04084},{"mean":0.153619,"std":0.046209},{"mean":0.177409,"std":0.051481},{"mean":0.203488,"std":0.050573},{"mean":0.231365,"std":0.046945},{"mean":0.262813,"std":0.043126},{"mean":0.300413,"std":0.041164}],"accuracy_8":[{"mean":0.82297,"std":0.001155},{"mean":0.826,"std":0.001967},{"mean":0.828061,"std":0.001766},{"mean":0.831212,"std":0.002962},{"mean":0.835394,"std":0.004914},{"mean":0.838182,"std":0.006735},{"mean":0.840545,"std":0.006909},{"mean":0.841333,"std":0.008277},{"mean":0.841697,"std":0.009373},{"mean":0.841576,"std":0.011645},{"mean":0.841091,"std":0.016015},{"mean":0.835758,"std":0.017835},{"mean":0.830364,"std":0.022202},{"mean":0.824,"std":0.026027},{"mean":0.812182,"std":0.031222},{"mean":0.797697,"std":0.038458},{"mean":0.778848,"std":0.039913},{"mean":0.758909,"std":0.038446},{"mean":0.734364,"std":0.038269},{"mean":0.699515,"std":0.041114}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.001,"std":0.001},{"mean":0.003333,"std":0.001155},{"mean":0.006667,"std":0.001155},{"mean":0.010667,"std":0.004163},{"mean":0.015,"std":0.00755},{"mean":0.021,"std":0.008544},{"mean":0.033,"std":0.01179},{"mean":0.048667,"std":0.015011},{"mean":0.064333,"std":0.022121},{"mean":0.09,"std":0.030512},{"mean":0.125667,"std":0.037687},{"mean":0.156333,"std":0.044004},{"mean":0.194667,"std":0.047004},{"mean":0.256667,"std":0.060501},{"mean":0.325333,"std":0.072037},{"mean":0.416,"std":0.075319},{"mean":0.524333,"std":0.069082},{"mean":0.661333,"std":0.053201},{"mean":0.851667,"std":0.043501}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432667,"std":0.000577},{"mean":0.432333,"std":0.001155},{"mean":0.433333,"std":0.003055},{"mean":0.434333,"std":0.004163},{"mean":0.436,"std":0.003464},{"mean":0.440667,"std":0.005033},{"mean":0.445667,"std":0.007024},{"mean":0.452,"std":0.00755},{"mean":0.460333,"std":0.014468},{"mean":0.475333,"std":0.015308},{"mean":0.489667,"std":0.021962},{"mean":0.504667,"std":0.024583},{"mean":0.536333,"std":0.037608},{"mean":0.572667,"std":0.047962},{"mean":0.625,"std":0.052259},{"mean":0.682,"std":0.048135},{"mean":0.750333,"std":0.040501},{"mean":0.852333,"std":0.043004}],"llm_call_rate":[{"mean":0.954788,"std":0.005611},{"mean":0.913758,"std":0.010722},{"mean":0.874303,"std":0.012435},{"mean":0.832242,"std":0.010114},{"mean":0.788606,"std":0.010742},{"mean":0.743758,"std":0.005819},{"mean":0.70103,"std":0.011467},{"mean":0.655515,"std":0.008293},{"mean":0.610061,"std":0.002123},{"mean":0.566121,"std":0.002169},{"mean":0.512909,"std":0.009585},{"mean":0.463636,"std":0.010982},{"mean":0.417455,"std":0.014491},{"mean":0.369939,"std":0.016001},{"mean":0.317091,"std":0.017163},{"mean":0.261939,"std":0.015301},{"mean":0.202061,"std":0.015327},{"mean":0.143394,"std":0.013739},{"mean":0.077394,"std":0.008637},{"mean":0.000182,"std":0.000182}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.005455,"std":0.00481},{"mean":0.009697,"std":0.004848},{"mean":0.012727,"std":0.004724},{"mean":0.016,"std":0.004542},{"mean":0.021616,"std":0.009589},{"mean":0.02684,"std":0.013329},{"mean":0.032727,"std":0.015634},{"mean":0.043098,"std":0.018561},{"mean":0.054667,"std":0.021728},{"mean":0.069642,"std":0.027779},{"mean":0.087576,"std":0.033511},{"mean":0.107319,"std":0.036579},{"mean":0.129177,"std":0.036401},{"mean":0.151192,"std":0.037116},{"mean":0.178333,"std":0.037473},{"mean":0.204635,"std":0.037546},{"mean":0.232997,"std":0.037209},{"mean":0.261882,"std":0.035251},{"mean":0.290061,"std":0.033965}],"operating":{"tau":[{"mean":0.996197,"std":0.001293},{"mean":0.989622,"std":0.004334},{"mean":0.979568,"std":0.007722},{"mean":0.965115,"std":0.01442},{"mean":0.945299,"std":0.021965},{"mean":0.91966,"std":0.031392},{"mean":0.890212,"std":0.043458},{"mean":0.857209,"std":0.048836},{"mean":0.816531,"std":0.055422},{"mean":0.773888,"std":0.063691},{"mean":0.726989,"std":0.070324},{"mean":0.675083,"std":0.071047},{"mean":0.620237,"std":0.072113},{"mean":0.562876,"std":0.067977},{"mean":0.507724,"std":0.056712},{"mean":0.454671,"std":0.048125},{"mean":0.403604,"std":0.037284},{"mean":0.350321,"std":0.030803},{"mean":0.296555,"std":0.026376},{"mean":0.18799,"std":0.011881}],"coverage":[{"mean":0.047273,"std":0.00982},{"mean":0.088606,"std":0.014318},{"mean":0.129939,"std":0.012497},{"mean":0.167879,"std":0.008762},{"mean":0.208424,"std":0.003642},{"mean":0.254,"std":0.002886},{"mean":0.296909,"std":0.004502},{"mean":0.337212,"std":0.00488},{"mean":0.386242,"std":0.007491},{"mean":0.429394,"std":0.004312},{"mean":0.47,"std":0.006082},{"mean":0.515879,"std":0.004455},{"mean":0.564848,"std":0.008021},{"mean":0.616485,"std":0.013177},{"mean":0.667152,"std":0.011133},{"mean":0.721636,"std":0.012889},{"mean":0.77903,"std":0.01211},{"mean":0.845879,"std":0.013422},{"mean":0.917273,"std":0.017662},{"mean":0.998424,"std":0.000584}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.002641,"std":0.002342},{"mean":0.007614,"std":0.004997},{"mean":0.011654,"std":0.006245},{"mean":0.013423,"std":0.004544},{"mean":0.016463,"std":0.004714},{"mean":0.021577,"std":0.009785},{"mean":0.024994,"std":0.01245},{"mean":0.031204,"std":0.0142},{"mean":0.038074,"std":0.018914},{"mean":0.048622,"std":0.021089},{"mean":0.059764,"std":0.026285},{"mean":0.075772,"std":0.032316},{"mean":0.094194,"std":0.040614},{"mean":0.114588,"std":0.04069},{"mean":0.138591,"std":0.040627},{"mean":0.16714,"std":0.041531},{"mean":0.202434,"std":0.044415},{"mean":0.242699,"std":0.045306},{"mean":0.289292,"std":0.034227}],"accuracy_8":[{"mean":0.822848,"std":0.001001},{"mean":0.82497,"std":0.00183},{"mean":0.827515,"std":0.001546},{"mean":0.829879,"std":0.001821},{"mean":0.833152,"std":0.002522},{"mean":0.836606,"std":0.003968},{"mean":0.839818,"std":0.00531},{"mean":0.842303,"std":0.006531},{"mean":0.844848,"std":0.007276},{"mean":0.846121,"std":0.008044},{"mean":0.845818,"std":0.009291},{"mean":0.845758,"std":0.011151},{"mean":0.843636,"std":0.014227},{"mean":0.838667,"std":0.018564},{"mean":0.832303,"std":0.023341},{"mean":0.822485,"std":0.027731},{"mean":0.806727,"std":0.030591},{"mean":0.784242,"std":0.0368},{"mean":0.753818,"std":0.040848},{"mean":0.710788,"std":0.03424}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.001,"std":0.001},{"mean":0.003667,"std":0.001528},{"mean":0.006333,"std":0.001528},{"mean":0.008667,"std":0.001528},{"mean":0.012333,"std":0.003786},{"mean":0.019,"std":0.008185},{"mean":0.023,"std":0.009},{"mean":0.030667,"std":0.01106},{"mean":0.042667,"std":0.017502},{"mean":0.061,"std":0.023},{"mean":0.082333,"std":0.033171},{"mean":0.116,"std":0.04471},{"mean":0.156667,"std":0.062324},{"mean":0.211333,"std":0.058046},{"mean":0.276,"std":0.063095},{"mean":0.363667,"std":0.059685},{"mean":0.484333,"std":0.069788},{"mean":0.652333,"std":0.087535},{"mean":0.886,"std":0.044911}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432667,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.433,"std":0.001732},{"mean":0.434333,"std":0.002887},{"mean":0.435333,"std":0.004041},{"mean":0.436333,"std":0.004041},{"mean":0.439667,"std":0.004509},{"mean":0.447667,"std":0.008021},{"mean":0.452667,"std":0.011015},{"mean":0.467667,"std":0.017156},{"mean":0.484333,"std":0.026006},{"mean":0.507333,"std":0.030139},{"mean":0.535,"std":0.035086},{"mean":0.579,"std":0.037242},{"mean":0.642667,"std":0.050461},{"mean":0.741,"std":0.062378},{"mean":0.887333,"std":0.044004}],"llm_call_rate":[{"mean":0.952727,"std":0.00982},{"mean":0.911394,"std":0.014318},{"mean":0.870061,"std":0.012497},{"mean":0.832121,"std":0.008762},{"mean":0.791576,"std":0.003642},{"mean":0.746,"std":0.002886},{"mean":0.703091,"std":0.004502},{"mean":0.662788,"std":0.00488},{"mean":0.613758,"std":0.007491},{"mean":0.570606,"std":0.004312},{"mean":0.53,"std":0.006082},{"mean":0.484121,"std":0.004455},{"mean":0.435152,"std":0.008021},{"mean":0.383515,"std":0.013177},{"mean":0.332848,"std":0.011133},{"mean":0.278364,"std":0.012889},{"mean":0.22097,"std":0.01211},{"mean":0.154121,"std":0.013422},{"mean":0.082727,"std":0.017662},{"mean":0.001576,"std":0.000584}]}},"entropy":{"risk_coverage":[{"mean":0.0,"std":0.0},{"mean":0.004848,"std":0.003785},{"mean":0.010101,"std":0.005466},{"mean":0.014242,"std":0.005169},{"mean":0.01697,"std":0.005507},{"mean":0.022828,"std":0.0116},{"mean":0.02684,"std":0.013186},{"mean":0.035,"std":0.016818},{"mean":0.045522,"std":0.020336},{"mean":0.057576,"std":0.022663},{"mean":0.073168,"std":0.027785},{"mean":0.091717,"std":0.032273},{"mean":0.10965,"std":0.034968},{"mean":0.132727,"std":0.036268},{"mean":0.155313,"std":0.038844},{"mean":0.180682,"std":0.03817},{"mean":0.205847,"std":0.03727},{"mean":0.233468,"std":0.036543},{"mean":0.262392,"std":0.034995},{"mean":0.290061,"std":0.033965}],"operating":{"tau":[{"mean":-0.030958,"std":0.00938},{"mean":-0.073395,"std":0.026955},{"mean":-0.131161,"std":0.042795},{"mean":-0.202671,"std":0.07021},{"mean":-0.294722,"std":0.096639},{"mean":-0.400693,"std":0.124447},{"mean":-0.506983,"std":0.154593},{"mean":-0.621714,"std":0.159683},{"mean":-0.741469,"std":0.163743},{"mean":-0.859343,"std":0.170563},{"mean":-0.977254,"std":0.176521},{"mean":-1.096269,"std":0.159474},{"mean":-1.207085,"std":0.143856},{"mean":-1.313726,"std":0.134556},{"mean":-1.416101,"std":0.108664},{"mean":-1.50833,"std":0.086116},{"mean":-1.597718,"std":0.069908},{"mean":-1.68922,"std":0.054317},{"mean":-1.783406,"std":0.037909},{"mean":-2.016159,"std":0.00517}],"coverage":[{"mean":0.047091,"std":0.009111},{"mean":0.088424,"std":0.014037},{"mean":0.130182,"std":0.012629},{"mean":0.167212,"std":0.007938},{"mean":0.208848,"std":0.004107},{"mean":0.255212,"std":0.004312},{"mean":0.297152,"std":0.004536},{"mean":0.33897,"std":0.009102},{"mean":0.387697,"std":0.007043},{"mean":0.430485,"std":0.004462},{"mean":0.471273,"std":0.007948},{"mean":0.516545,"std":0.002799},{"mean":0.563818,"std":0.001575},{"mean":0.612788,"std":0.009193},{"mean":0.664061,"std":0.010686},{"mean":0.713576,"std":0.008265},{"mean":0.770667,"std":0.013959},{"mean":0.839091,"std":0.017232},{"mean":0.910485,"std":0.012916},{"mean":0.999455,"std":0.000364}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.003221,"std":0.002816},{"mean":0.008111,"std":0.00589},{"mean":0.011712,"std":0.006331},{"mean":0.014562,"std":0.004326},{"mean":0.017802,"std":0.005821},{"mean":0.022572,"std":0.011445},{"mean":0.025989,"std":0.012599},{"mean":0.033012,"std":0.015511},{"mean":0.041401,"std":0.018358},{"mean":0.050834,"std":0.022876},{"mean":0.062925,"std":0.023274},{"mean":0.079067,"std":0.029033},{"mean":0.096173,"std":0.035864},{"mean":0.11683,"std":0.040526},{"mean":0.139271,"std":0.040157},{"mean":0.165129,"std":0.04453},{"mean":0.199353,"std":0.04517},{"mean":0.239487,"std":0.043614},{"mean":0.289803,"std":0.033632}],"accuracy_8":[{"mean":0.822788,"std":0.000915},{"mean":0.82497,"std":0.00183},{"mean":0.827515,"std":0.001589},{"mean":0.829939,"std":0.001766},{"mean":0.833152,"std":0.002729},{"mean":0.836788,"std":0.003674},{"mean":0.840242,"std":0.005587},{"mean":0.842667,"std":0.006748},{"mean":0.845273,"std":0.007742},{"mean":0.845939,"std":0.008199},{"mean":0.845818,"std":0.00983},{"mean":0.846061,"std":0.010571},{"mean":0.842545,"std":0.014014},{"mean":0.837455,"std":0.018366},{"mean":0.830061,"std":0.024107},{"mean":0.821394,"std":0.026223},{"mean":0.808364,"std":0.03119},{"mean":0.785273,"std":0.036388},{"mean":0.756364,"std":0.038042},{"mean":0.710182,"std":0.03359}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.001333,"std":0.001528},{"mean":0.003667,"std":0.001528},{"mean":0.006333,"std":0.001528},{"mean":0.008667,"std":0.001528},{"mean":0.012333,"std":0.003786},{"mean":0.018667,"std":0.008083},{"mean":0.021333,"std":0.008505},{"mean":0.031,"std":0.012},{"mean":0.041,"std":0.014526},{"mean":0.053667,"std":0.019553},{"mean":0.076,"std":0.02623},{"mean":0.100667,"std":0.033975},{"mean":0.136667,"std":0.050213},{"mean":0.183667,"std":0.061044},{"mean":0.242333,"std":0.060178},{"mean":0.320667,"std":0.07259},{"mean":0.439,"std":0.079228},{"mean":0.612667,"std":0.087552},{"mean":0.888333,"std":0.041405}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432667,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.433,"std":0.001732},{"mean":0.434333,"std":0.002887},{"mean":0.434333,"std":0.004041},{"mean":0.436333,"std":0.003512},{"mean":0.439,"std":0.004},{"mean":0.444,"std":0.006},{"mean":0.449333,"std":0.009713},{"mean":0.457333,"std":0.011504},{"mean":0.474667,"std":0.018009},{"mean":0.492333,"std":0.028148},{"mean":0.513,"std":0.030512},{"mean":0.547333,"std":0.041645},{"mean":0.612333,"std":0.052367},{"mean":0.710333,"std":0.059181},{"mean":0.889,"std":0.041581}],"llm_call_rate":[{"mean":0.952909,"std":0.009111},{"mean":0.911576,"std":0.014037},{"mean":0.869818,"std":0.012629},{"mean":0.832788,"std":0.007938},{"mean":0.791152,"std":0.004107},{"mean":0.744788,"std":0.004312},{"mean":0.702848,"std":0.004536},{"mean":0.66103,"std":0.009102},{"mean":0.612303,"std":0.007043},{"mean":0.569515,"std":0.004462},{"mean":0.528727,"std":0.007948},{"mean":0.483455,"std":0.002799},{"mean":0.436182,"std":0.001575},{"mean":0.387212,"std":0.009193},{"mean":0.335939,"std":0.010686},{"mean":0.286424,"std":0.008265},{"mean":0.229333,"std":0.013959},{"mean":0.160909,"std":0.017232},{"mean":0.089515,"std":0.012916},{"mean":0.000545,"std":0.000364}]}},"margin":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.003636,"std":0.003636},{"mean":0.009697,"std":0.005555},{"mean":0.011818,"std":0.003963},{"mean":0.014788,"std":0.006354},{"mean":0.019596,"std":0.007887},{"mean":0.025628,"std":0.011689},{"mean":0.034091,"std":0.014309},{"mean":0.045791,"std":0.019412},{"mean":0.058788,"std":0.02316},{"mean":0.07405,"std":0.028596},{"mean":0.09303,"std":0.033943},{"mean":0.114033,"std":0.037933},{"mean":0.137489,"std":0.040393},{"mean":0.161293,"std":0.039696},{"mean":0.186439,"std":0.039466},{"mean":0.213262,"std":0.039622},{"mean":0.239259,"std":0.039136},{"mean":0.265518,"std":0.037758},{"mean":0.290061,"std":0.033965}],"operating":{"tau":[{"mean":7.412659,"std":0.671447},{"mean":6.249826,"std":0.68915},{"mean":5.521604,"std":0.625465},{"mean":4.903822,"std":0.639564},{"mean":4.357784,"std":0.633806},{"mean":3.931005,"std":0.598122},{"mean":3.486906,"std":0.564107},{"mean":3.123482,"std":0.525233},{"mean":2.796757,"std":0.520429},{"mean":2.464916,"std":0.480543},{"mean":2.171784,"std":0.443391},{"mean":1.87623,"std":0.408687},{"mean":1.602042,"std":0.393952},{"mean":1.316788,"std":0.337126},{"mean":1.056205,"std":0.289525},{"mean":0.791059,"std":0.203001},{"mean":0.557571,"std":0.156077},{"mean":0.350257,"std":0.10246},{"mean":0.162954,"std":0.052201},{"mean":0.000186,"std":0.00014}],"coverage":[{"mean":0.047152,"std":0.007757},{"mean":0.091091,"std":0.012001},{"mean":0.129455,"std":0.01118},{"mean":0.168606,"std":0.010742},{"mean":0.212727,"std":0.003988},{"mean":0.251152,"std":0.00864},{"mean":0.297939,"std":0.006516},{"mean":0.341818,"std":0.008572},{"mean":0.384727,"std":0.006203},{"mean":0.429333,"std":0.004676},{"mean":0.472909,"std":0.007098},{"mean":0.519758,"std":0.006188},{"mean":0.566121,"std":0.014231},{"mean":0.622121,"std":0.016846},{"mean":0.676545,"std":0.019342},{"mean":0.737879,"std":0.01599},{"mean":0.800485,"std":0.014143},{"mean":0.866545,"std":0.016111},{"mean":0.933697,"std":0.008788},{"mean":0.999939,"std":0.000105}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.00258,"std":0.00227},{"mean":0.006766,"std":0.004699},{"mean":0.010164,"std":0.003825},{"mean":0.012518,"std":0.003957},{"mean":0.015865,"std":0.006768},{"mean":0.019073,"std":0.00758},{"mean":0.024186,"std":0.010774},{"mean":0.031749,"std":0.012239},{"mean":0.040702,"std":0.018475},{"mean":0.052837,"std":0.021939},{"mean":0.065206,"std":0.02695},{"mean":0.081242,"std":0.036218},{"mean":0.10272,"std":0.042008},{"mean":0.127085,"std":0.049222},{"mean":0.155806,"std":0.044808},{"mean":0.18716,"std":0.045014},{"mean":0.221912,"std":0.045222},{"mean":0.256841,"std":0.041757},{"mean":0.290016,"std":0.03403}],"accuracy_8":[{"mean":0.822424,"std":0.000458},{"mean":0.824545,"std":0.001553},{"mean":0.826788,"std":0.001412},{"mean":0.829939,"std":0.001589},{"mean":0.83297,"std":0.002586},{"mean":0.835697,"std":0.004131},{"mean":0.838485,"std":0.004756},{"mean":0.841697,"std":0.005912},{"mean":0.843939,"std":0.006566},{"mean":0.844667,"std":0.008788},{"mean":0.843879,"std":0.009373},{"mean":0.843455,"std":0.012153},{"mean":0.840303,"std":0.017064},{"mean":0.835212,"std":0.021569},{"mean":0.825697,"std":0.027708},{"mean":0.812788,"std":0.030541},{"mean":0.793576,"std":0.033325},{"mean":0.769091,"std":0.036851},{"mean":0.740788,"std":0.03803},{"mean":0.71,"std":0.034052}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.001,"std":0.001},{"mean":0.003333,"std":0.001528},{"mean":0.006667,"std":0.001155},{"mean":0.009,"std":0.002646},{"mean":0.014667,"std":0.005859},{"mean":0.019,"std":0.00781},{"mean":0.026,"std":0.010536},{"mean":0.039333,"std":0.014048},{"mean":0.052,"std":0.020075},{"mean":0.076667,"std":0.029006},{"mean":0.106333,"std":0.039501},{"mean":0.141,"std":0.051565},{"mean":0.199333,"std":0.063058},{"mean":0.265,"std":0.080554},{"mean":0.355,"std":0.061539},{"mean":0.461667,"std":0.068135},{"mean":0.594667,"std":0.074715},{"mean":0.738,"std":0.068695},{"mean":0.889667,"std":0.043466}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.001155},{"mean":0.432333,"std":0.001528},{"mean":0.434,"std":0.002646},{"mean":0.435333,"std":0.003786},{"mean":0.437333,"std":0.004041},{"mean":0.441333,"std":0.003512},{"mean":0.444,"std":0.005},{"mean":0.454667,"std":0.009609},{"mean":0.465,"std":0.017059},{"mean":0.481333,"std":0.022811},{"mean":0.508,"std":0.030512},{"mean":0.54,"std":0.041146},{"mean":0.586333,"std":0.039107},{"mean":0.646667,"std":0.040204},{"mean":0.719333,"std":0.049095},{"mean":0.801333,"std":0.054243},{"mean":0.889667,"std":0.043466}],"llm_call_rate":[{"mean":0.952848,"std":0.007757},{"mean":0.908909,"std":0.012001},{"mean":0.870545,"std":0.01118},{"mean":0.831394,"std":0.010742},{"mean":0.787273,"std":0.003988},{"mean":0.748848,"std":0.00864},{"mean":0.702061,"std":0.006516},{"mean":0.658182,"std":0.008572},{"mean":0.615273,"std":0.006203},{"mean":0.570667,"std":0.004676},{"mean":0.527091,"std":0.007098},{"mean":0.480242,"std":0.006188},{"mean":0.433879,"std":0.014231},{"mean":0.377879,"std":0.016846},{"mean":0.323455,"std":0.019342},{"mean":0.262121,"std":0.01599},{"mean":0.199515,"std":0.014143},{"mean":0.133455,"std":0.016111},{"mean":0.066303,"std":0.008788},{"mean":6.1e-05,"std":0.000105}]}},"msp_t":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.005455,"std":0.00481},{"mean":0.009697,"std":0.005555},{"mean":0.01303,"std":0.005249},{"mean":0.015273,"std":0.005244},{"mean":0.021212,"std":0.00933},{"mean":0.027186,"std":0.013268},{"mean":0.033485,"std":0.01539},{"mean":0.043502,"std":0.01909},{"mean":0.054909,"std":0.022191},{"mean":0.070854,"std":0.027607},{"mean":0.088485,"std":0.033789},{"mean":0.107972,"std":0.036936},{"mean":0.129437,"std":0.037285},{"mean":0.153051,"std":0.036769},{"mean":0.178182,"std":0.038336},{"mean":0.204349,"std":0.038235},{"mean":0.232458,"std":0.037834},{"mean":0.260925,"std":0.035853},{"mean":0.290061,"std":0.033965}],"operating":{"tau":[{"mean":0.998138,"std":0.00136},{"mean":0.994304,"std":0.00439},{"mean":0.988221,"std":0.008118},{"mean":0.978482,"std":0.015753},{"mean":0.96435,"std":0.025192},{"mean":0.945539,"std":0.036003},{"mean":0.921075,"std":0.050601},{"mean":0.8949,"std":0.05976},{"mean":0.860493,"std":0.067766},{"mean":0.823055,"std":0.08059},{"mean":0.779508,"std":0.087164},{"mean":0.728553,"std":0.090983},{"mean":0.672361,"std":0.094917},{"mean":0.61269,"std":0.093257},{"mean":0.551924,"std":0.079486},{"mean":0.489438,"std":0.065495},{"mean":0.435686,"std":0.056495},{"mean":0.376608,"std":0.046674},{"mean":0.315244,"std":0.039656},{"mean":0.190046,"std":0.009712}],"coverage":[{"mean":0.046788,"std":0.009209},{"mean":0.088848,"std":0.013771},{"mean":0.130182,"std":0.013585},{"mean":0.167394,"std":0.008459},{"mean":0.209394,"std":0.004791},{"mean":0.254545,"std":0.004556},{"mean":0.298364,"std":0.005953},{"mean":0.338606,"std":0.007543},{"mean":0.387273,"std":0.006966},{"mean":0.428909,"std":0.005818},{"mean":0.468727,"std":0.006964},{"mean":0.515576,"std":0.00382},{"mean":0.565091,"std":0.00732},{"mean":0.615515,"std":0.015432},{"mean":0.669455,"std":0.010045},{"mean":0.725636,"std":0.010383},{"mean":0.781758,"std":0.010994},{"mean":0.847152,"std":0.014756},{"mean":0.917939,"std":0.018734},{"mean":0.99897,"std":0.000105}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.003385,"std":0.003342},{"mean":0.007633,"std":0.005083},{"mean":0.011308,"std":0.005596},{"mean":0.012787,"std":0.005028},{"mean":0.016173,"std":0.006124},{"mean":0.020856,"std":0.009739},{"mean":0.025391,"std":0.012621},{"mean":0.031454,"std":0.013762},{"mean":0.038514,"std":0.018261},{"mean":0.049266,"std":0.020214},{"mean":0.060296,"std":0.025306},{"mean":0.076511,"std":0.032926},{"mean":0.095522,"std":0.041982},{"mean":0.11675,"std":0.04076},{"mean":0.140858,"std":0.038527},{"mean":0.167931,"std":0.042499},{"mean":0.203185,"std":0.045433},{"mean":0.242507,"std":0.045642},{"mean":0.28945,"std":0.034086}],"accuracy_8":[{"mean":0.822667,"std":0.00082},{"mean":0.824909,"std":0.001967},{"mean":0.827455,"std":0.001734},{"mean":0.83,"std":0.001734},{"mean":0.833455,"std":0.002799},{"mean":0.836606,"std":0.003926},{"mean":0.840061,"std":0.005182},{"mean":0.842545,"std":0.006613},{"mean":0.844909,"std":0.007098},{"mean":0.845697,"std":0.007959},{"mean":0.846303,"std":0.009388},{"mean":0.845879,"std":0.011285},{"mean":0.843515,"std":0.014896},{"mean":0.838182,"std":0.019184},{"mean":0.831273,"std":0.023474},{"mean":0.821091,"std":0.026122},{"mean":0.805879,"std":0.031566},{"mean":0.783455,"std":0.037588},{"mean":0.752727,"std":0.04114},{"mean":0.710424,"std":0.034041}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.001,"std":0.001},{"mean":0.003667,"std":0.001528},{"mean":0.006333,"std":0.001528},{"mean":0.008333,"std":0.001528},{"mean":0.012,"std":0.004359},{"mean":0.018667,"std":0.009074},{"mean":0.023,"std":0.009539},{"mean":0.032,"std":0.010536},{"mean":0.043667,"std":0.016503},{"mean":0.064,"std":0.021284},{"mean":0.083,"std":0.032078},{"mean":0.118333,"std":0.043981},{"mean":0.158667,"std":0.064314},{"mean":0.215667,"std":0.059936},{"mean":0.283667,"std":0.055501},{"mean":0.364667,"std":0.060666},{"mean":0.483333,"std":0.075501},{"mean":0.649333,"std":0.092813},{"mean":0.887,"std":0.043555}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432667,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.433,"std":0.001732},{"mean":0.434667,"std":0.003215},{"mean":0.435333,"std":0.004726},{"mean":0.437,"std":0.003},{"mean":0.441333,"std":0.003786},{"mean":0.448333,"std":0.006506},{"mean":0.453333,"std":0.01159},{"mean":0.47,"std":0.016093},{"mean":0.485333,"std":0.026577},{"mean":0.511333,"std":0.030022},{"mean":0.538667,"std":0.02928},{"mean":0.579,"std":0.038743},{"mean":0.642667,"std":0.053818},{"mean":0.742667,"std":0.066935},{"mean":0.888667,"std":0.043108}],"llm_call_rate":[{"mean":0.953212,"std":0.009209},{"mean":0.911152,"std":0.013771},{"mean":0.869818,"std":0.013585},{"mean":0.832606,"std":0.008459},{"mean":0.790606,"std":0.004791},{"mean":0.745455,"std":0.004556},{"mean":0.701636,"std":0.005953},{"mean":0.661394,"std":0.007543},{"mean":0.612727,"std":0.006966},{"mean":0.571091,"std":0.005818},{"mean":0.531273,"std":0.006964},{"mean":0.484424,"std":0.00382},{"mean":0.434909,"std":0.00732},{"mean":0.384485,"std":0.015432},{"mean":0.330545,"std":0.010045},{"mean":0.274364,"std":0.010383},{"mean":0.218242,"std":0.010994},{"mean":0.152848,"std":0.014756},{"mean":0.082061,"std":0.018734},{"mean":0.00103,"std":0.000105}]}}}}},"modernbert/k50":{"model":"modernbert","k":50,"oos_train":null,"kind":"encoder","seeds":[42,43,44],"run_names":["ModernBERT-base-k50-seed42","ModernBERT-base-k50-seed43","ModernBERT-base-k50-seed44"],"logits_sha256":["86bcc65f899f519154881c6e1ccb10ba468e055b7a26049fc11727fc74347a4c","0f72b3ebe5b54dcee8b7857d339de335b31230f6dd06d4f43297730e19d955ee","c7427794ce3f205347c4a1bc703409bfece96251974823dcb075bfc2dd3d6da3"],"result":{"argmax":{"msp":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.001212},{"mean":0.001212,"std":0.000525},{"mean":0.001455,"std":0.0},{"mean":0.001818,"std":0.000606},{"mean":0.001732,"std":0.0006},{"mean":0.001818,"std":0.000455},{"mean":0.00202,"std":0.001069},{"mean":0.002303,"std":0.001169},{"mean":0.003085,"std":0.001063},{"mean":0.003434,"std":0.000763},{"mean":0.004569,"std":0.000582},{"mean":0.007186,"std":0.001281},{"mean":0.011556,"std":0.00122},{"mean":0.018182,"std":0.002192},{"mean":0.029376,"std":0.003756},{"mean":0.045522,"std":0.003593},{"mean":0.066348,"std":0.004093},{"mean":0.096061,"std":0.003772}],"operating":{"tau":[{"mean":0.999957,"std":6e-06},{"mean":0.99993,"std":5e-06},{"mean":0.9999,"std":5e-06},{"mean":0.99987,"std":6e-06},{"mean":0.999828,"std":4e-06},{"mean":0.999779,"std":9e-06},{"mean":0.999725,"std":1e-05},{"mean":0.999648,"std":2.2e-05},{"mean":0.999533,"std":2.3e-05},{"mean":0.999382,"std":4e-05},{"mean":0.999147,"std":6.2e-05},{"mean":0.998826,"std":6.2e-05},{"mean":0.998246,"std":0.000151},{"mean":0.997206,"std":0.00031},{"mean":0.995092,"std":0.000582},{"mean":0.989796,"std":0.001443},{"mean":0.974637,"std":0.00236},{"mean":0.91595,"std":0.005026},{"mean":0.684684,"std":0.015021},{"mean":0.116744,"std":0.016312}],"coverage":[{"mean":0.046606,"std":0.004818},{"mean":0.092242,"std":0.003785},{"mean":0.133758,"std":0.006099},{"mean":0.171273,"std":0.005518},{"mean":0.215636,"std":0.007662},{"mean":0.258667,"std":0.009896},{"mean":0.295515,"std":0.009741},{"mean":0.333818,"std":0.013399},{"mean":0.378848,"std":0.010221},{"mean":0.420545,"std":0.008876},{"mean":0.465697,"std":0.010417},{"mean":0.505576,"std":0.003135},{"mean":0.551697,"std":0.002288},{"mean":0.594485,"std":0.004211},{"mean":0.643394,"std":0.003274},{"mean":0.693091,"std":0.003912},{"mean":0.742545,"std":0.003581},{"mean":0.80297,"std":0.004055},{"mean":0.881879,"std":0.004042},{"mean":0.998788,"std":0.000525}],"selective_risk":[{"mean":0.001475,"std":0.002555},{"mean":0.001377,"std":0.002386},{"mean":0.001397,"std":0.001436},{"mean":0.001082,"std":0.001103},{"mean":0.001126,"std":0.000491},{"mean":0.001635,"std":0.000361},{"mean":0.001849,"std":0.000614},{"mean":0.001635,"std":0.000534},{"mean":0.001758,"std":0.000719},{"mean":0.002007,"std":0.000854},{"mean":0.0022,"std":0.001076},{"mean":0.002276,"std":0.001152},{"mean":0.003073,"std":0.001047},{"mean":0.00336,"std":0.000895},{"mean":0.00433,"std":0.000844},{"mean":0.006815,"std":0.001424},{"mean":0.010531,"std":0.00069},{"mean":0.018573,"std":0.00173},{"mean":0.039028,"std":0.003433},{"mean":0.095084,"std":0.004146}],"accuracy_8":[{"mean":0.825758,"std":0.000859},{"mean":0.83,"std":0.00126},{"mean":0.834485,"std":0.00168},{"mean":0.839091,"std":0.001091},{"mean":0.844182,"std":0.00126},{"mean":0.849576,"std":0.001481},{"mean":0.852727,"std":0.00126},{"mean":0.855515,"std":0.002192},{"mean":0.860121,"std":0.001589},{"mean":0.864848,"std":0.001277},{"mean":0.870061,"std":0.003264},{"mean":0.87497,"std":0.003093},{"mean":0.879697,"std":0.001111},{"mean":0.884242,"std":0.000584},{"mean":0.890303,"std":0.001034},{"mean":0.895879,"std":0.00105},{"mean":0.901818,"std":0.001273},{"mean":0.909212,"std":0.001169},{"mean":0.914788,"std":0.001481},{"mean":0.904364,"std":0.004166}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.001,"std":0.0},{"mean":0.001667,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.002333,"std":0.001155},{"mean":0.002667,"std":0.000577},{"mean":0.003667,"std":0.001155},{"mean":0.003667,"std":0.001155},{"mean":0.006667,"std":0.001528},{"mean":0.008,"std":0.002646},{"mean":0.012,"std":0.002646},{"mean":0.022,"std":0.005196},{"mean":0.036,"std":0.004359},{"mean":0.067,"std":0.009539},{"mean":0.16,"std":0.019672},{"mean":0.456,"std":0.026287}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.431333,"std":0.001155},{"mean":0.430667,"std":0.001528},{"mean":0.429667,"std":0.001155},{"mean":0.428333,"std":0.001528},{"mean":0.43,"std":0.003606},{"mean":0.425,"std":0.005292},{"mean":0.412,"std":0.009165},{"mean":0.402,"std":0.008718},{"mean":0.459,"std":0.025515}],"llm_call_rate":[{"mean":0.953394,"std":0.004818},{"mean":0.907758,"std":0.003785},{"mean":0.866242,"std":0.006099},{"mean":0.828727,"std":0.005518},{"mean":0.784364,"std":0.007662},{"mean":0.741333,"std":0.009896},{"mean":0.704485,"std":0.009741},{"mean":0.666182,"std":0.013399},{"mean":0.621152,"std":0.010221},{"mean":0.579455,"std":0.008876},{"mean":0.534303,"std":0.010417},{"mean":0.494424,"std":0.003135},{"mean":0.448303,"std":0.002288},{"mean":0.405515,"std":0.004211},{"mean":0.356606,"std":0.003274},{"mean":0.306909,"std":0.003912},{"mean":0.257455,"std":0.003581},{"mean":0.19703,"std":0.004055},{"mean":0.118121,"std":0.004042},{"mean":0.001212,"std":0.000525}]}},"entropy":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.001212},{"mean":0.001212,"std":0.000525},{"mean":0.001212,"std":0.00042},{"mean":0.001818,"std":0.000606},{"mean":0.001732,"std":0.0006},{"mean":0.001818,"std":0.000455},{"mean":0.00202,"std":0.001069},{"mean":0.002303,"std":0.001169},{"mean":0.003196,"std":0.000954},{"mean":0.003232,"std":0.000763},{"mean":0.004662,"std":0.000646},{"mean":0.006753,"std":0.001374},{"mean":0.011071,"std":0.001335},{"mean":0.017727,"std":0.00202},{"mean":0.027736,"std":0.002579},{"mean":0.044108,"std":0.004244},{"mean":0.066603,"std":0.004573},{"mean":0.096061,"std":0.003772}],"operating":{"tau":[{"mean":-0.000599,"std":7.5e-05},{"mean":-0.000948,"std":6.8e-05},{"mean":-0.001321,"std":5.8e-05},{"mean":-0.001681,"std":7.7e-05},{"mean":-0.002158,"std":5e-05},{"mean":-0.002712,"std":7.3e-05},{"mean":-0.003289,"std":0.000125},{"mean":-0.00411,"std":0.000195},{"mean":-0.005297,"std":0.000222},{"mean":-0.006763,"std":0.000339},{"mean":-0.008859,"std":0.000419},{"mean":-0.011994,"std":0.000635},{"mean":-0.017178,"std":0.001381},{"mean":-0.025628,"std":0.002271},{"mean":-0.042012,"std":0.005422},{"mean":-0.076269,"std":0.008972},{"mean":-0.165909,"std":0.009041},{"mean":-0.432829,"std":0.024339},{"mean":-1.078652,"std":0.047647},{"mean":-3.651834,"std":0.171912}],"coverage":[{"mean":0.046303,"std":0.004203},{"mean":0.090667,"std":0.003161},{"mean":0.133152,"std":0.004226},{"mean":0.171697,"std":0.006118},{"mean":0.215758,"std":0.007903},{"mean":0.259455,"std":0.008998},{"mean":0.293576,"std":0.012254},{"mean":0.334242,"std":0.013441},{"mean":0.378727,"std":0.00982},{"mean":0.420182,"std":0.008706},{"mean":0.463333,"std":0.008531},{"mean":0.504424,"std":0.004842},{"mean":0.550727,"std":0.002378},{"mean":0.595515,"std":0.002442},{"mean":0.643576,"std":0.007341},{"mean":0.691152,"std":0.000555},{"mean":0.74097,"std":0.005321},{"mean":0.799879,"std":0.003926},{"mean":0.872242,"std":0.006031},{"mean":0.99903,"std":0.001238}],"selective_risk":[{"mean":0.001462,"std":0.002532},{"mean":0.001392,"std":0.002411},{"mean":0.001389,"std":0.001417},{"mean":0.001084,"std":0.001104},{"mean":0.001123,"std":0.000479},{"mean":0.001388,"std":0.000663},{"mean":0.001865,"std":0.00063},{"mean":0.001634,"std":0.000538},{"mean":0.001919,"std":0.00046},{"mean":0.001871,"std":0.000633},{"mean":0.002214,"std":0.001094},{"mean":0.002277,"std":0.001137},{"mean":0.00319,"std":0.000943},{"mean":0.003153,"std":0.000757},{"mean":0.004044,"std":0.000862},{"mean":0.006313,"std":0.001461},{"mean":0.009903,"std":0.001302},{"mean":0.017886,"std":0.001723},{"mean":0.034046,"std":0.002248},{"mean":0.095301,"std":0.004546}],"accuracy_8":[{"mean":0.825697,"std":0.00082},{"mean":0.829818,"std":0.001106},{"mean":0.834606,"std":0.001155},{"mean":0.838848,"std":0.000915},{"mean":0.844242,"std":0.001365},{"mean":0.849394,"std":0.001794},{"mean":0.852667,"std":0.001961},{"mean":0.855758,"std":0.002455},{"mean":0.860061,"std":0.001821},{"mean":0.864909,"std":0.002278},{"mean":0.869939,"std":0.003254},{"mean":0.874727,"std":0.003217},{"mean":0.879758,"std":0.00147},{"mean":0.884848,"std":0.000525},{"mean":0.890061,"std":0.000278},{"mean":0.895455,"std":0.000182},{"mean":0.901939,"std":0.00134},{"mean":0.908242,"std":0.001211},{"mean":0.914182,"std":0.001636},{"mean":0.904364,"std":0.00411}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.003667,"std":0.001155},{"mean":0.003667,"std":0.001155},{"mean":0.006667,"std":0.001528},{"mean":0.007333,"std":0.002082},{"mean":0.011333,"std":0.001528},{"mean":0.02,"std":0.005196},{"mean":0.033,"std":0.007},{"mean":0.064,"std":0.009165},{"mean":0.133,"std":0.013077},{"mean":0.456667,"std":0.028449}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432333,"std":0.001155},{"mean":0.431667,"std":0.001528},{"mean":0.430667,"std":0.001528},{"mean":0.429667,"std":0.001155},{"mean":0.43,"std":0.002646},{"mean":0.431667,"std":0.001528},{"mean":0.426,"std":0.005568},{"mean":0.416333,"std":0.011372},{"mean":0.403333,"std":0.01159},{"mean":0.459,"std":0.025357}],"llm_call_rate":[{"mean":0.953697,"std":0.004203},{"mean":0.909333,"std":0.003161},{"mean":0.866848,"std":0.004226},{"mean":0.828303,"std":0.006118},{"mean":0.784242,"std":0.007903},{"mean":0.740545,"std":0.008998},{"mean":0.706424,"std":0.012254},{"mean":0.665758,"std":0.013441},{"mean":0.621273,"std":0.00982},{"mean":0.579818,"std":0.008706},{"mean":0.536667,"std":0.008531},{"mean":0.495576,"std":0.004842},{"mean":0.449273,"std":0.002378},{"mean":0.404485,"std":0.002442},{"mean":0.356424,"std":0.007341},{"mean":0.308848,"std":0.000555},{"mean":0.25903,"std":0.005321},{"mean":0.200121,"std":0.003926},{"mean":0.127758,"std":0.006031},{"mean":0.00097,"std":0.001238}]}},"margin":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001818,"std":0.001818},{"mean":0.001616,"std":0.0007},{"mean":0.002121,"std":0.000525},{"mean":0.002182,"std":0.000727},{"mean":0.001818,"std":0.000606},{"mean":0.001558,"std":0.000519},{"mean":0.001818,"std":0.001203},{"mean":0.002155,"std":0.001419},{"mean":0.002424,"std":0.001377},{"mean":0.003196,"std":0.000954},{"mean":0.003636,"std":0.001093},{"mean":0.005594,"std":0.001009},{"mean":0.008485,"std":0.001081},{"mean":0.012364,"std":0.001748},{"mean":0.02,"std":0.002972},{"mean":0.031658,"std":0.004175},{"mean":0.048552,"std":0.003145},{"mean":0.070813,"std":0.003482},{"mean":0.096061,"std":0.003772}],"operating":{"tau":[{"mean":10.605658,"std":0.428896},{"mean":10.083402,"std":0.368462},{"mean":9.687023,"std":0.332603},{"mean":9.395405,"std":0.309771},{"mean":9.1028,"std":0.298023},{"mean":8.835367,"std":0.272713},{"mean":8.592854,"std":0.277076},{"mean":8.319243,"std":0.263328},{"mean":8.036451,"std":0.253687},{"mean":7.709885,"std":0.253873},{"mean":7.390019,"std":0.227438},{"mean":7.037292,"std":0.229197},{"mean":6.653104,"std":0.252998},{"mean":6.202824,"std":0.246184},{"mean":5.594621,"std":0.269552},{"mean":4.889947,"std":0.271589},{"mean":3.993487,"std":0.200416},{"mean":2.799468,"std":0.107544},{"mean":1.37392,"std":0.127893},{"mean":0.003608,"std":0.004438}],"coverage":[{"mean":0.048545,"std":0.00553},{"mean":0.091455,"std":0.007415},{"mean":0.136,"std":0.009764},{"mean":0.174182,"std":0.008397},{"mean":0.21697,"std":0.006838},{"mean":0.257212,"std":0.009113},{"mean":0.293333,"std":0.011912},{"mean":0.332909,"std":0.015238},{"mean":0.37503,"std":0.013481},{"mean":0.421152,"std":0.008577},{"mean":0.464545,"std":0.003288},{"mean":0.508121,"std":0.004604},{"mean":0.550848,"std":0.001961},{"mean":0.596667,"std":0.003841},{"mean":0.647939,"std":0.003254},{"mean":0.698606,"std":0.006299},{"mean":0.752303,"std":0.003028},{"mean":0.814121,"std":0.003674},{"mean":0.892242,"std":0.006843},{"mean":0.999636,"std":0.00063}],"selective_risk":[{"mean":0.001437,"std":0.002489},{"mean":0.002098,"std":0.002199},{"mean":0.001828,"std":0.000942},{"mean":0.001413,"std":0.000686},{"mean":0.002229,"std":0.000448},{"mean":0.002111,"std":0.000647},{"mean":0.001854,"std":0.000577},{"mean":0.001636,"std":0.000517},{"mean":0.001451,"std":0.000457},{"mean":0.001858,"std":0.001345},{"mean":0.00222,"std":0.001581},{"mean":0.002741,"std":0.001078},{"mean":0.003188,"std":0.00094},{"mean":0.003652,"std":0.001073},{"mean":0.005421,"std":0.001303},{"mean":0.008494,"std":0.001363},{"mean":0.01249,"std":0.00161},{"mean":0.023003,"std":0.003248},{"mean":0.04541,"std":0.006403},{"mean":0.095854,"std":0.0035}],"accuracy_8":[{"mean":0.826485,"std":0.000933},{"mean":0.830364,"std":0.002234},{"mean":0.834909,"std":0.002378},{"mean":0.839152,"std":0.00165},{"mean":0.842909,"std":0.000962},{"mean":0.848061,"std":0.000735},{"mean":0.851939,"std":0.001065},{"mean":0.855879,"std":0.00163},{"mean":0.860242,"std":0.001821},{"mean":0.864909,"std":0.001575},{"mean":0.869939,"std":0.000735},{"mean":0.875273,"std":0.001916},{"mean":0.880606,"std":0.00129},{"mean":0.885515,"std":0.001155},{"mean":0.891818,"std":0.000962},{"mean":0.897394,"std":0.001481},{"mean":0.903636,"std":0.001273},{"mean":0.911455,"std":0.001373},{"mean":0.914727,"std":0.003102},{"mean":0.903939,"std":0.003772}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.002082},{"mean":0.003333,"std":0.002082},{"mean":0.005,"std":0.001732},{"mean":0.007,"std":0.001},{"mean":0.009333,"std":0.002082},{"mean":0.015667,"std":0.004933},{"mean":0.027333,"std":0.006351},{"mean":0.043333,"std":0.008505},{"mean":0.086667,"std":0.017502},{"mean":0.191667,"std":0.034588},{"mean":0.460333,"std":0.022143}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.431333,"std":0.001528},{"mean":0.43,"std":0.001},{"mean":0.429,"std":0.001},{"mean":0.428333,"std":0.000577},{"mean":0.427,"std":0.001732},{"mean":0.428333,"std":0.002082},{"mean":0.423333,"std":0.005508},{"mean":0.416,"std":0.006557},{"mean":0.398667,"std":0.01193},{"mean":0.400667,"std":0.019858},{"mean":0.461333,"std":0.023245}],"llm_call_rate":[{"mean":0.951455,"std":0.00553},{"mean":0.908545,"std":0.007415},{"mean":0.864,"std":0.009764},{"mean":0.825818,"std":0.008397},{"mean":0.78303,"std":0.006838},{"mean":0.742788,"std":0.009113},{"mean":0.706667,"std":0.011912},{"mean":0.667091,"std":0.015238},{"mean":0.62497,"std":0.013481},{"mean":0.578848,"std":0.008577},{"mean":0.535455,"std":0.003288},{"mean":0.491879,"std":0.004604},{"mean":0.449152,"std":0.001961},{"mean":0.403333,"std":0.003841},{"mean":0.352061,"std":0.003254},{"mean":0.301394,"std":0.006299},{"mean":0.247697,"std":0.003028},{"mean":0.185879,"std":0.003674},{"mean":0.107758,"std":0.006843},{"mean":0.000364,"std":0.00063}]}},"msp_t":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.001212},{"mean":0.000909,"std":0.000909},{"mean":0.001212,"std":0.00042},{"mean":0.001818,"std":0.000606},{"mean":0.001732,"std":0.0006},{"mean":0.001818,"std":0.000455},{"mean":0.00202,"std":0.001069},{"mean":0.002303,"std":0.001169},{"mean":0.003085,"std":0.001063},{"mean":0.003232,"std":0.000763},{"mean":0.004662,"std":0.000646},{"mean":0.0071,"std":0.0012},{"mean":0.011394,"std":0.001212},{"mean":0.017727,"std":0.00186},{"mean":0.028663,"std":0.003151},{"mean":0.044848,"std":0.004101},{"mean":0.066411,"std":0.003996},{"mean":0.096061,"std":0.003772}],"operating":{"tau":[{"mean":0.99985,"std":6.3e-05},{"mean":0.999768,"std":8.5e-05},{"mean":0.999681,"std":0.000106},{"mean":0.999595,"std":0.000132},{"mean":0.999487,"std":0.000157},{"mean":0.99936,"std":0.000189},{"mean":0.999227,"std":0.000221},{"mean":0.999031,"std":0.000275},{"mean":0.998749,"std":0.000366},{"mean":0.998408,"std":0.000482},{"mean":0.997908,"std":0.0006},{"mean":0.997173,"std":0.000804},{"mean":0.995925,"std":0.001148},{"mean":0.993875,"std":0.001772},{"mean":0.989844,"std":0.003019},{"mean":0.981154,"std":0.005004},{"mean":0.957687,"std":0.00809},{"mean":0.87568,"std":0.018764},{"mean":0.634981,"std":0.029388},{"mean":0.100762,"std":0.010261}],"coverage":[{"mean":0.046182,"std":0.003988},{"mean":0.090364,"std":0.003004},{"mean":0.13297,"std":0.004791},{"mean":0.171758,"std":0.006637},{"mean":0.215333,"std":0.007637},{"mean":0.258848,"std":0.008197},{"mean":0.293394,"std":0.011173},{"mean":0.334667,"std":0.013972},{"mean":0.37903,"std":0.010247},{"mean":0.419576,"std":0.008577},{"mean":0.463758,"std":0.008837},{"mean":0.504909,"std":0.005292},{"mean":0.551515,"std":0.00134},{"mean":0.595515,"std":0.003364},{"mean":0.643515,"std":0.006408},{"mean":0.691576,"std":0.001365},{"mean":0.74103,"std":0.005013},{"mean":0.803152,"std":0.004155},{"mean":0.878242,"std":0.005157},{"mean":0.998848,"std":0.00042}],"selective_risk":[{"mean":0.001456,"std":0.002521},{"mean":0.001395,"std":0.002416},{"mean":0.001392,"std":0.001426},{"mean":0.001086,"std":0.001108},{"mean":0.001126,"std":0.000487},{"mean":0.001392,"std":0.000667},{"mean":0.001661,"std":0.000402},{"mean":0.001632,"std":0.000538},{"mean":0.001916,"std":0.000456},{"mean":0.001872,"std":0.000628},{"mean":0.002212,"std":0.001095},{"mean":0.002275,"std":0.001137},{"mean":0.003076,"std":0.001056},{"mean":0.003254,"std":0.000754},{"mean":0.004139,"std":0.000772},{"mean":0.006571,"std":0.001453},{"mean":0.009982,"std":0.000928},{"mean":0.018341,"std":0.001354},{"mean":0.03746,"std":0.004427},{"mean":0.095139,"std":0.004055}],"accuracy_8":[{"mean":0.825697,"std":0.00082},{"mean":0.829758,"std":0.001065},{"mean":0.834606,"std":0.001315},{"mean":0.838848,"std":0.00082},{"mean":0.844364,"std":0.001012},{"mean":0.849273,"std":0.001898},{"mean":0.852727,"std":0.001791},{"mean":0.855818,"std":0.00246},{"mean":0.86,"std":0.001666},{"mean":0.864727,"std":0.001916},{"mean":0.87,"std":0.003397},{"mean":0.874727,"std":0.003397},{"mean":0.879515,"std":0.001211},{"mean":0.884848,"std":0.000525},{"mean":0.890182,"std":0.00063},{"mean":0.895333,"std":0.00021},{"mean":0.901697,"std":0.001001},{"mean":0.909515,"std":0.001065},{"mean":0.914424,"std":0.002107},{"mean":0.904303,"std":0.004067}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.000577},{"mean":0.001667,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.003667,"std":0.001155},{"mean":0.003667,"std":0.001155},{"mean":0.006667,"std":0.001528},{"mean":0.007667,"std":0.002517},{"mean":0.011667,"std":0.001528},{"mean":0.021,"std":0.005196},{"mean":0.033667,"std":0.005508},{"mean":0.066333,"std":0.008505},{"mean":0.151,"std":0.023302},{"mean":0.456333,"std":0.025716}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432333,"std":0.001155},{"mean":0.431667,"std":0.001528},{"mean":0.430667,"std":0.001528},{"mean":0.429667,"std":0.001155},{"mean":0.429667,"std":0.002082},{"mean":0.431333,"std":0.001528},{"mean":0.425667,"std":0.004163},{"mean":0.412,"std":0.008185},{"mean":0.401333,"std":0.013429},{"mean":0.459333,"std":0.024947}],"llm_call_rate":[{"mean":0.953818,"std":0.003988},{"mean":0.909636,"std":0.003004},{"mean":0.86703,"std":0.004791},{"mean":0.828242,"std":0.006637},{"mean":0.784667,"std":0.007637},{"mean":0.741152,"std":0.008197},{"mean":0.706606,"std":0.011173},{"mean":0.665333,"std":0.013972},{"mean":0.62097,"std":0.010247},{"mean":0.580424,"std":0.008577},{"mean":0.536242,"std":0.008837},{"mean":0.495091,"std":0.005292},{"mean":0.448485,"std":0.00134},{"mean":0.404485,"std":0.003364},{"mean":0.356485,"std":0.006408},{"mean":0.308424,"std":0.001365},{"mean":0.25897,"std":0.005013},{"mean":0.196848,"std":0.004155},{"mean":0.121758,"std":0.005157},{"mean":0.001152,"std":0.00042}]}}},"summed":{"msp":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.000606,"std":0.00105},{"mean":0.000808,"std":0.0014},{"mean":0.000909,"std":0.000909},{"mean":0.00097,"std":0.001111},{"mean":0.001414,"std":0.00035},{"mean":0.002078,"std":0.000519},{"mean":0.002424,"std":0.000694},{"mean":0.00229,"std":0.000841},{"mean":0.002545,"std":0.000727},{"mean":0.003306,"std":0.001322},{"mean":0.003737,"std":0.001262},{"mean":0.004009,"std":0.001165},{"mean":0.005801,"std":0.000983},{"mean":0.009212,"std":0.000485},{"mean":0.016212,"std":0.001389},{"mean":0.026168,"std":0.00353},{"mean":0.044714,"std":0.003407},{"mean":0.070431,"std":0.003513},{"mean":0.102727,"std":0.002278}],"operating":{"tau":[{"mean":0.99998,"std":2e-06},{"mean":0.999968,"std":4e-06},{"mean":0.999955,"std":4e-06},{"mean":0.999941,"std":4e-06},{"mean":0.999923,"std":3e-06},{"mean":0.999902,"std":2e-06},{"mean":0.999876,"std":5e-06},{"mean":0.99984,"std":6e-06},{"mean":0.9998,"std":8e-06},{"mean":0.999737,"std":1.8e-05},{"mean":0.999645,"std":2.7e-05},{"mean":0.999509,"std":3.2e-05},{"mean":0.999312,"std":4.4e-05},{"mean":0.998957,"std":9e-05},{"mean":0.998248,"std":0.00013},{"mean":0.996818,"std":0.000277},{"mean":0.992687,"std":0.000508},{"mean":0.976684,"std":0.001351},{"mean":0.8731,"std":0.008775},{"mean":0.242522,"std":0.034306}],"coverage":[{"mean":0.047879,"std":0.004403},{"mean":0.089697,"std":0.008694},{"mean":0.134848,"std":0.006393},{"mean":0.171333,"std":0.003989},{"mean":0.212303,"std":0.003413},{"mean":0.253515,"std":0.005881},{"mean":0.29,"std":0.01086},{"mean":0.334848,"std":0.004234},{"mean":0.373939,"std":0.003193},{"mean":0.419091,"std":0.004717},{"mean":0.464242,"std":0.005299},{"mean":0.505576,"std":0.004732},{"mean":0.545636,"std":0.005381},{"mean":0.592061,"std":0.000278},{"mean":0.63903,"std":0.004055},{"mean":0.682242,"std":0.004135},{"mean":0.732,"std":0.004375},{"mean":0.784727,"std":0.006309},{"mean":0.861636,"std":0.002857},{"mean":0.998242,"std":0.001689}],"selective_risk":[{"mean":0.001355,"std":0.002347},{"mean":0.000759,"std":0.001315},{"mean":0.00095,"std":0.001645},{"mean":0.000727,"std":0.001259},{"mean":0.000862,"std":0.000859},{"mean":0.00095,"std":0.001072},{"mean":0.001243,"std":0.000599},{"mean":0.001625,"std":0.000529},{"mean":0.002109,"std":0.000748},{"mean":0.002312,"std":0.000648},{"mean":0.002343,"std":0.001005},{"mean":0.002515,"std":0.000705},{"mean":0.00311,"std":0.001165},{"mean":0.003685,"std":0.00134},{"mean":0.004082,"std":0.001207},{"mean":0.004616,"std":0.001055},{"mean":0.007785,"std":0.001027},{"mean":0.01282,"std":0.000545},{"mean":0.030179,"std":0.002633},{"mean":0.101326,"std":0.003499}],"accuracy_8":[{"mean":0.824121,"std":0.001065},{"mean":0.827758,"std":0.00129},{"mean":0.831697,"std":0.000555},{"mean":0.835152,"std":0.001096},{"mean":0.839394,"std":0.000584},{"mean":0.844121,"std":0.001211},{"mean":0.847879,"std":0.002455},{"mean":0.852121,"std":0.002586},{"mean":0.85703,"std":0.003213},{"mean":0.86103,"std":0.002956},{"mean":0.865515,"std":0.003093},{"mean":0.869939,"std":0.003461},{"mean":0.874727,"std":0.004276},{"mean":0.880545,"std":0.002909},{"mean":0.885818,"std":0.001273},{"mean":0.892485,"std":0.001096},{"mean":0.898242,"std":0.001211},{"mean":0.904727,"std":0.001106},{"mean":0.913273,"std":0.001273},{"mean":0.898182,"std":0.003107}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.001155},{"mean":0.001,"std":0.001},{"mean":0.002,"std":0.001},{"mean":0.003,"std":0.001},{"mean":0.004,"std":0.001},{"mean":0.004333,"std":0.001528},{"mean":0.005333,"std":0.001155},{"mean":0.007333,"std":0.002082},{"mean":0.009333,"std":0.002517},{"mean":0.011667,"std":0.002517},{"mean":0.014333,"std":0.002517},{"mean":0.026667,"std":0.004163},{"mean":0.046667,"std":0.004041},{"mean":0.122333,"std":0.013868},{"mean":0.494333,"std":0.024194}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.431667,"std":0.001528},{"mean":0.431,"std":0.002},{"mean":0.430333,"std":0.002309},{"mean":0.431333,"std":0.002517},{"mean":0.426667,"std":0.005859},{"mean":0.410667,"std":0.008505},{"mean":0.498,"std":0.021166}],"llm_call_rate":[{"mean":0.952121,"std":0.004403},{"mean":0.910303,"std":0.008694},{"mean":0.865152,"std":0.006393},{"mean":0.828667,"std":0.003989},{"mean":0.787697,"std":0.003413},{"mean":0.746485,"std":0.005881},{"mean":0.71,"std":0.01086},{"mean":0.665152,"std":0.004234},{"mean":0.626061,"std":0.003193},{"mean":0.580909,"std":0.004717},{"mean":0.535758,"std":0.005299},{"mean":0.494424,"std":0.004732},{"mean":0.454364,"std":0.005381},{"mean":0.407939,"std":0.000278},{"mean":0.36097,"std":0.004055},{"mean":0.317758,"std":0.004135},{"mean":0.268,"std":0.004375},{"mean":0.215273,"std":0.006309},{"mean":0.138364,"std":0.002857},{"mean":0.001758,"std":0.001689}]}},"entropy":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.000606,"std":0.00105},{"mean":0.000808,"std":0.0014},{"mean":0.000606,"std":0.00105},{"mean":0.00097,"std":0.001111},{"mean":0.001414,"std":0.00035},{"mean":0.002078,"std":0.000519},{"mean":0.002273,"std":0.000787},{"mean":0.002155,"std":0.000617},{"mean":0.002545,"std":0.000727},{"mean":0.003085,"std":0.001161},{"mean":0.003737,"std":0.001262},{"mean":0.004103,"std":0.001059},{"mean":0.005714,"std":0.001132},{"mean":0.009293,"std":0.00061},{"mean":0.016439,"std":0.001765},{"mean":0.02631,"std":0.00376},{"mean":0.044646,"std":0.003974},{"mean":0.070558,"std":0.003928},{"mean":0.102727,"std":0.002278}],"operating":{"tau":[{"mean":-0.000268,"std":3e-05},{"mean":-0.000413,"std":4.1e-05},{"mean":-0.000569,"std":4.7e-05},{"mean":-0.000728,"std":4e-05},{"mean":-0.00093,"std":3.1e-05},{"mean":-0.001159,"std":2.1e-05},{"mean":-0.001418,"std":4.8e-05},{"mean":-0.001796,"std":6.3e-05},{"mean":-0.002203,"std":8.3e-05},{"mean":-0.002809,"std":0.000159},{"mean":-0.00368,"std":0.000236},{"mean":-0.004934,"std":0.000289},{"mean":-0.006717,"std":0.000509},{"mean":-0.0096,"std":0.000743},{"mean":-0.015447,"std":0.001085},{"mean":-0.025839,"std":0.001852},{"mean":-0.05255,"std":0.003308},{"mean":-0.136541,"std":0.006974},{"mean":-0.513934,"std":0.039607},{"mean":-1.879365,"std":0.065926}],"coverage":[{"mean":0.047636,"std":0.00512},{"mean":0.089636,"std":0.008072},{"mean":0.134061,"std":0.00646},{"mean":0.17103,"std":0.003951},{"mean":0.213212,"std":0.003532},{"mean":0.253636,"std":0.007079},{"mean":0.289455,"std":0.009588},{"mean":0.335212,"std":0.006086},{"mean":0.374182,"std":0.002745},{"mean":0.418424,"std":0.00305},{"mean":0.464303,"std":0.005976},{"mean":0.505455,"std":0.004848},{"mean":0.546606,"std":0.005079},{"mean":0.591394,"std":0.00042},{"mean":0.639818,"std":0.003338},{"mean":0.682606,"std":0.004147},{"mean":0.73103,"std":0.004672},{"mean":0.784364,"std":0.007334},{"mean":0.857879,"std":0.006217},{"mean":0.997091,"std":0.00317}],"selective_risk":[{"mean":0.001383,"std":0.002396},{"mean":0.000754,"std":0.001306},{"mean":0.000956,"std":0.001657},{"mean":0.000728,"std":0.001261},{"mean":0.000859,"std":0.000856},{"mean":0.000948,"std":0.001068},{"mean":0.001246,"std":0.000604},{"mean":0.001623,"std":0.000525},{"mean":0.002107,"std":0.000747},{"mean":0.002317,"std":0.000659},{"mean":0.002342,"std":0.001003},{"mean":0.002755,"std":0.001083},{"mean":0.003212,"std":0.001323},{"mean":0.003689,"std":0.001339},{"mean":0.004171,"std":0.001093},{"mean":0.004615,"std":0.00106},{"mean":0.007629,"std":0.001283},{"mean":0.012747,"std":0.000767},{"mean":0.028755,"std":0.002493},{"mean":0.10083,"std":0.004152}],"accuracy_8":[{"mean":0.824182,"std":0.000962},{"mean":0.828,"std":0.001192},{"mean":0.831818,"std":0.000656},{"mean":0.835212,"std":0.001001},{"mean":0.839515,"std":0.000688},{"mean":0.844242,"std":0.00129},{"mean":0.848121,"std":0.002366},{"mean":0.852727,"std":0.002678},{"mean":0.857273,"std":0.00317},{"mean":0.86103,"std":0.003066},{"mean":0.865333,"std":0.003274},{"mean":0.869879,"std":0.003918},{"mean":0.875273,"std":0.004012},{"mean":0.880667,"std":0.003011},{"mean":0.886303,"std":0.000859},{"mean":0.892606,"std":0.00129},{"mean":0.898303,"std":0.001183},{"mean":0.905091,"std":0.00126},{"mean":0.912788,"std":0.00147},{"mean":0.898424,"std":0.003379}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.001155},{"mean":0.001,"std":0.001},{"mean":0.002,"std":0.001},{"mean":0.003,"std":0.001},{"mean":0.004,"std":0.001},{"mean":0.004333,"std":0.001528},{"mean":0.006,"std":0.002},{"mean":0.007333,"std":0.002082},{"mean":0.009333,"std":0.002517},{"mean":0.011667,"std":0.002517},{"mean":0.014333,"std":0.002517},{"mean":0.026,"std":0.005292},{"mean":0.046,"std":0.003606},{"mean":0.114333,"std":0.012423},{"mean":0.491,"std":0.029445}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.432,"std":0.001},{"mean":0.431667,"std":0.001528},{"mean":0.430667,"std":0.002082},{"mean":0.430333,"std":0.002309},{"mean":0.431667,"std":0.003055},{"mean":0.427333,"std":0.005686},{"mean":0.411333,"std":0.011015},{"mean":0.496667,"std":0.02318}],"llm_call_rate":[{"mean":0.952364,"std":0.00512},{"mean":0.910364,"std":0.008072},{"mean":0.865939,"std":0.00646},{"mean":0.82897,"std":0.003951},{"mean":0.786788,"std":0.003532},{"mean":0.746364,"std":0.007079},{"mean":0.710545,"std":0.009588},{"mean":0.664788,"std":0.006086},{"mean":0.625818,"std":0.002745},{"mean":0.581576,"std":0.00305},{"mean":0.535697,"std":0.005976},{"mean":0.494545,"std":0.004848},{"mean":0.453394,"std":0.005079},{"mean":0.408606,"std":0.00042},{"mean":0.360182,"std":0.003338},{"mean":0.317394,"std":0.004147},{"mean":0.26897,"std":0.004672},{"mean":0.215636,"std":0.007334},{"mean":0.142121,"std":0.006217},{"mean":0.002909,"std":0.00317}]}},"margin":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.001212,"std":0.002099},{"mean":0.000808,"std":0.0014},{"mean":0.000909,"std":0.000909},{"mean":0.00097,"std":0.00042},{"mean":0.001818,"std":0.0},{"mean":0.002424,"std":0.000794},{"mean":0.002424,"std":0.000694},{"mean":0.002694,"std":0.000617},{"mean":0.002667,"std":0.000757},{"mean":0.003196,"std":0.00101},{"mean":0.003737,"std":0.001555},{"mean":0.004848,"std":0.001261},{"mean":0.006494,"std":0.001132},{"mean":0.009374,"std":0.001009},{"mean":0.016515,"std":0.001479},{"mean":0.027451,"std":0.002579},{"mean":0.046263,"std":0.003017},{"mean":0.072727,"std":0.004197},{"mean":0.102727,"std":0.002278}],"operating":{"tau":[{"mean":10.625564,"std":0.45132},{"mean":10.177656,"std":0.423586},{"mean":9.852569,"std":0.401727},{"mean":9.575558,"std":0.375571},{"mean":9.345477,"std":0.349734},{"mean":9.097795,"std":0.323765},{"mean":8.878197,"std":0.32021},{"mean":8.640571,"std":0.323197},{"mean":8.398799,"std":0.312571},{"mean":8.13268,"std":0.310391},{"mean":7.85318,"std":0.304346},{"mean":7.545265,"std":0.314236},{"mean":7.24222,"std":0.322542},{"mean":6.842688,"std":0.291221},{"mean":6.36742,"std":0.300974},{"mean":5.784959,"std":0.293693},{"mean":4.993021,"std":0.183298},{"mean":3.956721,"std":0.177582},{"mean":2.263456,"std":0.104896},{"mean":0.012661,"std":0.015284}],"coverage":[{"mean":0.049152,"std":0.002637},{"mean":0.090545,"std":0.006077},{"mean":0.134485,"std":0.007191},{"mean":0.176364,"std":0.002857},{"mean":0.211152,"std":0.004246},{"mean":0.252485,"std":0.00438},{"mean":0.291333,"std":0.004686},{"mean":0.332909,"std":0.006938},{"mean":0.37503,"std":0.007834},{"mean":0.421333,"std":0.005575},{"mean":0.463394,"std":0.006185},{"mean":0.506242,"std":0.0033},{"mean":0.544364,"std":0.003186},{"mean":0.591818,"std":0.006563},{"mean":0.637212,"std":0.002586},{"mean":0.683818,"std":0.006364},{"mean":0.733636,"std":0.005671},{"mean":0.785394,"std":0.005007},{"mean":0.864848,"std":0.00883},{"mean":0.998606,"std":0.001821}],"selective_risk":[{"mean":0.001235,"std":0.002138},{"mean":0.000723,"std":0.001252},{"mean":0.000961,"std":0.001664},{"mean":0.001035,"std":0.001046},{"mean":0.000865,"std":0.000858},{"mean":0.001196,"std":0.000399},{"mean":0.00145,"std":0.000708},{"mean":0.001998,"std":0.000604},{"mean":0.002584,"std":0.000721},{"mean":0.002588,"std":0.000849},{"mean":0.002746,"std":0.000773},{"mean":0.002633,"std":0.000744},{"mean":0.003226,"std":0.001004},{"mean":0.003583,"std":0.001441},{"mean":0.004378,"std":0.001302},{"mean":0.005402,"std":0.001431},{"mean":0.00851,"std":0.000583},{"mean":0.013658,"std":0.001036},{"mean":0.032708,"std":0.00376},{"mean":0.101835,"std":0.003533}],"accuracy_8":[{"mean":0.823758,"std":0.001111},{"mean":0.826606,"std":0.001096},{"mean":0.830121,"std":0.001277},{"mean":0.834606,"std":0.000897},{"mean":0.838364,"std":0.001273},{"mean":0.842667,"std":0.001169},{"mean":0.846727,"std":0.001666},{"mean":0.850545,"std":0.003232},{"mean":0.85503,"std":0.004399},{"mean":0.859939,"std":0.003354},{"mean":0.865152,"std":0.00347},{"mean":0.869697,"std":0.003461},{"mean":0.873758,"std":0.003732},{"mean":0.879273,"std":0.004345},{"mean":0.884848,"std":0.002302},{"mean":0.892,"std":0.002049},{"mean":0.898121,"std":0.001111},{"mean":0.904182,"std":0.00142},{"mean":0.912485,"std":0.001169},{"mean":0.897818,"std":0.003037}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.000577},{"mean":0.001333,"std":0.001155},{"mean":0.002667,"std":0.001155},{"mean":0.004,"std":0.001},{"mean":0.004667,"std":0.001528},{"mean":0.005333,"std":0.001155},{"mean":0.005667,"std":0.001528},{"mean":0.007667,"std":0.001528},{"mean":0.009333,"std":0.003512},{"mean":0.012667,"std":0.003215},{"mean":0.017667,"std":0.004163},{"mean":0.029667,"std":0.002309},{"mean":0.051667,"std":0.003215},{"mean":0.137,"std":0.018682},{"mean":0.497667,"std":0.024214}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.431667,"std":0.000577},{"mean":0.431,"std":0.001},{"mean":0.431333,"std":0.000577},{"mean":0.431,"std":0.001732},{"mean":0.430667,"std":0.004163},{"mean":0.428333,"std":0.007371},{"mean":0.414333,"std":0.010017},{"mean":0.499667,"std":0.021572}],"llm_call_rate":[{"mean":0.950848,"std":0.002637},{"mean":0.909455,"std":0.006077},{"mean":0.865515,"std":0.007191},{"mean":0.823636,"std":0.002857},{"mean":0.788848,"std":0.004246},{"mean":0.747515,"std":0.00438},{"mean":0.708667,"std":0.004686},{"mean":0.667091,"std":0.006938},{"mean":0.62497,"std":0.007834},{"mean":0.578667,"std":0.005575},{"mean":0.536606,"std":0.006185},{"mean":0.493758,"std":0.0033},{"mean":0.455636,"std":0.003186},{"mean":0.408182,"std":0.006563},{"mean":0.362788,"std":0.002586},{"mean":0.316182,"std":0.006364},{"mean":0.266364,"std":0.005671},{"mean":0.214606,"std":0.005007},{"mean":0.135152,"std":0.00883},{"mean":0.001394,"std":0.001821}]}},"msp_t":{"risk_coverage":[{"mean":0.001212,"std":0.002099},{"mean":0.000606,"std":0.00105},{"mean":0.000808,"std":0.0014},{"mean":0.000909,"std":0.000909},{"mean":0.00097,"std":0.001111},{"mean":0.001414,"std":0.00035},{"mean":0.001732,"std":0.0003},{"mean":0.002273,"std":0.000787},{"mean":0.002155,"std":0.000617},{"mean":0.002545,"std":0.000727},{"mean":0.003085,"std":0.001161},{"mean":0.003737,"std":0.001262},{"mean":0.004103,"std":0.001059},{"mean":0.005801,"std":0.0012},{"mean":0.009374,"std":0.000741},{"mean":0.016212,"std":0.001837},{"mean":0.026667,"std":0.003768},{"mean":0.044983,"std":0.003086},{"mean":0.071388,"std":0.003482},{"mean":0.102727,"std":0.002278}],"operating":{"tau":[{"mean":0.999923,"std":3.3e-05},{"mean":0.999882,"std":4.8e-05},{"mean":0.99984,"std":6e-05},{"mean":0.999799,"std":6.9e-05},{"mean":0.999744,"std":8.6e-05},{"mean":0.999683,"std":0.000102},{"mean":0.999616,"std":0.000117},{"mean":0.999511,"std":0.000159},{"mean":0.999399,"std":0.00019},{"mean":0.999229,"std":0.000257},{"mean":0.999,"std":0.000329},{"mean":0.998669,"std":0.000415},{"mean":0.998184,"std":0.000576},{"mean":0.997377,"std":0.000831},{"mean":0.995814,"std":0.001212},{"mean":0.992972,"std":0.001979},{"mean":0.985284,"std":0.00342},{"mean":0.960225,"std":0.008098},{"mean":0.829725,"std":0.022946},{"mean":0.228757,"std":0.031056}],"coverage":[{"mean":0.047576,"std":0.004784},{"mean":0.090242,"std":0.00872},{"mean":0.13297,"std":0.005299},{"mean":0.170667,"std":0.003254},{"mean":0.212727,"std":0.005894},{"mean":0.253515,"std":0.007893},{"mean":0.288485,"std":0.00849},{"mean":0.335939,"std":0.005147},{"mean":0.375152,"std":0.001944},{"mean":0.419576,"std":0.004091},{"mean":0.46503,"std":0.005623},{"mean":0.505939,"std":0.006468},{"mean":0.547212,"std":0.00378},{"mean":0.592606,"std":0.000757},{"mean":0.639636,"std":0.003963},{"mean":0.682364,"std":0.005536},{"mean":0.731455,"std":0.004983},{"mean":0.784848,"std":0.006886},{"mean":0.859394,"std":0.007036},{"mean":0.997636,"std":0.00246}],"selective_risk":[{"mean":0.001361,"std":0.002357},{"mean":0.000756,"std":0.001309},{"mean":0.000955,"std":0.001654},{"mean":0.000726,"std":0.001258},{"mean":0.000865,"std":0.000859},{"mean":0.000947,"std":0.001064},{"mean":0.00125,"std":0.000601},{"mean":0.001619,"std":0.000521},{"mean":0.002103,"std":0.000751},{"mean":0.00231,"std":0.000651},{"mean":0.002339,"std":0.001002},{"mean":0.002634,"std":0.00089},{"mean":0.0031,"std":0.001157},{"mean":0.003681,"std":0.001332},{"mean":0.004171,"std":0.00109},{"mean":0.004702,"std":0.001197},{"mean":0.00779,"std":0.00102},{"mean":0.013127,"std":0.000793},{"mean":0.030115,"std":0.001925},{"mean":0.101385,"std":0.003783}],"accuracy_8":[{"mean":0.824121,"std":0.000897},{"mean":0.828,"std":0.001273},{"mean":0.831758,"std":0.000378},{"mean":0.835212,"std":0.001096},{"mean":0.839515,"std":0.00084},{"mean":0.844182,"std":0.001192},{"mean":0.848121,"std":0.002273},{"mean":0.852909,"std":0.002678},{"mean":0.857091,"std":0.003059},{"mean":0.861515,"std":0.002842},{"mean":0.865273,"std":0.003273},{"mean":0.870242,"std":0.004022},{"mean":0.875212,"std":0.004006},{"mean":0.880606,"std":0.003193},{"mean":0.886182,"std":0.001311},{"mean":0.892727,"std":0.001012},{"mean":0.898303,"std":0.001001},{"mean":0.904788,"std":0.000897},{"mean":0.912667,"std":0.001211},{"mean":0.898182,"std":0.003288}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000333,"std":0.000577},{"mean":0.000667,"std":0.001155},{"mean":0.001,"std":0.001},{"mean":0.002,"std":0.001},{"mean":0.003,"std":0.001},{"mean":0.004,"std":0.001},{"mean":0.004333,"std":0.001528},{"mean":0.005667,"std":0.001528},{"mean":0.007333,"std":0.002082},{"mean":0.009333,"std":0.002517},{"mean":0.011667,"std":0.002517},{"mean":0.014667,"std":0.003055},{"mean":0.026667,"std":0.004163},{"mean":0.047333,"std":0.004041},{"mean":0.121,"std":0.009539},{"mean":0.494333,"std":0.026102}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432333,"std":0.000577},{"mean":0.432,"std":0.001},{"mean":0.431667,"std":0.001528},{"mean":0.431,"std":0.002},{"mean":0.430333,"std":0.002309},{"mean":0.431667,"std":0.002517},{"mean":0.427667,"std":0.005132},{"mean":0.413667,"std":0.007767},{"mean":0.498,"std":0.022113}],"llm_call_rate":[{"mean":0.952424,"std":0.004784},{"mean":0.909758,"std":0.00872},{"mean":0.86703,"std":0.005299},{"mean":0.829333,"std":0.003254},{"mean":0.787273,"std":0.005894},{"mean":0.746485,"std":0.007893},{"mean":0.711515,"std":0.00849},{"mean":0.664061,"std":0.005147},{"mean":0.624848,"std":0.001944},{"mean":0.580424,"std":0.004091},{"mean":0.53497,"std":0.005623},{"mean":0.494061,"std":0.006468},{"mean":0.452788,"std":0.00378},{"mean":0.407394,"std":0.000757},{"mean":0.360364,"std":0.003963},{"mean":0.317636,"std":0.005536},{"mean":0.268545,"std":0.004983},{"mean":0.215152,"std":0.006886},{"mean":0.140606,"std":0.007036},{"mean":0.002364,"std":0.00246}]}}}}},"tfidf-centroid/k1":{"model":"tfidf-centroid","k":1,"oos_train":null,"kind":"tfidf","seeds":[42,43,44],"run_names":["tfidf-centroid-k1-seed42","tfidf-centroid-k1-seed43","tfidf-centroid-k1-seed44"],"logits_sha256":["94258de1803c08b0ba3fd2066c15b4814dfb701f499959b33319623e70d6089d","d2d86e9f3d9cb80ca61f3323aa17dcb00ce80afbddf52beddf89da9ccb841380","fe1f9f4b8c84c652cec9717dd20fd94591d7134748764883360c9fe27ef5dbc5"],"result":{"argmax":{"margin":{"risk_coverage":[{"mean":0.082667,"std":0.04197},{"mean":0.152121,"std":0.03203},{"mean":0.193131,"std":0.020725},{"mean":0.231818,"std":0.015851},{"mean":0.265697,"std":0.009901},{"mean":0.293939,"std":0.010087},{"mean":0.321732,"std":0.015972},{"mean":0.34803,"std":0.017618},{"mean":0.366599,"std":0.018579},{"mean":0.382909,"std":0.019797},{"mean":0.402094,"std":0.017406},{"mean":0.416061,"std":0.019373},{"mean":0.428718,"std":0.019303},{"mean":0.442597,"std":0.015098},{"mean":0.457455,"std":0.015365},{"mean":0.469318,"std":0.016368},{"mean":0.478217,"std":0.017166},{"mean":0.488418,"std":0.016053},{"mean":0.499904,"std":0.014435},{"mean":0.512121,"std":0.013706}],"operating":{"tau":[{"mean":5.519222,"std":0.366089},{"mean":4.312441,"std":0.214886},{"mean":3.563542,"std":0.205446},{"mean":2.996633,"std":0.165707},{"mean":2.523727,"std":0.101008},{"mean":2.130906,"std":0.054919},{"mean":1.815015,"std":0.07288},{"mean":1.551153,"std":0.078065},{"mean":1.332999,"std":0.077712},{"mean":1.11669,"std":0.056401},{"mean":0.923641,"std":0.038896},{"mean":0.761195,"std":0.017155},{"mean":0.619259,"std":0.031781},{"mean":0.492418,"std":0.036491},{"mean":0.391636,"std":0.028562},{"mean":0.296981,"std":0.027526},{"mean":0.214763,"std":0.018975},{"mean":0.1261,"std":0.013978},{"mean":0.052701,"std":0.010731},{"mean":0.0,"std":0.0}],"coverage":[{"mean":0.046242,"std":0.007528},{"mean":0.088667,"std":0.013314},{"mean":0.129515,"std":0.015439},{"mean":0.171394,"std":0.017689},{"mean":0.216848,"std":0.021185},{"mean":0.263394,"std":0.021255},{"mean":0.313091,"std":0.021861},{"mean":0.360848,"std":0.027039},{"mean":0.405455,"std":0.02946},{"mean":0.460242,"std":0.026455},{"mean":0.514909,"std":0.023578},{"mean":0.565455,"std":0.019337},{"mean":0.620909,"std":0.022759},{"mean":0.679091,"std":0.014038},{"mean":0.733515,"std":0.009},{"mean":0.790788,"std":0.010565},{"mean":0.843818,"std":0.006322},{"mean":0.901455,"std":0.005178},{"mean":0.953515,"std":0.006367},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.075359,"std":0.054378},{"mean":0.142505,"std":0.039661},{"mean":0.184035,"std":0.027789},{"mean":0.204041,"std":0.030699},{"mean":0.240202,"std":0.023998},{"mean":0.270579,"std":0.012548},{"mean":0.298178,"std":0.002503},{"mean":0.327495,"std":0.003913},{"mean":0.349527,"std":0.004436},{"mean":0.370313,"std":0.012099},{"mean":0.391076,"std":0.010329},{"mean":0.406269,"std":0.013671},{"mean":0.421676,"std":0.015378},{"mean":0.437269,"std":0.014852},{"mean":0.452363,"std":0.012335},{"mean":0.466727,"std":0.014492},{"mean":0.477708,"std":0.016318},{"mean":0.488551,"std":0.016296},{"mean":0.499845,"std":0.015505},{"mean":0.512121,"std":0.013706}],"accuracy_8":[{"mean":0.822848,"std":0.003119},{"mean":0.820061,"std":0.005214},{"mean":0.815212,"std":0.005993},{"mean":0.809818,"std":0.008129},{"mean":0.800242,"std":0.008069},{"mean":0.789212,"std":0.005819},{"mean":0.774061,"std":0.003888},{"mean":0.756485,"std":0.002309},{"mean":0.74097,"std":0.003364},{"mean":0.722545,"std":0.004299},{"mean":0.701333,"std":0.002956},{"mean":0.682061,"std":0.005912},{"mean":0.661091,"std":0.009326},{"mean":0.637394,"std":0.01059},{"mean":0.613212,"std":0.00696},{"mean":0.586848,"std":0.009388},{"mean":0.563758,"std":0.010928},{"mean":0.537636,"std":0.015111},{"mean":0.512424,"std":0.016859},{"mean":0.487879,"std":0.013706}],"high_conf_oos_misroute_rate":[{"mean":0.011333,"std":0.007767},{"mean":0.034333,"std":0.013577},{"mean":0.061333,"std":0.012503},{"mean":0.083,"std":0.013748},{"mean":0.111667,"std":0.019604},{"mean":0.151333,"std":0.015631},{"mean":0.190333,"std":0.022723},{"mean":0.239667,"std":0.025502},{"mean":0.289333,"std":0.028711},{"mean":0.340667,"std":0.023714},{"mean":0.395667,"std":0.021362},{"mean":0.450667,"std":0.012503},{"mean":0.506667,"std":0.020502},{"mean":0.571,"std":0.028618},{"mean":0.639667,"std":0.020257},{"mean":0.716333,"std":0.020551},{"mean":0.791,"std":0.01},{"mean":0.862,"std":0.020952},{"mean":0.930667,"std":0.015011},{"mean":0.988,"std":0.005196}],"oos_misroute_rate":[{"mean":0.438667,"std":0.005508},{"mean":0.452,"std":0.009539},{"mean":0.467,"std":0.010536},{"mean":0.479667,"std":0.011372},{"mean":0.493667,"std":0.015948},{"mean":0.515333,"std":0.015011},{"mean":0.538,"std":0.02265},{"mean":0.568333,"std":0.02318},{"mean":0.599333,"std":0.028378},{"mean":0.626667,"std":0.028378},{"mean":0.655667,"std":0.021733},{"mean":0.688,"std":0.017521},{"mean":0.718,"std":0.021633},{"mean":0.753667,"std":0.023965},{"mean":0.793667,"std":0.021502},{"mean":0.836333,"std":0.013317},{"mean":0.88,"std":0.006},{"mean":0.919333,"std":0.013204},{"mean":0.959333,"std":0.01159},{"mean":0.988,"std":0.005196}],"llm_call_rate":[{"mean":0.953758,"std":0.007528},{"mean":0.911333,"std":0.013314},{"mean":0.870485,"std":0.015439},{"mean":0.828606,"std":0.017689},{"mean":0.783152,"std":0.021185},{"mean":0.736606,"std":0.021255},{"mean":0.686909,"std":0.021861},{"mean":0.639152,"std":0.027039},{"mean":0.594545,"std":0.02946},{"mean":0.539758,"std":0.026455},{"mean":0.485091,"std":0.023578},{"mean":0.434545,"std":0.019337},{"mean":0.379091,"std":0.022759},{"mean":0.320909,"std":0.014038},{"mean":0.266485,"std":0.009},{"mean":0.209212,"std":0.010565},{"mean":0.156182,"std":0.006322},{"mean":0.098545,"std":0.005178},{"mean":0.046485,"std":0.006367},{"mean":0.0,"std":0.0}]}},"msp_t":{"risk_coverage":[{"mean":0.107879,"std":0.064675},{"mean":0.153939,"std":0.038412},{"mean":0.183838,"std":0.025608},{"mean":0.22303,"std":0.007943},{"mean":0.258667,"std":0.003359},{"mean":0.289697,"std":0.010196},{"mean":0.316364,"std":0.014141},{"mean":0.336667,"std":0.016416},{"mean":0.35771,"std":0.01701},{"mean":0.376121,"std":0.017764},{"mean":0.392287,"std":0.021693},{"mean":0.407374,"std":0.021599},{"mean":0.425641,"std":0.020888},{"mean":0.437403,"std":0.022279},{"mean":0.451313,"std":0.019455},{"mean":0.464167,"std":0.017982},{"mean":0.475223,"std":0.015755},{"mean":0.487542,"std":0.014942},{"mean":0.498501,"std":0.014199},{"mean":0.512121,"std":0.013706}],"operating":{"tau":[{"mean":0.948472,"std":0.016349},{"mean":0.849157,"std":0.029549},{"mean":0.739285,"std":0.026102},{"mean":0.633807,"std":0.016157},{"mean":0.552861,"std":0.029026},{"mean":0.474207,"std":0.023665},{"mean":0.410574,"std":0.014144},{"mean":0.357427,"std":0.012331},{"mean":0.309239,"std":0.011568},{"mean":0.268051,"std":0.011136},{"mean":0.230334,"std":0.012519},{"mean":0.200765,"std":0.012646},{"mean":0.173306,"std":0.011895},{"mean":0.149195,"std":0.008905},{"mean":0.12889,"std":0.005523},{"mean":0.109897,"std":0.003634},{"mean":0.093645,"std":0.003198},{"mean":0.077103,"std":0.001656},{"mean":0.059131,"std":0.000885},{"mean":0.006623,"std":0.0}],"coverage":[{"mean":0.042121,"std":0.009066},{"mean":0.087152,"std":0.0116},{"mean":0.128242,"std":0.015885},{"mean":0.176,"std":0.017815},{"mean":0.215758,"std":0.026973},{"mean":0.26303,"std":0.028111},{"mean":0.308182,"std":0.02632},{"mean":0.356364,"std":0.024938},{"mean":0.405394,"std":0.02324},{"mean":0.457212,"std":0.025458},{"mean":0.511697,"std":0.028638},{"mean":0.564061,"std":0.02931},{"mean":0.621818,"std":0.026521},{"mean":0.677333,"std":0.02347},{"mean":0.732303,"std":0.014995},{"mean":0.784121,"std":0.011021},{"mean":0.838606,"std":0.012954},{"mean":0.891939,"std":0.005144},{"mean":0.946061,"std":0.002819},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.100247,"std":0.071304},{"mean":0.138342,"std":0.056138},{"mean":0.169946,"std":0.039927},{"mean":0.198333,"std":0.025955},{"mean":0.228819,"std":0.021744},{"mean":0.264077,"std":0.012918},{"mean":0.290963,"std":0.007747},{"mean":0.31698,"std":0.0095},{"mean":0.338413,"std":0.004471},{"mean":0.360641,"std":0.012187},{"mean":0.378952,"std":0.012895},{"mean":0.394454,"std":0.011729},{"mean":0.415676,"std":0.013727},{"mean":0.432179,"std":0.018065},{"mean":0.446634,"std":0.017289},{"mean":0.460152,"std":0.015391},{"mean":0.471899,"std":0.011776},{"mean":0.485991,"std":0.014899},{"mean":0.497554,"std":0.014812},{"mean":0.512121,"std":0.013706}],"accuracy_8":[{"mean":0.82103,"std":0.003564},{"mean":0.820061,"std":0.006385},{"mean":0.816485,"std":0.007723},{"mean":0.810121,"std":0.006414},{"mean":0.801879,"std":0.008865},{"mean":0.789939,"std":0.007147},{"mean":0.775939,"std":0.005675},{"mean":0.76103,"std":0.004791},{"mean":0.744848,"std":0.003564},{"mean":0.726242,"std":0.00491},{"mean":0.706303,"std":0.005456},{"mean":0.686667,"std":0.004994},{"mean":0.662,"std":0.004639},{"mean":0.638364,"std":0.008659},{"mean":0.614606,"std":0.010922},{"mean":0.591515,"std":0.009344},{"mean":0.568303,"std":0.007341},{"mean":0.541636,"std":0.013993},{"mean":0.516364,"std":0.014936},{"mean":0.487879,"std":0.013706}],"high_conf_oos_misroute_rate":[{"mean":0.012,"std":0.00781},{"mean":0.030333,"std":0.011846},{"mean":0.051333,"std":0.014503},{"mean":0.075,"std":0.020518},{"mean":0.103333,"std":0.025007},{"mean":0.142333,"std":0.033081},{"mean":0.180667,"std":0.034819},{"mean":0.218333,"std":0.043547},{"mean":0.258667,"std":0.041041},{"mean":0.308,"std":0.040951},{"mean":0.363333,"std":0.041861},{"mean":0.413333,"std":0.042852},{"mean":0.482667,"std":0.036529},{"mean":0.553,"std":0.03005},{"mean":0.615333,"std":0.021939},{"mean":0.682333,"std":0.023072},{"mean":0.749333,"std":0.026102},{"mean":0.825333,"std":0.002309},{"mean":0.908667,"std":0.008386},{"mean":0.988,"std":0.005196}],"oos_misroute_rate":[{"mean":0.439667,"std":0.005508},{"mean":0.449333,"std":0.010116},{"mean":0.459667,"std":0.010786},{"mean":0.473667,"std":0.014012},{"mean":0.49,"std":0.020298},{"mean":0.509333,"std":0.027574},{"mean":0.536,"std":0.033645},{"mean":0.555667,"std":0.038083},{"mean":0.577,"std":0.040112},{"mean":0.605333,"std":0.040216},{"mean":0.637,"std":0.038743},{"mean":0.667333,"std":0.034646},{"mean":0.703,"std":0.032512},{"mean":0.742,"std":0.031607},{"mean":0.778,"std":0.023643},{"mean":0.815667,"std":0.025541},{"mean":0.849333,"std":0.018556},{"mean":0.894667,"std":0.004509},{"mean":0.942333,"std":0.006506},{"mean":0.988,"std":0.005196}],"llm_call_rate":[{"mean":0.957879,"std":0.009066},{"mean":0.912848,"std":0.0116},{"mean":0.871758,"std":0.015885},{"mean":0.824,"std":0.017815},{"mean":0.784242,"std":0.026973},{"mean":0.73697,"std":0.028111},{"mean":0.691818,"std":0.02632},{"mean":0.643636,"std":0.024938},{"mean":0.594606,"std":0.02324},{"mean":0.542788,"std":0.025458},{"mean":0.488303,"std":0.028638},{"mean":0.435939,"std":0.02931},{"mean":0.378182,"std":0.026521},{"mean":0.322667,"std":0.02347},{"mean":0.267697,"std":0.014995},{"mean":0.215879,"std":0.011021},{"mean":0.161394,"std":0.012954},{"mean":0.108061,"std":0.005144},{"mean":0.053939,"std":0.002819},{"mean":0.0,"std":0.0}]}}},"summed":{"margin":{"risk_coverage":[{"mean":0.082424,"std":0.051469},{"mean":0.128485,"std":0.049638},{"mean":0.156364,"std":0.02772},{"mean":0.178712,"std":0.022082},{"mean":0.208727,"std":0.011006},{"mean":0.227879,"std":0.002642},{"mean":0.249524,"std":0.007053},{"mean":0.27197,"std":0.011821},{"mean":0.294411,"std":0.015398},{"mean":0.316606,"std":0.015056},{"mean":0.33708,"std":0.014003},{"mean":0.359293,"std":0.013567},{"mean":0.38014,"std":0.012846},{"mean":0.398701,"std":0.013604},{"mean":0.418828,"std":0.015398},{"mean":0.435833,"std":0.016202},{"mean":0.455401,"std":0.015474},{"mean":0.471515,"std":0.015262},{"mean":0.487719,"std":0.013079},{"mean":0.502,"std":0.011789}],"operating":{"tau":[{"mean":4.35213,"std":0.297778},{"mean":3.230181,"std":0.167548},{"mean":2.598748,"std":0.097989},{"mean":2.12894,"std":0.057534},{"mean":1.797543,"std":0.061085},{"mean":1.549748,"std":0.05109},{"mean":1.329,"std":0.045969},{"mean":1.156141,"std":0.051166},{"mean":0.995692,"std":0.04668},{"mean":0.848293,"std":0.053475},{"mean":0.721045,"std":0.058253},{"mean":0.592319,"std":0.064188},{"mean":0.484036,"std":0.050032},{"mean":0.388588,"std":0.033507},{"mean":0.305466,"std":0.027866},{"mean":0.233419,"std":0.020887},{"mean":0.171564,"std":0.008171},{"mean":0.116501,"std":0.008271},{"mean":0.053961,"std":0.003074},{"mean":0.000312,"std":0.000204}],"coverage":[{"mean":0.042,"std":0.007948},{"mean":0.087333,"std":0.008498},{"mean":0.128606,"std":0.010009},{"mean":0.174364,"std":0.014891},{"mean":0.222606,"std":0.018444},{"mean":0.264,"std":0.022318},{"mean":0.307091,"std":0.024715},{"mean":0.349879,"std":0.026879},{"mean":0.396667,"std":0.025895},{"mean":0.445636,"std":0.032237},{"mean":0.497394,"std":0.038019},{"mean":0.552424,"std":0.04223},{"mean":0.608424,"std":0.036914},{"mean":0.662,"std":0.031064},{"mean":0.715636,"std":0.027713},{"mean":0.77097,"std":0.021653},{"mean":0.829758,"std":0.015978},{"mean":0.882424,"std":0.014382},{"mean":0.942788,"std":0.007938},{"mean":0.999576,"std":0.000278}],"selective_risk":[{"mean":0.079359,"std":0.057643},{"mean":0.110326,"std":0.059936},{"mean":0.146398,"std":0.039728},{"mean":0.168136,"std":0.029005},{"mean":0.190776,"std":0.026235},{"mean":0.211505,"std":0.01515},{"mean":0.229025,"std":0.009822},{"mean":0.249367,"std":0.008419},{"mean":0.268385,"std":0.004125},{"mean":0.29273,"std":0.006382},{"mean":0.31525,"std":0.004711},{"mean":0.338832,"std":0.005108},{"mean":0.362958,"std":0.004877},{"mean":0.384706,"std":0.003438},{"mean":0.405204,"std":0.008437},{"mean":0.425453,"std":0.012743},{"mean":0.448101,"std":0.015271},{"mean":0.466455,"std":0.015452},{"mean":0.485592,"std":0.012823},{"mean":0.501909,"std":0.012064}],"accuracy_8":[{"mean":0.821394,"std":0.003315},{"mean":0.821333,"std":0.00646},{"mean":0.817515,"std":0.007164},{"mean":0.814242,"std":0.006884},{"mean":0.807576,"std":0.008052},{"mean":0.800121,"std":0.008176},{"mean":0.791818,"std":0.007494},{"mean":0.781394,"std":0.007598},{"mean":0.769455,"std":0.007079},{"mean":0.753576,"std":0.00631},{"mean":0.73697,"std":0.006934},{"mean":0.716727,"std":0.009616},{"mean":0.692182,"std":0.009091},{"mean":0.670303,"std":0.006549},{"mean":0.646424,"std":0.008011},{"mean":0.620182,"std":0.010332},{"mean":0.590667,"std":0.01434},{"mean":0.561455,"std":0.015273},{"mean":0.529636,"std":0.013228},{"mean":0.498182,"std":0.011968}],"high_conf_oos_misroute_rate":[{"mean":0.009333,"std":0.006658},{"mean":0.028,"std":0.01562},{"mean":0.048,"std":0.015},{"mean":0.072,"std":0.021071},{"mean":0.099,"std":0.027622},{"mean":0.125,"std":0.027622},{"mean":0.152667,"std":0.024685},{"mean":0.186333,"std":0.039004},{"mean":0.226,"std":0.034598},{"mean":0.274667,"std":0.041741},{"mean":0.333667,"std":0.046458},{"mean":0.393333,"std":0.051733},{"mean":0.455667,"std":0.046458},{"mean":0.526667,"std":0.039463},{"mean":0.593667,"std":0.031943},{"mean":0.67,"std":0.020518},{"mean":0.758667,"std":0.013051},{"mean":0.831,"std":0.011533},{"mean":0.917333,"std":0.01097},{"mean":0.998333,"std":0.000577}],"oos_misroute_rate":[{"mean":0.438333,"std":0.004933},{"mean":0.448333,"std":0.013577},{"mean":0.459,"std":0.013115},{"mean":0.471,"std":0.015716},{"mean":0.488667,"std":0.022502},{"mean":0.500667,"std":0.027683},{"mean":0.514,"std":0.031749},{"mean":0.533667,"std":0.04236},{"mean":0.553333,"std":0.041016},{"mean":0.579333,"std":0.041187},{"mean":0.612,"std":0.035},{"mean":0.644,"std":0.039837},{"mean":0.682333,"std":0.036116},{"mean":0.722667,"std":0.036665},{"mean":0.764333,"std":0.030892},{"mean":0.810333,"std":0.024786},{"mean":0.863,"std":0.019157},{"mean":0.904333,"std":0.014364},{"mean":0.952,"std":0.010536},{"mean":0.998667,"std":0.001155}],"llm_call_rate":[{"mean":0.958,"std":0.007948},{"mean":0.912667,"std":0.008498},{"mean":0.871394,"std":0.010009},{"mean":0.825636,"std":0.014891},{"mean":0.777394,"std":0.018444},{"mean":0.736,"std":0.022318},{"mean":0.692909,"std":0.024715},{"mean":0.650121,"std":0.026879},{"mean":0.603333,"std":0.025895},{"mean":0.554364,"std":0.032237},{"mean":0.502606,"std":0.038019},{"mean":0.447576,"std":0.04223},{"mean":0.391576,"std":0.036914},{"mean":0.338,"std":0.031064},{"mean":0.284364,"std":0.027713},{"mean":0.22903,"std":0.021653},{"mean":0.170242,"std":0.015978},{"mean":0.117576,"std":0.014382},{"mean":0.057212,"std":0.007938},{"mean":0.000424,"std":0.000278}]}},"msp_t":{"risk_coverage":[{"mean":0.092121,"std":0.050168},{"mean":0.135152,"std":0.050135},{"mean":0.160808,"std":0.026063},{"mean":0.188384,"std":0.013801},{"mean":0.215758,"std":0.003279},{"mean":0.239798,"std":0.006648},{"mean":0.262857,"std":0.009394},{"mean":0.281818,"std":0.014179},{"mean":0.303704,"std":0.010761},{"mean":0.324,"std":0.014069},{"mean":0.340606,"std":0.010164},{"mean":0.359596,"std":0.011993},{"mean":0.378182,"std":0.013569},{"mean":0.397662,"std":0.014545},{"mean":0.416323,"std":0.015047},{"mean":0.433485,"std":0.014829},{"mean":0.450909,"std":0.017087},{"mean":0.468418,"std":0.015056},{"mean":0.484785,"std":0.014474},{"mean":0.502,"std":0.011789}],"operating":{"tau":[{"mean":0.961023,"std":0.011341},{"mean":0.892075,"std":0.015838},{"mean":0.813693,"std":0.019916},{"mean":0.735702,"std":0.015801},{"mean":0.667323,"std":0.012253},{"mean":0.611599,"std":0.011205},{"mean":0.559416,"std":0.013042},{"mean":0.514215,"std":0.010579},{"mean":0.476004,"std":0.009312},{"mean":0.442523,"std":0.013202},{"mean":0.408338,"std":0.012965},{"mean":0.379413,"std":0.012925},{"mean":0.351086,"std":0.013237},{"mean":0.326355,"std":0.011573},{"mean":0.30615,"std":0.01223},{"mean":0.287278,"std":0.010216},{"mean":0.267329,"std":0.009537},{"mean":0.252323,"std":0.005816},{"mean":0.231022,"std":0.007022},{"mean":0.18622,"std":0.008079}],"coverage":[{"mean":0.041394,"std":0.008474},{"mean":0.085152,"std":0.009106},{"mean":0.128061,"std":0.012184},{"mean":0.173697,"std":0.017211},{"mean":0.22,"std":0.020548},{"mean":0.261758,"std":0.022509},{"mean":0.306909,"std":0.024208},{"mean":0.350909,"std":0.027259},{"mean":0.398485,"std":0.02542},{"mean":0.445273,"std":0.025567},{"mean":0.498121,"std":0.027786},{"mean":0.548424,"std":0.03291},{"mean":0.608364,"std":0.034397},{"mean":0.664242,"std":0.030542},{"mean":0.717636,"std":0.029845},{"mean":0.775455,"std":0.026872},{"mean":0.834909,"std":0.025985},{"mean":0.887636,"std":0.015738},{"mean":0.948424,"std":0.010455},{"mean":0.999455,"std":0.000793}],"selective_risk":[{"mean":0.085294,"std":0.057369},{"mean":0.12127,"std":0.0614},{"mean":0.149172,"std":0.035776},{"mean":0.172825,"std":0.024221},{"mean":0.197481,"std":0.01864},{"mean":0.219206,"std":0.009246},{"mean":0.24255,"std":0.008874},{"mean":0.257973,"std":0.008902},{"mean":0.281447,"std":0.003088},{"mean":0.30116,"std":0.002662},{"mean":0.323098,"std":0.002908},{"mean":0.340228,"std":0.002511},{"mean":0.361736,"std":0.001447},{"mean":0.384014,"std":0.003928},{"mean":0.404374,"std":0.00962},{"mean":0.42614,"std":0.012637},{"mean":0.445869,"std":0.015894},{"mean":0.464408,"std":0.014776},{"mean":0.484403,"std":0.015054},{"mean":0.501726,"std":0.011904}],"accuracy_8":[{"mean":0.821212,"std":0.003098},{"mean":0.82103,"std":0.006102},{"mean":0.818242,"std":0.006948},{"mean":0.813576,"std":0.006874},{"mean":0.807152,"std":0.006862},{"mean":0.799939,"std":0.006779},{"mean":0.789394,"std":0.006924},{"mean":0.779939,"std":0.007702},{"mean":0.765758,"std":0.004644},{"mean":0.750667,"std":0.004034},{"mean":0.732424,"std":0.005144},{"mean":0.715515,"std":0.005779},{"mean":0.694424,"std":0.00677},{"mean":0.671273,"std":0.004738},{"mean":0.64703,"std":0.009841},{"mean":0.61903,"std":0.012661},{"mean":0.590364,"std":0.015474},{"mean":0.562,"std":0.01547},{"mean":0.529636,"std":0.014497},{"mean":0.498364,"std":0.011963}],"high_conf_oos_misroute_rate":[{"mean":0.009667,"std":0.006351},{"mean":0.029,"std":0.01562},{"mean":0.048,"std":0.014422},{"mean":0.071,"std":0.017521},{"mean":0.097,"std":0.024515},{"mean":0.129667,"std":0.023502},{"mean":0.158333,"std":0.026577},{"mean":0.188667,"std":0.02829},{"mean":0.228333,"std":0.03177},{"mean":0.273,"std":0.043863},{"mean":0.328667,"std":0.050292},{"mean":0.383667,"std":0.059197},{"mean":0.452667,"std":0.054169},{"mean":0.525333,"std":0.051598},{"mean":0.599,"std":0.04471},{"mean":0.678667,"std":0.045015},{"mean":0.759333,"std":0.036199},{"mean":0.832,"std":0.024021},{"mean":0.923333,"std":0.028042},{"mean":0.997,"std":0.002646}],"oos_misroute_rate":[{"mean":0.438,"std":0.003464},{"mean":0.447667,"std":0.013317},{"mean":0.458333,"std":0.013051},{"mean":0.469,"std":0.01253},{"mean":0.485,"std":0.020664},{"mean":0.502667,"std":0.023029},{"mean":0.519333,"std":0.028431},{"mean":0.533,"std":0.033867},{"mean":0.556333,"std":0.036116},{"mean":0.582333,"std":0.042336},{"mean":0.614333,"std":0.047606},{"mean":0.643,"std":0.051098},{"mean":0.678,"std":0.045902},{"mean":0.718333,"std":0.043879},{"mean":0.761,"std":0.035086},{"mean":0.807667,"std":0.037005},{"mean":0.854,"std":0.031796},{"mean":0.899333,"std":0.024007},{"mean":0.950333,"std":0.021548},{"mean":0.998,"std":0.001}],"llm_call_rate":[{"mean":0.958606,"std":0.008474},{"mean":0.914848,"std":0.009106},{"mean":0.871939,"std":0.012184},{"mean":0.826303,"std":0.017211},{"mean":0.78,"std":0.020548},{"mean":0.738242,"std":0.022509},{"mean":0.693091,"std":0.024208},{"mean":0.649091,"std":0.027259},{"mean":0.601515,"std":0.02542},{"mean":0.554727,"std":0.025567},{"mean":0.501879,"std":0.027786},{"mean":0.451576,"std":0.03291},{"mean":0.391636,"std":0.034397},{"mean":0.335758,"std":0.030542},{"mean":0.282364,"std":0.029845},{"mean":0.224545,"std":0.026872},{"mean":0.165091,"std":0.025985},{"mean":0.112364,"std":0.015738},{"mean":0.051576,"std":0.010455},{"mean":0.000545,"std":0.000793}]}}}}},"tfidf-centroid/k10":{"model":"tfidf-centroid","k":10,"oos_train":null,"kind":"tfidf","seeds":[42,43,44],"run_names":["tfidf-centroid-k10-seed42","tfidf-centroid-k10-seed43","tfidf-centroid-k10-seed44"],"logits_sha256":["efa61924b5913a37d54b9d4a3d13cd5be52fa35b56491dbfa9709fc8755b1aef","aa4fc1ec5e6cdcac08cd404d778130f36725e6151a54f9ab2be795d38ed523e2","cfb9163965d84af33aef86d28145073b143ebb604bd9901d37b18d54d842db22"],"result":{"argmax":{"margin":{"risk_coverage":[{"mean":0.004848,"std":0.002099},{"mean":0.017576,"std":0.007348},{"mean":0.02101,"std":0.005728},{"mean":0.024848,"std":0.00864},{"mean":0.029091,"std":0.007014},{"mean":0.03596,"std":0.008492},{"mean":0.042944,"std":0.009122},{"mean":0.053636,"std":0.008708},{"mean":0.066667,"std":0.009872},{"mean":0.080364,"std":0.008387},{"mean":0.093664,"std":0.008167},{"mean":0.108687,"std":0.011477},{"mean":0.127832,"std":0.011577},{"mean":0.149524,"std":0.010628},{"mean":0.169212,"std":0.007882},{"mean":0.187803,"std":0.008186},{"mean":0.209198,"std":0.007298},{"mean":0.229697,"std":0.006423},{"mean":0.249378,"std":0.00635},{"mean":0.268788,"std":0.004543}],"operating":{"tau":[{"mean":10.734296,"std":0.218853},{"mean":8.862022,"std":0.24621},{"mean":7.70325,"std":0.175823},{"mean":6.838078,"std":0.170527},{"mean":6.034575,"std":0.210114},{"mean":5.355008,"std":0.235491},{"mean":4.757876,"std":0.192476},{"mean":4.223174,"std":0.156942},{"mean":3.693809,"std":0.138138},{"mean":3.203142,"std":0.131201},{"mean":2.756695,"std":0.1079},{"mean":2.317057,"std":0.069284},{"mean":1.90506,"std":0.024612},{"mean":1.550137,"std":0.029453},{"mean":1.192372,"std":0.054255},{"mean":0.890538,"std":0.034146},{"mean":0.614487,"std":0.015422},{"mean":0.368299,"std":0.017535},{"mean":0.159789,"std":0.000566},{"mean":0.0,"std":0.0}],"coverage":[{"mean":0.039152,"std":0.001546},{"mean":0.083515,"std":0.001689},{"mean":0.124848,"std":0.003066},{"mean":0.167333,"std":0.002911},{"mean":0.212061,"std":0.005258},{"mean":0.256848,"std":0.010375},{"mean":0.302121,"std":0.009972},{"mean":0.343939,"std":0.008447},{"mean":0.389576,"std":0.005707},{"mean":0.435455,"std":0.007184},{"mean":0.483455,"std":0.006472},{"mean":0.532061,"std":0.007005},{"mean":0.583515,"std":0.002649},{"mean":0.634909,"std":0.006117},{"mean":0.696424,"std":0.010343},{"mean":0.756061,"std":0.004842},{"mean":0.814,"std":0.005491},{"mean":0.879576,"std":0.004937},{"mean":0.944606,"std":0.003432},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.002963,"std":0.005132},{"mean":0.016757,"std":0.004913},{"mean":0.019039,"std":0.006709},{"mean":0.022734,"std":0.007214},{"mean":0.025862,"std":0.008843},{"mean":0.029075,"std":0.007633},{"mean":0.036705,"std":0.009731},{"mean":0.040848,"std":0.012919},{"mean":0.050007,"std":0.009951},{"mean":0.062397,"std":0.010763},{"mean":0.076127,"std":0.010495},{"mean":0.088428,"std":0.008973},{"mean":0.104141,"std":0.011524},{"mean":0.120859,"std":0.013723},{"mean":0.148441,"std":0.01335},{"mean":0.171194,"std":0.008832},{"mean":0.19324,"std":0.010349},{"mean":0.222871,"std":0.00881},{"mean":0.247286,"std":0.005349},{"mean":0.268788,"std":0.004543}],"accuracy_8":[{"mean":0.822303,"std":0.000525},{"mean":0.825091,"std":0.000727},{"mean":0.828121,"std":0.000105},{"mean":0.831152,"std":0.000584},{"mean":0.834485,"std":0.001065},{"mean":0.839818,"std":0.001666},{"mean":0.843273,"std":0.001091},{"mean":0.846606,"std":0.002273},{"mean":0.848303,"std":0.002783},{"mean":0.848424,"std":0.003909},{"mean":0.846667,"std":0.004846},{"mean":0.845455,"std":0.004062},{"mean":0.842667,"std":0.005728},{"mean":0.835818,"std":0.007468},{"mean":0.823697,"std":0.006748},{"mean":0.810545,"std":0.005282},{"mean":0.794424,"std":0.005912},{"mean":0.773273,"std":0.006563},{"mean":0.751697,"std":0.004883},{"mean":0.731212,"std":0.004543}],"high_conf_oos_misroute_rate":[{"mean":0.000667,"std":0.001155},{"mean":0.006,"std":0.001732},{"mean":0.01,"std":0.003606},{"mean":0.016333,"std":0.004619},{"mean":0.022,"std":0.007211},{"mean":0.028667,"std":0.009074},{"mean":0.042,"std":0.013},{"mean":0.054667,"std":0.019035},{"mean":0.075,"std":0.017521},{"mean":0.104333,"std":0.023714},{"mean":0.139667,"std":0.020207},{"mean":0.176667,"std":0.018502},{"mean":0.228333,"std":0.025106},{"mean":0.288667,"std":0.033858},{"mean":0.387667,"std":0.038501},{"mean":0.481667,"std":0.029366},{"mean":0.576333,"std":0.039463},{"mean":0.720333,"std":0.034078},{"mean":0.858667,"std":0.018877},{"mean":0.979333,"std":0.015885}],"oos_misroute_rate":[{"mean":0.432667,"std":0.001155},{"mean":0.435,"std":0.002},{"mean":0.437333,"std":0.003215},{"mean":0.440333,"std":0.003055},{"mean":0.443,"std":0.003606},{"mean":0.447,"std":0.005},{"mean":0.454333,"std":0.007095},{"mean":0.462333,"std":0.01106},{"mean":0.474333,"std":0.01159},{"mean":0.492,"std":0.013892},{"mean":0.513667,"std":0.01193},{"mean":0.531,"std":0.011269},{"mean":0.558,"std":0.018193},{"mean":0.591,"std":0.024331},{"mean":0.644,"std":0.02488},{"mean":0.697,"std":0.018735},{"mean":0.754333,"std":0.022942},{"mean":0.835667,"std":0.023159},{"mean":0.910333,"std":0.020404},{"mean":0.979333,"std":0.015885}],"llm_call_rate":[{"mean":0.960848,"std":0.001546},{"mean":0.916485,"std":0.001689},{"mean":0.875152,"std":0.003066},{"mean":0.832667,"std":0.002911},{"mean":0.787939,"std":0.005258},{"mean":0.743152,"std":0.010375},{"mean":0.697879,"std":0.009972},{"mean":0.656061,"std":0.008447},{"mean":0.610424,"std":0.005707},{"mean":0.564545,"std":0.007184},{"mean":0.516545,"std":0.006472},{"mean":0.467939,"std":0.007005},{"mean":0.416485,"std":0.002649},{"mean":0.365091,"std":0.006117},{"mean":0.303576,"std":0.010343},{"mean":0.243939,"std":0.004842},{"mean":0.186,"std":0.005491},{"mean":0.120424,"std":0.004937},{"mean":0.055394,"std":0.003432},{"mean":0.0,"std":0.0}]}},"msp_t":{"risk_coverage":[{"mean":0.008485,"std":0.00757},{"mean":0.018182,"std":0.006556},{"mean":0.02101,"std":0.006892},{"mean":0.026061,"std":0.006823},{"mean":0.028121,"std":0.006354},{"mean":0.034949,"std":0.007578},{"mean":0.044848,"std":0.009612},{"mean":0.052273,"std":0.009393},{"mean":0.061684,"std":0.009742},{"mean":0.070788,"std":0.009901},{"mean":0.082534,"std":0.009079},{"mean":0.09596,"std":0.0086},{"mean":0.11049,"std":0.010357},{"mean":0.128571,"std":0.011057},{"mean":0.148606,"std":0.011704},{"mean":0.167121,"std":0.008655},{"mean":0.190731,"std":0.006208},{"mean":0.21367,"std":0.006789},{"mean":0.238915,"std":0.004724},{"mean":0.268788,"std":0.004543}],"operating":{"tau":[{"mean":0.999839,"std":4.5e-05},{"mean":0.999225,"std":0.000223},{"mean":0.997622,"std":0.000486},{"mean":0.99456,"std":0.000965},{"mean":0.989152,"std":0.001982},{"mean":0.979663,"std":0.004941},{"mean":0.964627,"std":0.008772},{"mean":0.942458,"std":0.0141},{"mean":0.909256,"std":0.015914},{"mean":0.865802,"std":0.020011},{"mean":0.804429,"std":0.025398},{"mean":0.736345,"std":0.031611},{"mean":0.659036,"std":0.033934},{"mean":0.56818,"std":0.021954},{"mean":0.481234,"std":0.019069},{"mean":0.386632,"std":0.016441},{"mean":0.294791,"std":0.013836},{"mean":0.209888,"std":0.009447},{"mean":0.121252,"std":0.001859},{"mean":0.006623,"std":0.0}],"coverage":[{"mean":0.041515,"std":0.003611},{"mean":0.080667,"std":0.005699},{"mean":0.128182,"std":0.003154},{"mean":0.170364,"std":0.002823},{"mean":0.21103,"std":0.000378},{"mean":0.255152,"std":0.005699},{"mean":0.296303,"std":0.010779},{"mean":0.338606,"std":0.013472},{"mean":0.384485,"std":0.010388},{"mean":0.429333,"std":0.006671},{"mean":0.478182,"std":0.006426},{"mean":0.524121,"std":0.009144},{"mean":0.572545,"std":0.00817},{"mean":0.624182,"std":0.006301},{"mean":0.676364,"std":0.002144},{"mean":0.729758,"std":0.003279},{"mean":0.789273,"std":0.005566},{"mean":0.849818,"std":0.010084},{"mean":0.924485,"std":0.006637},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.010215,"std":0.009466},{"mean":0.016536,"std":0.006658},{"mean":0.019775,"std":0.007096},{"mean":0.021775,"std":0.006779},{"mean":0.025842,"std":0.007327},{"mean":0.030092,"std":0.007673},{"mean":0.034748,"std":0.009459},{"mean":0.042503,"std":0.011118},{"mean":0.049318,"std":0.010519},{"mean":0.05821,"std":0.01051},{"mean":0.067968,"std":0.010633},{"mean":0.076536,"std":0.011189},{"mean":0.087357,"std":0.008592},{"mean":0.102507,"std":0.009916},{"mean":0.118534,"std":0.011838},{"mean":0.14092,"std":0.010665},{"mean":0.162425,"std":0.007776},{"mean":0.190493,"std":0.001701},{"mean":0.225582,"std":0.004171},{"mean":0.268788,"std":0.004543}],"accuracy_8":[{"mean":0.822424,"std":0.000735},{"mean":0.824788,"std":0.000525},{"mean":0.829091,"std":0.000727},{"mean":0.831273,"std":0.000909},{"mean":0.83297,"std":0.001183},{"mean":0.837879,"std":0.00082},{"mean":0.840727,"std":0.001488},{"mean":0.84297,"std":0.001747},{"mean":0.845636,"std":0.001012},{"mean":0.846848,"std":0.002819},{"mean":0.84697,"std":0.002842},{"mean":0.847212,"std":0.003234},{"mean":0.847152,"std":0.003422},{"mean":0.843212,"std":0.004462},{"mean":0.838485,"std":0.007462},{"mean":0.827939,"std":0.006156},{"mean":0.815091,"std":0.003938},{"mean":0.796,"std":0.001192},{"mean":0.768485,"std":0.005227},{"mean":0.731212,"std":0.004543}],"high_conf_oos_misroute_rate":[{"mean":0.001667,"std":0.001528},{"mean":0.005,"std":0.001732},{"mean":0.011,"std":0.004},{"mean":0.014667,"std":0.003215},{"mean":0.020667,"std":0.004041},{"mean":0.029333,"std":0.007024},{"mean":0.039333,"std":0.012662},{"mean":0.055333,"std":0.019009},{"mean":0.071667,"std":0.019088},{"mean":0.093333,"std":0.021548},{"mean":0.124,"std":0.022338},{"mean":0.148,"std":0.024249},{"mean":0.183,"std":0.024556},{"mean":0.231333,"std":0.034078},{"mean":0.281,"std":0.035679},{"mean":0.357667,"std":0.021385},{"mean":0.446333,"std":0.012503},{"mean":0.575333,"std":0.01365},{"mean":0.746667,"std":0.018448},{"mean":0.979333,"std":0.015885}],"oos_misroute_rate":[{"mean":0.433667,"std":0.001528},{"mean":0.434667,"std":0.002517},{"mean":0.437,"std":0.003},{"mean":0.44,"std":0.002646},{"mean":0.443333,"std":0.002887},{"mean":0.446333,"std":0.003512},{"mean":0.452667,"std":0.008021},{"mean":0.461667,"std":0.011676},{"mean":0.471667,"std":0.012702},{"mean":0.485,"std":0.01179},{"mean":0.501333,"std":0.011547},{"mean":0.514,"std":0.013528},{"mean":0.532667,"std":0.017786},{"mean":0.556667,"std":0.022053},{"mean":0.580333,"std":0.027025},{"mean":0.619,"std":0.014107},{"mean":0.668333,"std":0.010599},{"mean":0.740333,"std":0.007767},{"mean":0.839333,"std":0.012342},{"mean":0.979333,"std":0.015885}],"llm_call_rate":[{"mean":0.958485,"std":0.003611},{"mean":0.919333,"std":0.005699},{"mean":0.871818,"std":0.003154},{"mean":0.829636,"std":0.002823},{"mean":0.78897,"std":0.000378},{"mean":0.744848,"std":0.005699},{"mean":0.703697,"std":0.010779},{"mean":0.661394,"std":0.013472},{"mean":0.615515,"std":0.010388},{"mean":0.570667,"std":0.006671},{"mean":0.521818,"std":0.006426},{"mean":0.475879,"std":0.009144},{"mean":0.427455,"std":0.00817},{"mean":0.375818,"std":0.006301},{"mean":0.323636,"std":0.002144},{"mean":0.270242,"std":0.003279},{"mean":0.210727,"std":0.005566},{"mean":0.150182,"std":0.010084},{"mean":0.075515,"std":0.006637},{"mean":0.0,"std":0.0}]}}},"summed":{"margin":{"risk_coverage":[{"mean":0.008485,"std":0.005555},{"mean":0.013333,"std":0.004576},{"mean":0.016162,"std":0.006676},{"mean":0.018182,"std":0.003963},{"mean":0.023515,"std":0.005459},{"mean":0.029293,"std":0.007406},{"mean":0.034805,"std":0.008739},{"mean":0.039242,"std":0.008592},{"mean":0.046869,"std":0.011226},{"mean":0.056242,"std":0.011234},{"mean":0.067438,"std":0.010087},{"mean":0.080606,"std":0.009467},{"mean":0.096503,"std":0.009205},{"mean":0.112035,"std":0.008651},{"mean":0.132364,"std":0.006827},{"mean":0.157803,"std":0.006389},{"mean":0.185597,"std":0.004111},{"mean":0.214007,"std":0.005007},{"mean":0.243445,"std":0.003736},{"mean":0.271394,"std":0.003413}],"operating":{"tau":[{"mean":10.891061,"std":0.233596},{"mean":9.376087,"std":0.320634},{"mean":8.346508,"std":0.298419},{"mean":7.450921,"std":0.256211},{"mean":6.782101,"std":0.273855},{"mean":6.115709,"std":0.323008},{"mean":5.478696,"std":0.290303},{"mean":4.919935,"std":0.325158},{"mean":4.417175,"std":0.336717},{"mean":3.906549,"std":0.306378},{"mean":3.438139,"std":0.248701},{"mean":2.981563,"std":0.210881},{"mean":2.524065,"std":0.119478},{"mean":2.080559,"std":0.145261},{"mean":1.65366,"std":0.136716},{"mean":1.216716,"std":0.094834},{"mean":0.812363,"std":0.049778},{"mean":0.490638,"std":0.04248},{"mean":0.196336,"std":0.01047},{"mean":0.002337,"std":0.000319}],"coverage":[{"mean":0.038485,"std":0.000688},{"mean":0.082364,"std":0.004524},{"mean":0.123333,"std":0.006716},{"mean":0.16897,"std":0.002414},{"mean":0.210606,"std":0.001892},{"mean":0.255273,"std":0.006771},{"mean":0.300848,"std":0.006941},{"mean":0.347758,"std":0.010163},{"mean":0.388303,"std":0.009768},{"mean":0.431697,"std":0.014224},{"mean":0.478121,"std":0.015403},{"mean":0.525758,"std":0.010343},{"mean":0.571152,"std":0.004433},{"mean":0.619636,"std":0.008797},{"mean":0.672545,"std":0.009954},{"mean":0.73303,"std":0.008644},{"mean":0.797818,"std":0.002993},{"mean":0.858909,"std":0.007098},{"mean":0.933091,"std":0.003623},{"mean":0.99903,"std":0.000584}],"selective_risk":[{"mean":0.011053,"std":0.007354},{"mean":0.014074,"std":0.003943},{"mean":0.014886,"std":0.005568},{"mean":0.016161,"std":0.004763},{"mean":0.01899,"std":0.00375},{"mean":0.023881,"std":0.005779},{"mean":0.029297,"std":0.008112},{"mean":0.03438,"std":0.009186},{"mean":0.037614,"std":0.009812},{"mean":0.044253,"std":0.01141},{"mean":0.05179,"std":0.01443},{"mean":0.061365,"std":0.014592},{"mean":0.073161,"std":0.012865},{"mean":0.087125,"std":0.012763},{"mean":0.102797,"std":0.01166},{"mean":0.124428,"std":0.011191},{"mean":0.156719,"std":0.005993},{"mean":0.190836,"std":0.007447},{"mean":0.233052,"std":0.003157},{"mean":0.270809,"std":0.003064}],"accuracy_8":[{"mean":0.822182,"std":0.000545},{"mean":0.824424,"std":0.000897},{"mean":0.82697,"std":0.001458},{"mean":0.830909,"std":0.001575},{"mean":0.834121,"std":0.001365},{"mean":0.837333,"std":0.000378},{"mean":0.839576,"std":0.000915},{"mean":0.842667,"std":0.002302},{"mean":0.846,"std":0.001135},{"mean":0.848242,"std":0.001111},{"mean":0.849636,"std":0.002745},{"mean":0.850182,"std":0.004703},{"mean":0.850242,"std":0.006225},{"mean":0.847455,"std":0.006332},{"mean":0.843576,"std":0.006403},{"mean":0.835879,"std":0.005623},{"mean":0.817697,"std":0.004693},{"mean":0.796121,"std":0.006696},{"mean":0.763455,"std":0.002049},{"mean":0.729091,"std":0.003154}],"high_conf_oos_misroute_rate":[{"mean":0.002333,"std":0.001528},{"mean":0.005,"std":0.001},{"mean":0.007667,"std":0.002517},{"mean":0.011333,"std":0.004041},{"mean":0.016667,"std":0.004509},{"mean":0.026333,"std":0.006429},{"mean":0.037,"std":0.010149},{"mean":0.050333,"std":0.014012},{"mean":0.061333,"std":0.017156},{"mean":0.081667,"std":0.023965},{"mean":0.104,"std":0.030414},{"mean":0.130667,"std":0.037005},{"mean":0.17,"std":0.037363},{"mean":0.215,"std":0.035791},{"mean":0.271667,"std":0.032005},{"mean":0.355333,"std":0.033561},{"mean":0.477667,"std":0.01464},{"mean":0.617667,"std":0.031565},{"mean":0.815667,"std":0.01193},{"mean":0.990333,"std":0.010017}],"oos_misroute_rate":[{"mean":0.434333,"std":0.001528},{"mean":0.435667,"std":0.002082},{"mean":0.437,"std":0.002646},{"mean":0.437667,"std":0.003215},{"mean":0.440333,"std":0.003055},{"mean":0.445667,"std":0.004933},{"mean":0.452333,"std":0.008145},{"mean":0.458,"std":0.009539},{"mean":0.463667,"std":0.010408},{"mean":0.471667,"std":0.014572},{"mean":0.484333,"std":0.017954},{"mean":0.5,"std":0.021633},{"mean":0.520333,"std":0.023029},{"mean":0.543,"std":0.024},{"mean":0.573667,"std":0.022189},{"mean":0.617,"std":0.017349},{"mean":0.685333,"std":0.01097},{"mean":0.764,"std":0.022539},{"mean":0.884667,"std":0.006429},{"mean":0.991667,"std":0.009292}],"llm_call_rate":[{"mean":0.961515,"std":0.000688},{"mean":0.917636,"std":0.004524},{"mean":0.876667,"std":0.006716},{"mean":0.83103,"std":0.002414},{"mean":0.789394,"std":0.001892},{"mean":0.744727,"std":0.006771},{"mean":0.699152,"std":0.006941},{"mean":0.652242,"std":0.010163},{"mean":0.611697,"std":0.009768},{"mean":0.568303,"std":0.014224},{"mean":0.521879,"std":0.015403},{"mean":0.474242,"std":0.010343},{"mean":0.428848,"std":0.004433},{"mean":0.380364,"std":0.008797},{"mean":0.327455,"std":0.009954},{"mean":0.26697,"std":0.008644},{"mean":0.202182,"std":0.002993},{"mean":0.141091,"std":0.007098},{"mean":0.066909,"std":0.003623},{"mean":0.00097,"std":0.000584}]}},"msp_t":{"risk_coverage":[{"mean":0.010909,"std":0.009621},{"mean":0.014545,"std":0.005455},{"mean":0.01697,"std":0.00437},{"mean":0.019091,"std":0.004545},{"mean":0.022303,"std":0.004384},{"mean":0.029697,"std":0.007348},{"mean":0.035671,"std":0.007279},{"mean":0.041515,"std":0.007665},{"mean":0.051178,"std":0.012137},{"mean":0.059515,"std":0.010941},{"mean":0.071515,"std":0.010164},{"mean":0.083939,"std":0.011039},{"mean":0.099114,"std":0.010017},{"mean":0.116364,"std":0.009547},{"mean":0.136646,"std":0.008001},{"mean":0.15947,"std":0.006789},{"mean":0.182959,"std":0.005241},{"mean":0.210707,"std":0.00404},{"mean":0.240893,"std":0.003541},{"mean":0.271394,"std":0.003413}],"operating":{"tau":[{"mean":0.999946,"std":1.7e-05},{"mean":0.999769,"std":8.2e-05},{"mean":0.999355,"std":0.000195},{"mean":0.998485,"std":0.000389},{"mean":0.996914,"std":0.000849},{"mean":0.994007,"std":0.001822},{"mean":0.988886,"std":0.003698},{"mean":0.980766,"std":0.007148},{"mean":0.967746,"std":0.010309},{"mean":0.949347,"std":0.017303},{"mean":0.923059,"std":0.020843},{"mean":0.881429,"std":0.023585},{"mean":0.827855,"std":0.025965},{"mean":0.763193,"std":0.027933},{"mean":0.680836,"std":0.030252},{"mean":0.583975,"std":0.022768},{"mean":0.486148,"std":0.01664},{"mean":0.383644,"std":0.010163},{"mean":0.291774,"std":0.002936},{"mean":0.187098,"std":0.007381}],"coverage":[{"mean":0.039455,"std":0.003004},{"mean":0.082364,"std":0.005022},{"mean":0.122727,"std":0.005052},{"mean":0.166485,"std":0.001155},{"mean":0.211636,"std":0.004166},{"mean":0.253212,"std":0.004604},{"mean":0.300485,"std":0.009441},{"mean":0.347212,"std":0.013536},{"mean":0.391697,"std":0.009727},{"mean":0.431273,"std":0.016071},{"mean":0.478606,"std":0.017145},{"mean":0.526,"std":0.009759},{"mean":0.572848,"std":0.009106},{"mean":0.618848,"std":0.007757},{"mean":0.666909,"std":0.007441},{"mean":0.72097,"std":0.002956},{"mean":0.784,"std":0.004556},{"mean":0.852727,"std":0.012479},{"mean":0.925818,"std":0.006916},{"mean":0.99903,"std":0.000897}],"selective_risk":[{"mean":0.011139,"std":0.008141},{"mean":0.012658,"std":0.006146},{"mean":0.015823,"std":0.005143},{"mean":0.017461,"std":0.004695},{"mean":0.019742,"std":0.005092},{"mean":0.022919,"std":0.005028},{"mean":0.030484,"std":0.008429},{"mean":0.035216,"std":0.009193},{"mean":0.039608,"std":0.009871},{"mean":0.047742,"std":0.012894},{"mean":0.055468,"std":0.015787},{"mean":0.065076,"std":0.011405},{"mean":0.077968,"std":0.011919},{"mean":0.088723,"std":0.013189},{"mean":0.104061,"std":0.01198},{"mean":0.124061,"std":0.010168},{"mean":0.152611,"std":0.006276},{"mean":0.184772,"std":0.003599},{"mean":0.226432,"std":0.000285},{"mean":0.270688,"std":0.002788}],"accuracy_8":[{"mean":0.822485,"std":0.000278},{"mean":0.825091,"std":0.001273},{"mean":0.827758,"std":0.001718},{"mean":0.832,"std":0.001106},{"mean":0.835515,"std":0.000915},{"mean":0.838121,"std":0.000735},{"mean":0.840182,"std":0.001443},{"mean":0.843273,"std":0.001753},{"mean":0.845939,"std":0.002482},{"mean":0.84703,"std":0.001927},{"mean":0.848848,"std":0.002939},{"mean":0.84897,"std":0.002956},{"mean":0.847273,"std":0.004194},{"mean":0.846909,"std":0.006027},{"mean":0.843515,"std":0.006577},{"mean":0.835697,"std":0.005785},{"mean":0.820667,"std":0.003679},{"mean":0.800182,"std":0.002025},{"mean":0.768727,"std":0.000364},{"mean":0.729212,"std":0.002842}],"high_conf_oos_misroute_rate":[{"mean":0.002333,"std":0.001528},{"mean":0.004333,"std":0.001528},{"mean":0.008,"std":0.002646},{"mean":0.012333,"std":0.004041},{"mean":0.017333,"std":0.003786},{"mean":0.024667,"std":0.005774},{"mean":0.037333,"std":0.010017},{"mean":0.051333,"std":0.014844},{"mean":0.064333,"std":0.017786},{"mean":0.083667,"std":0.025146},{"mean":0.107,"std":0.036373},{"mean":0.134667,"std":0.032146},{"mean":0.176,"std":0.035791},{"mean":0.214,"std":0.036346},{"mean":0.262333,"std":0.034775},{"mean":0.329667,"std":0.030616},{"mean":0.43,"std":0.011533},{"mean":0.570667,"std":0.024542},{"mean":0.776,"std":0.013892},{"mean":0.989333,"std":0.009609}],"oos_misroute_rate":[{"mean":0.434333,"std":0.001528},{"mean":0.435667,"std":0.002082},{"mean":0.437,"std":0.002646},{"mean":0.438,"std":0.002646},{"mean":0.441333,"std":0.003786},{"mean":0.444667,"std":0.004933},{"mean":0.451667,"std":0.008505},{"mean":0.458,"std":0.009539},{"mean":0.463,"std":0.011269},{"mean":0.472333,"std":0.015044},{"mean":0.484,"std":0.019975},{"mean":0.501,"std":0.017521},{"mean":0.521,"std":0.022},{"mean":0.541,"std":0.023516},{"mean":0.565667,"std":0.023007},{"mean":0.602667,"std":0.019553},{"mean":0.654333,"std":0.015373},{"mean":0.732,"std":0.013},{"mean":0.856,"std":0.004359},{"mean":0.991,"std":0.009849}],"llm_call_rate":[{"mean":0.960545,"std":0.003004},{"mean":0.917636,"std":0.005022},{"mean":0.877273,"std":0.005052},{"mean":0.833515,"std":0.001155},{"mean":0.788364,"std":0.004166},{"mean":0.746788,"std":0.004604},{"mean":0.699515,"std":0.009441},{"mean":0.652788,"std":0.013536},{"mean":0.608303,"std":0.009727},{"mean":0.568727,"std":0.016071},{"mean":0.521394,"std":0.017145},{"mean":0.474,"std":0.009759},{"mean":0.427152,"std":0.009106},{"mean":0.381152,"std":0.007757},{"mean":0.333091,"std":0.007441},{"mean":0.27903,"std":0.002956},{"mean":0.216,"std":0.004556},{"mean":0.147273,"std":0.012479},{"mean":0.074182,"std":0.006916},{"mean":0.00097,"std":0.000897}]}}}}},"tfidf-centroid/k100":{"model":"tfidf-centroid","k":100,"oos_train":null,"kind":"tfidf","seeds":[42,43,44],"run_names":["tfidf-centroid-k100-seed42","tfidf-centroid-k100-seed43","tfidf-centroid-k100-seed44"],"logits_sha256":["0ae4c761890317d76efc8d3b08a3920eab94633e32f710f0059cb7b96f82cb75","01e56ce77413fd0e6dcb11398fdd0eb8024997983daf7bac5fea2a410d1bbe33","ae69c656b6f00892ace610769c997b03dcfe588cb8568b4cd4e94bba9fee14bf"],"result":{"argmax":{"margin":{"risk_coverage":[{"mean":0.003636,"std":0.0},{"mean":0.010909,"std":0.0},{"mean":0.013333,"std":0.0},{"mean":0.014545,"std":0.0},{"mean":0.017455,"std":0.0},{"mean":0.021212,"std":0.0},{"mean":0.023377,"std":0.0},{"mean":0.027273,"std":0.0},{"mean":0.032323,"std":0.0},{"mean":0.040727,"std":0.0},{"mean":0.050579,"std":0.0},{"mean":0.067273,"std":0.0},{"mean":0.07972,"std":0.0},{"mean":0.096364,"std":0.0},{"mean":0.114909,"std":0.0},{"mean":0.135227,"std":0.0},{"mean":0.157219,"std":0.0},{"mean":0.178182,"std":0.0},{"mean":0.200766,"std":0.0},{"mean":0.223818,"std":0.0}],"operating":{"tau":[{"mean":13.730887,"std":0.0},{"mean":11.508179,"std":0.0},{"mean":10.152398,"std":0.0},{"mean":9.146228,"std":0.0},{"mean":8.200481,"std":0.0},{"mean":7.366282,"std":0.0},{"mean":6.575495,"std":0.0},{"mean":5.876117,"std":0.0},{"mean":5.269077,"std":0.0},{"mean":4.671511,"std":0.0},{"mean":3.983152,"std":0.0},{"mean":3.343007,"std":0.0},{"mean":2.789659,"std":0.0},{"mean":2.305379,"std":0.0},{"mean":1.850945,"std":0.0},{"mean":1.384631,"std":0.0},{"mean":0.94564,"std":0.0},{"mean":0.547687,"std":0.0},{"mean":0.280672,"std":0.0},{"mean":0.0,"std":0.0}],"coverage":[{"mean":0.041273,"std":0.0},{"mean":0.080545,"std":0.0},{"mean":0.128182,"std":0.0},{"mean":0.172364,"std":0.0},{"mean":0.218,"std":0.0},{"mean":0.268364,"std":0.0},{"mean":0.310909,"std":0.0},{"mean":0.352545,"std":0.0},{"mean":0.396182,"std":0.0},{"mean":0.433273,"std":0.0},{"mean":0.485636,"std":0.0},{"mean":0.538727,"std":0.0},{"mean":0.584545,"std":0.0},{"mean":0.626545,"std":0.0},{"mean":0.675273,"std":0.0},{"mean":0.738364,"std":0.0},{"mean":0.798364,"std":0.0},{"mean":0.870182,"std":0.0},{"mean":0.929818,"std":0.0},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.004405,"std":0.0},{"mean":0.006772,"std":0.0},{"mean":0.015603,"std":0.0},{"mean":0.015823,"std":0.0},{"mean":0.015847,"std":0.0},{"mean":0.019648,"std":0.0},{"mean":0.021637,"std":0.0},{"mean":0.024755,"std":0.0},{"mean":0.027077,"std":0.0},{"mean":0.031473,"std":0.0},{"mean":0.037439,"std":0.0},{"mean":0.048937,"std":0.0},{"mean":0.060342,"std":0.0},{"mean":0.074289,"std":0.0},{"mean":0.086968,"std":0.0},{"mean":0.110318,"std":0.0},{"mean":0.134366,"std":0.0},{"mean":0.165274,"std":0.0},{"mean":0.193195,"std":0.0},{"mean":0.223818,"std":0.0}],"accuracy_8":[{"mean":0.822364,"std":0.0},{"mean":0.825636,"std":0.0},{"mean":0.829091,"std":0.0},{"mean":0.832,"std":0.0},{"mean":0.836364,"std":0.0},{"mean":0.841455,"std":0.0},{"mean":0.846,"std":0.0},{"mean":0.849273,"std":0.0},{"mean":0.853818,"std":0.0},{"mean":0.856909,"std":0.0},{"mean":0.862182,"std":0.0},{"mean":0.862727,"std":0.0},{"mean":0.861636,"std":0.0},{"mean":0.859818,"std":0.0},{"mean":0.855636,"std":0.0},{"mean":0.848545,"std":0.0},{"mean":0.837455,"std":0.0},{"mean":0.818182,"std":0.0},{"mean":0.800182,"std":0.0},{"mean":0.776182,"std":0.0}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.0},{"mean":0.003,"std":0.0},{"mean":0.01,"std":0.0},{"mean":0.011,"std":0.0},{"mean":0.015,"std":0.0},{"mean":0.022,"std":0.0},{"mean":0.028,"std":0.0},{"mean":0.037,"std":0.0},{"mean":0.045,"std":0.0},{"mean":0.057,"std":0.0},{"mean":0.078,"std":0.0},{"mean":0.113,"std":0.0},{"mean":0.153,"std":0.0},{"mean":0.203,"std":0.0},{"mean":0.249,"std":0.0},{"mean":0.346,"std":0.0},{"mean":0.438,"std":0.0},{"mean":0.579,"std":0.0},{"mean":0.717,"std":0.0},{"mean":0.904,"std":0.0}],"oos_misroute_rate":[{"mean":0.433,"std":0.0},{"mean":0.435,"std":0.0},{"mean":0.435,"std":0.0},{"mean":0.436,"std":0.0},{"mean":0.438,"std":0.0},{"mean":0.443,"std":0.0},{"mean":0.447,"std":0.0},{"mean":0.453,"std":0.0},{"mean":0.455,"std":0.0},{"mean":0.463,"std":0.0},{"mean":0.474,"std":0.0},{"mean":0.496,"std":0.0},{"mean":0.519,"std":0.0},{"mean":0.545,"std":0.0},{"mean":0.57,"std":0.0},{"mean":0.616,"std":0.0},{"mean":0.665,"std":0.0},{"mean":0.736,"std":0.0},{"mean":0.808,"std":0.0},{"mean":0.904,"std":0.0}],"llm_call_rate":[{"mean":0.958727,"std":0.0},{"mean":0.919455,"std":0.0},{"mean":0.871818,"std":0.0},{"mean":0.827636,"std":0.0},{"mean":0.782,"std":0.0},{"mean":0.731636,"std":0.0},{"mean":0.689091,"std":0.0},{"mean":0.647455,"std":0.0},{"mean":0.603818,"std":0.0},{"mean":0.566727,"std":0.0},{"mean":0.514364,"std":0.0},{"mean":0.461273,"std":0.0},{"mean":0.415455,"std":0.0},{"mean":0.373455,"std":0.0},{"mean":0.324727,"std":0.0},{"mean":0.261636,"std":0.0},{"mean":0.201636,"std":0.0},{"mean":0.129818,"std":0.0},{"mean":0.070182,"std":0.0},{"mean":0.0,"std":0.0}]}},"msp_t":{"risk_coverage":[{"mean":0.003636,"std":0.0},{"mean":0.010909,"std":0.0},{"mean":0.015758,"std":0.0},{"mean":0.017273,"std":0.0},{"mean":0.018182,"std":0.0},{"mean":0.022424,"std":0.0},{"mean":0.023896,"std":0.0},{"mean":0.025,"std":0.0},{"mean":0.029495,"std":0.0},{"mean":0.036,"std":0.0},{"mean":0.046612,"std":0.0},{"mean":0.056364,"std":0.0},{"mean":0.068252,"std":0.0},{"mean":0.082857,"std":0.0},{"mean":0.099636,"std":0.0},{"mean":0.120455,"std":0.0},{"mean":0.143957,"std":0.0},{"mean":0.169495,"std":0.0},{"mean":0.196364,"std":0.0},{"mean":0.223818,"std":0.0}],"operating":{"tau":[{"mean":0.999994,"std":0.0},{"mean":0.999962,"std":0.0},{"mean":0.999847,"std":0.0},{"mean":0.999617,"std":0.0},{"mean":0.99902,"std":0.0},{"mean":0.997856,"std":0.0},{"mean":0.995417,"std":0.0},{"mean":0.991337,"std":0.0},{"mean":0.983331,"std":0.0},{"mean":0.969636,"std":0.0},{"mean":0.940576,"std":0.0},{"mean":0.904583,"std":0.0},{"mean":0.855804,"std":0.0},{"mean":0.77792,"std":0.0},{"mean":0.68442,"std":0.0},{"mean":0.578255,"std":0.0},{"mean":0.471669,"std":0.0},{"mean":0.331421,"std":0.0},{"mean":0.162595,"std":0.0},{"mean":0.006623,"std":0.0}],"coverage":[{"mean":0.044545,"std":0.0},{"mean":0.080182,"std":0.0},{"mean":0.129273,"std":0.0},{"mean":0.166545,"std":0.0},{"mean":0.213273,"std":0.0},{"mean":0.255273,"std":0.0},{"mean":0.306,"std":0.0},{"mean":0.341636,"std":0.0},{"mean":0.388182,"std":0.0},{"mean":0.438727,"std":0.0},{"mean":0.494364,"std":0.0},{"mean":0.537273,"std":0.0},{"mean":0.579091,"std":0.0},{"mean":0.630909,"std":0.0},{"mean":0.675273,"std":0.0},{"mean":0.725273,"std":0.0},{"mean":0.777636,"std":0.0},{"mean":0.834545,"std":0.0},{"mean":0.916364,"std":0.0},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.004082,"std":0.0},{"mean":0.004535,"std":0.0},{"mean":0.015471,"std":0.0},{"mean":0.015284,"std":0.0},{"mean":0.01705,"std":0.0},{"mean":0.018519,"std":0.0},{"mean":0.022579,"std":0.0},{"mean":0.024481,"std":0.0},{"mean":0.025293,"std":0.0},{"mean":0.028181,"std":0.0},{"mean":0.034939,"std":0.0},{"mean":0.043655,"std":0.0},{"mean":0.052119,"std":0.0},{"mean":0.064553,"std":0.0},{"mean":0.074044,"std":0.0},{"mean":0.091502,"std":0.0},{"mean":0.110825,"std":0.0},{"mean":0.135948,"std":0.0},{"mean":0.178373,"std":0.0},{"mean":0.223818,"std":0.0}],"accuracy_8":[{"mean":0.823091,"std":0.0},{"mean":0.826364,"std":0.0},{"mean":0.829455,"std":0.0},{"mean":0.830727,"std":0.0},{"mean":0.834,"std":0.0},{"mean":0.838364,"std":0.0},{"mean":0.844727,"std":0.0},{"mean":0.848182,"std":0.0},{"mean":0.853818,"std":0.0},{"mean":0.859091,"std":0.0},{"mean":0.861455,"std":0.0},{"mean":0.862364,"std":0.0},{"mean":0.862909,"std":0.0},{"mean":0.860545,"std":0.0},{"mean":0.86,"std":0.0},{"mean":0.854182,"std":0.0},{"mean":0.846545,"std":0.0},{"mean":0.834182,"std":0.0},{"mean":0.808182,"std":0.0},{"mean":0.776182,"std":0.0}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.0},{"mean":0.002,"std":0.0},{"mean":0.01,"std":0.0},{"mean":0.01,"std":0.0},{"mean":0.014,"std":0.0},{"mean":0.018,"std":0.0},{"mean":0.028,"std":0.0},{"mean":0.033,"std":0.0},{"mean":0.039,"std":0.0},{"mean":0.049,"std":0.0},{"mean":0.066,"std":0.0},{"mean":0.095,"std":0.0},{"mean":0.128,"std":0.0},{"mean":0.172,"std":0.0},{"mean":0.206,"std":0.0},{"mean":0.264,"std":0.0},{"mean":0.334,"std":0.0},{"mean":0.444,"std":0.0},{"mean":0.645,"std":0.0},{"mean":0.904,"std":0.0}],"oos_misroute_rate":[{"mean":0.433,"std":0.0},{"mean":0.433,"std":0.0},{"mean":0.435,"std":0.0},{"mean":0.435,"std":0.0},{"mean":0.437,"std":0.0},{"mean":0.439,"std":0.0},{"mean":0.448,"std":0.0},{"mean":0.451,"std":0.0},{"mean":0.454,"std":0.0},{"mean":0.458,"std":0.0},{"mean":0.472,"std":0.0},{"mean":0.485,"std":0.0},{"mean":0.501,"std":0.0},{"mean":0.526,"std":0.0},{"mean":0.542,"std":0.0},{"mean":0.576,"std":0.0},{"mean":0.611,"std":0.0},{"mean":0.666,"std":0.0},{"mean":0.768,"std":0.0},{"mean":0.904,"std":0.0}],"llm_call_rate":[{"mean":0.955455,"std":0.0},{"mean":0.919818,"std":0.0},{"mean":0.870727,"std":0.0},{"mean":0.833455,"std":0.0},{"mean":0.786727,"std":0.0},{"mean":0.744727,"std":0.0},{"mean":0.694,"std":0.0},{"mean":0.658364,"std":0.0},{"mean":0.611818,"std":0.0},{"mean":0.561273,"std":0.0},{"mean":0.505636,"std":0.0},{"mean":0.462727,"std":0.0},{"mean":0.420909,"std":0.0},{"mean":0.369091,"std":0.0},{"mean":0.324727,"std":0.0},{"mean":0.274727,"std":0.0},{"mean":0.222364,"std":0.0},{"mean":0.165455,"std":0.0},{"mean":0.083636,"std":0.0},{"mean":0.0,"std":0.0}]}}},"summed":{"margin":{"risk_coverage":[{"mean":0.003636,"std":0.0},{"mean":0.001818,"std":0.0},{"mean":0.006061,"std":0.0},{"mean":0.010909,"std":0.0},{"mean":0.012364,"std":0.0},{"mean":0.013939,"std":0.0},{"mean":0.018182,"std":0.0},{"mean":0.021364,"std":0.0},{"mean":0.023434,"std":0.0},{"mean":0.027636,"std":0.0},{"mean":0.031736,"std":0.0},{"mean":0.040303,"std":0.0},{"mean":0.056224,"std":0.0},{"mean":0.070649,"std":0.0},{"mean":0.087515,"std":0.0},{"mean":0.116591,"std":0.0},{"mean":0.146524,"std":0.0},{"mean":0.175354,"std":0.0},{"mean":0.208038,"std":0.0},{"mean":0.237273,"std":0.0}],"operating":{"tau":[{"mean":14.730121,"std":0.0},{"mean":12.65452,"std":0.0},{"mean":11.40673,"std":0.0},{"mean":10.3057,"std":0.0},{"mean":9.407725,"std":0.0},{"mean":8.678823,"std":0.0},{"mean":7.934788,"std":0.0},{"mean":7.289506,"std":0.0},{"mean":6.572843,"std":0.0},{"mean":5.926839,"std":0.0},{"mean":5.267307,"std":0.0},{"mean":4.654167,"std":0.0},{"mean":4.011025,"std":0.0},{"mean":3.343018,"std":0.0},{"mean":2.68217,"std":0.0},{"mean":2.020996,"std":0.0},{"mean":1.445009,"std":0.0},{"mean":0.847622,"std":0.0},{"mean":0.332395,"std":0.0},{"mean":0.000743,"std":0.0}],"coverage":[{"mean":0.042545,"std":0.0},{"mean":0.088909,"std":0.0},{"mean":0.125091,"std":0.0},{"mean":0.168545,"std":0.0},{"mean":0.212727,"std":0.0},{"mean":0.252,"std":0.0},{"mean":0.297091,"std":0.0},{"mean":0.341455,"std":0.0},{"mean":0.390545,"std":0.0},{"mean":0.433273,"std":0.0},{"mean":0.484364,"std":0.0},{"mean":0.531455,"std":0.0},{"mean":0.577091,"std":0.0},{"mean":0.626364,"std":0.0},{"mean":0.673273,"std":0.0},{"mean":0.725818,"std":0.0},{"mean":0.780909,"std":0.0},{"mean":0.841455,"std":0.0},{"mean":0.914364,"std":0.0},{"mean":0.999636,"std":0.0}],"selective_risk":[{"mean":0.004274,"std":0.0},{"mean":0.002045,"std":0.0},{"mean":0.005814,"std":0.0},{"mean":0.009709,"std":0.0},{"mean":0.011966,"std":0.0},{"mean":0.012266,"std":0.0},{"mean":0.013464,"std":0.0},{"mean":0.017572,"std":0.0},{"mean":0.021881,"std":0.0},{"mean":0.021402,"std":0.0},{"mean":0.026276,"std":0.0},{"mean":0.030448,"std":0.0},{"mean":0.036547,"std":0.0},{"mean":0.047896,"std":0.0},{"mean":0.064002,"std":0.0},{"mean":0.079409,"std":0.0},{"mean":0.103842,"std":0.0},{"mean":0.141746,"std":0.0},{"mean":0.184331,"std":0.0},{"mean":0.237177,"std":0.0}],"accuracy_8":[{"mean":0.822909,"std":0.0},{"mean":0.825091,"std":0.0},{"mean":0.828,"std":0.0},{"mean":0.830364,"std":0.0},{"mean":0.834,"std":0.0},{"mean":0.837636,"std":0.0},{"mean":0.844182,"std":0.0},{"mean":0.847818,"std":0.0},{"mean":0.851818,"std":0.0},{"mean":0.855818,"std":0.0},{"mean":0.860182,"std":0.0},{"mean":0.862727,"std":0.0},{"mean":0.866182,"std":0.0},{"mean":0.866,"std":0.0},{"mean":0.863818,"std":0.0},{"mean":0.86,"std":0.0},{"mean":0.851636,"std":0.0},{"mean":0.833636,"std":0.0},{"mean":0.805818,"std":0.0},{"mean":0.762909,"std":0.0}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.0},{"mean":0.001,"std":0.0},{"mean":0.003,"std":0.0},{"mean":0.008,"std":0.0},{"mean":0.012,"std":0.0},{"mean":0.013,"std":0.0},{"mean":0.015,"std":0.0},{"mean":0.025,"std":0.0},{"mean":0.037,"std":0.0},{"mean":0.04,"std":0.0},{"mean":0.055,"std":0.0},{"mean":0.069,"std":0.0},{"mean":0.092,"std":0.0},{"mean":0.13,"std":0.0},{"mean":0.192,"std":0.0},{"mean":0.256,"std":0.0},{"mean":0.353,"std":0.0},{"mean":0.512,"std":0.0},{"mean":0.704,"std":0.0},{"mean":0.975,"std":0.0}],"oos_misroute_rate":[{"mean":0.433,"std":0.0},{"mean":0.433,"std":0.0},{"mean":0.434,"std":0.0},{"mean":0.435,"std":0.0},{"mean":0.435,"std":0.0},{"mean":0.436,"std":0.0},{"mean":0.437,"std":0.0},{"mean":0.443,"std":0.0},{"mean":0.45,"std":0.0},{"mean":0.453,"std":0.0},{"mean":0.46,"std":0.0},{"mean":0.468,"std":0.0},{"mean":0.477,"std":0.0},{"mean":0.497,"std":0.0},{"mean":0.527,"std":0.0},{"mean":0.567,"std":0.0},{"mean":0.619,"std":0.0},{"mean":0.7,"std":0.0},{"mean":0.813,"std":0.0},{"mean":0.975,"std":0.0}],"llm_call_rate":[{"mean":0.957455,"std":0.0},{"mean":0.911091,"std":0.0},{"mean":0.874909,"std":0.0},{"mean":0.831455,"std":0.0},{"mean":0.787273,"std":0.0},{"mean":0.748,"std":0.0},{"mean":0.702909,"std":0.0},{"mean":0.658545,"std":0.0},{"mean":0.609455,"std":0.0},{"mean":0.566727,"std":0.0},{"mean":0.515636,"std":0.0},{"mean":0.468545,"std":0.0},{"mean":0.422909,"std":0.0},{"mean":0.373636,"std":0.0},{"mean":0.326727,"std":0.0},{"mean":0.274182,"std":0.0},{"mean":0.219091,"std":0.0},{"mean":0.158545,"std":0.0},{"mean":0.085636,"std":0.0},{"mean":0.000364,"std":0.0}]}},"msp_t":{"risk_coverage":[{"mean":0.003636,"std":0.0},{"mean":0.001818,"std":0.0},{"mean":0.007273,"std":0.0},{"mean":0.012727,"std":0.0},{"mean":0.013818,"std":0.0},{"mean":0.014545,"std":0.0},{"mean":0.019221,"std":0.0},{"mean":0.022727,"std":0.0},{"mean":0.023838,"std":0.0},{"mean":0.028727,"std":0.0},{"mean":0.033719,"std":0.0},{"mean":0.042424,"std":0.0},{"mean":0.056224,"std":0.0},{"mean":0.072727,"std":0.0},{"mean":0.090182,"std":0.0},{"mean":0.115682,"std":0.0},{"mean":0.144385,"std":0.0},{"mean":0.171515,"std":0.0},{"mean":0.204211,"std":0.0},{"mean":0.237273,"std":0.0}],"operating":{"tau":[{"mean":0.999999,"std":0.0},{"mean":0.999992,"std":0.0},{"mean":0.999972,"std":0.0},{"mean":0.999916,"std":0.0},{"mean":0.999812,"std":0.0},{"mean":0.999609,"std":0.0},{"mean":0.999195,"std":0.0},{"mean":0.998354,"std":0.0},{"mean":0.996686,"std":0.0},{"mean":0.993701,"std":0.0},{"mean":0.987774,"std":0.0},{"mean":0.977124,"std":0.0},{"mean":0.957524,"std":0.0},{"mean":0.919159,"std":0.0},{"mean":0.864697,"std":0.0},{"mean":0.778439,"std":0.0},{"mean":0.652089,"std":0.0},{"mean":0.496427,"std":0.0},{"mean":0.343143,"std":0.0},{"mean":0.197067,"std":0.0}],"coverage":[{"mean":0.044545,"std":0.0},{"mean":0.088727,"std":0.0},{"mean":0.127273,"std":0.0},{"mean":0.174,"std":0.0},{"mean":0.210727,"std":0.0},{"mean":0.249636,"std":0.0},{"mean":0.293455,"std":0.0},{"mean":0.335636,"std":0.0},{"mean":0.391818,"std":0.0},{"mean":0.435091,"std":0.0},{"mean":0.484909,"std":0.0},{"mean":0.532727,"std":0.0},{"mean":0.581636,"std":0.0},{"mean":0.629636,"std":0.0},{"mean":0.67,"std":0.0},{"mean":0.718727,"std":0.0},{"mean":0.774182,"std":0.0},{"mean":0.843273,"std":0.0},{"mean":0.909455,"std":0.0},{"mean":0.998,"std":0.0}],"selective_risk":[{"mean":0.004082,"std":0.0},{"mean":0.002049,"std":0.0},{"mean":0.004286,"std":0.0},{"mean":0.009404,"std":0.0},{"mean":0.012942,"std":0.0},{"mean":0.013838,"std":0.0},{"mean":0.01425,"std":0.0},{"mean":0.017876,"std":0.0},{"mean":0.022274,"std":0.0},{"mean":0.023402,"std":0.0},{"mean":0.026997,"std":0.0},{"mean":0.030717,"std":0.0},{"mean":0.037824,"std":0.0},{"mean":0.049379,"std":0.0},{"mean":0.062687,"std":0.0},{"mean":0.078674,"std":0.0},{"mean":0.101926,"std":0.0},{"mean":0.141009,"std":0.0},{"mean":0.177329,"std":0.0},{"mean":0.235744,"std":0.0}],"accuracy_8":[{"mean":0.823091,"std":0.0},{"mean":0.826545,"std":0.0},{"mean":0.828909,"std":0.0},{"mean":0.832182,"std":0.0},{"mean":0.835091,"std":0.0},{"mean":0.839273,"std":0.0},{"mean":0.843091,"std":0.0},{"mean":0.846545,"std":0.0},{"mean":0.852364,"std":0.0},{"mean":0.856727,"std":0.0},{"mean":0.860364,"std":0.0},{"mean":0.864,"std":0.0},{"mean":0.866,"std":0.0},{"mean":0.864727,"std":0.0},{"mean":0.863455,"std":0.0},{"mean":0.859455,"std":0.0},{"mean":0.851818,"std":0.0},{"mean":0.832182,"std":0.0},{"mean":0.812,"std":0.0},{"mean":0.764182,"std":0.0}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.0},{"mean":0.001,"std":0.0},{"mean":0.002,"std":0.0},{"mean":0.008,"std":0.0},{"mean":0.012,"std":0.0},{"mean":0.013,"std":0.0},{"mean":0.016,"std":0.0},{"mean":0.025,"std":0.0},{"mean":0.037,"std":0.0},{"mean":0.042,"std":0.0},{"mean":0.053,"std":0.0},{"mean":0.067,"std":0.0},{"mean":0.086,"std":0.0},{"mean":0.132,"std":0.0},{"mean":0.184,"std":0.0},{"mean":0.243,"std":0.0},{"mean":0.327,"std":0.0},{"mean":0.48,"std":0.0},{"mean":0.649,"std":0.0},{"mean":0.966,"std":0.0}],"oos_misroute_rate":[{"mean":0.433,"std":0.0},{"mean":0.433,"std":0.0},{"mean":0.433,"std":0.0},{"mean":0.434,"std":0.0},{"mean":0.435,"std":0.0},{"mean":0.435,"std":0.0},{"mean":0.437,"std":0.0},{"mean":0.443,"std":0.0},{"mean":0.45,"std":0.0},{"mean":0.454,"std":0.0},{"mean":0.459,"std":0.0},{"mean":0.466,"std":0.0},{"mean":0.476,"std":0.0},{"mean":0.501,"std":0.0},{"mean":0.521,"std":0.0},{"mean":0.553,"std":0.0},{"mean":0.599,"std":0.0},{"mean":0.682,"std":0.0},{"mean":0.765,"std":0.0},{"mean":0.969,"std":0.0}],"llm_call_rate":[{"mean":0.955455,"std":0.0},{"mean":0.911273,"std":0.0},{"mean":0.872727,"std":0.0},{"mean":0.826,"std":0.0},{"mean":0.789273,"std":0.0},{"mean":0.750364,"std":0.0},{"mean":0.706545,"std":0.0},{"mean":0.664364,"std":0.0},{"mean":0.608182,"std":0.0},{"mean":0.564909,"std":0.0},{"mean":0.515091,"std":0.0},{"mean":0.467273,"std":0.0},{"mean":0.418364,"std":0.0},{"mean":0.370364,"std":0.0},{"mean":0.33,"std":0.0},{"mean":0.281273,"std":0.0},{"mean":0.225818,"std":0.0},{"mean":0.156727,"std":0.0},{"mean":0.090545,"std":0.0},{"mean":0.002,"std":0.0}]}}}}},"tfidf-centroid/k25":{"model":"tfidf-centroid","k":25,"oos_train":null,"kind":"tfidf","seeds":[42,43,44],"run_names":["tfidf-centroid-k25-seed42","tfidf-centroid-k25-seed43","tfidf-centroid-k25-seed44"],"logits_sha256":["be4fdf7bd9f789ab99de9042ffbc8b0d41d6cb152e08a7c079fd3bf623909a47","c09e6879bf292df29c7fa24208687bdc43a2513f62033313cc100d5d6a1f0fa4","e7aa6e711a2a6b5322a78def95a33738c6bf0f80f6006710d203af2c9c3fdcd5"],"result":{"argmax":{"margin":{"risk_coverage":[{"mean":0.002424,"std":0.004199},{"mean":0.010303,"std":0.006385},{"mean":0.014141,"std":0.0014},{"mean":0.016364,"std":0.000909},{"mean":0.021333,"std":0.003588},{"mean":0.025859,"std":0.005154},{"mean":0.029957,"std":0.003829},{"mean":0.035152,"std":0.002663},{"mean":0.042963,"std":0.001299},{"mean":0.05503,"std":0.002003},{"mean":0.066446,"std":0.004663},{"mean":0.081212,"std":0.004009},{"mean":0.097249,"std":0.00439},{"mean":0.116364,"std":0.005553},{"mean":0.137778,"std":0.007294},{"mean":0.159167,"std":0.006385},{"mean":0.181319,"std":0.006615},{"mean":0.202222,"std":0.0084},{"mean":0.22303,"std":0.008683},{"mean":0.244182,"std":0.008458}],"operating":{"tau":[{"mean":12.251444,"std":0.152096},{"mean":10.178382,"std":0.302331},{"mean":8.953643,"std":0.30229},{"mean":8.002161,"std":0.205625},{"mean":7.180714,"std":0.142296},{"mean":6.496394,"std":0.140329},{"mean":5.813307,"std":0.036609},{"mean":5.171067,"std":0.039969},{"mean":4.559614,"std":0.049337},{"mean":3.958013,"std":0.008746},{"mean":3.378718,"std":0.037391},{"mean":2.866394,"std":0.03025},{"mean":2.367453,"std":0.044116},{"mean":1.902152,"std":0.02823},{"mean":1.490755,"std":0.054027},{"mean":1.090667,"std":0.045688},{"mean":0.762684,"std":0.041912},{"mean":0.450206,"std":0.030046},{"mean":0.202095,"std":0.034691},{"mean":0.0,"std":0.0}],"coverage":[{"mean":0.040606,"std":0.005575},{"mean":0.084364,"std":0.005961},{"mean":0.128909,"std":0.007805},{"mean":0.172848,"std":0.01076},{"mean":0.216848,"std":0.008311},{"mean":0.256121,"std":0.007863},{"mean":0.299636,"std":0.004367},{"mean":0.343576,"std":0.001377},{"mean":0.390364,"std":0.002321},{"mean":0.442242,"std":0.001238},{"mean":0.491697,"std":0.004055},{"mean":0.536667,"std":0.003789},{"mean":0.584424,"std":0.002068},{"mean":0.636061,"std":0.005339},{"mean":0.688485,"std":0.010674},{"mean":0.747091,"std":0.012141},{"mean":0.807152,"std":0.012668},{"mean":0.874667,"std":0.011229},{"mean":0.937879,"std":0.007719},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.002861,"std":0.004956},{"mean":0.006758,"std":0.006359},{"mean":0.010484,"std":0.003496},{"mean":0.015414,"std":0.001732},{"mean":0.017842,"std":0.001778},{"mean":0.021366,"std":0.003737},{"mean":0.026131,"std":0.005172},{"mean":0.029465,"std":0.002618},{"mean":0.033212,"std":0.003776},{"mean":0.041522,"std":0.001385},{"mean":0.052365,"std":0.003707},{"mean":0.063119,"std":0.004274},{"mean":0.075901,"std":0.004342},{"mean":0.091924,"std":0.005301},{"mean":0.112322,"std":0.008879},{"mean":0.136728,"std":0.011933},{"mean":0.161983,"std":0.012339},{"mean":0.19136,"std":0.011132},{"mean":0.218016,"std":0.009391},{"mean":0.244182,"std":0.008458}],"accuracy_8":[{"mean":0.822182,"std":0.001273},{"mean":0.826606,"std":0.001892},{"mean":0.831394,"std":0.002068},{"mean":0.835152,"std":0.001389},{"mean":0.838545,"std":0.001012},{"mean":0.841697,"std":0.001994},{"mean":0.845879,"std":0.001766},{"mean":0.84897,"std":0.002068},{"mean":0.852,"std":0.004252},{"mean":0.856,"std":0.004012},{"mean":0.857212,"std":0.003532},{"mean":0.857152,"std":0.004572},{"mean":0.854182,"std":0.004524},{"mean":0.849152,"std":0.004829},{"mean":0.841758,"std":0.005092},{"mean":0.831758,"std":0.005808},{"mean":0.816667,"std":0.006818},{"mean":0.798364,"std":0.007202},{"mean":0.778545,"std":0.007975},{"mean":0.755818,"std":0.008458}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.002,"std":0.002646},{"mean":0.005667,"std":0.002082},{"mean":0.012333,"std":0.002082},{"mean":0.017667,"std":0.000577},{"mean":0.023667,"std":0.006429},{"mean":0.033333,"std":0.009452},{"mean":0.042333,"std":0.004933},{"mean":0.054333,"std":0.008737},{"mean":0.076333,"std":0.002887},{"mean":0.109,"std":0.009539},{"mean":0.141667,"std":0.012342},{"mean":0.183,"std":0.017776},{"mean":0.238333,"std":0.021548},{"mean":0.314667,"std":0.035233},{"mean":0.410333,"std":0.042525},{"mean":0.52,"std":0.054111},{"mean":0.663333,"std":0.047173},{"mean":0.806667,"std":0.038889},{"mean":0.951667,"std":0.035019}],"oos_misroute_rate":[{"mean":0.432333,"std":0.000577},{"mean":0.433,"std":0.001},{"mean":0.435667,"std":0.001528},{"mean":0.439333,"std":0.000577},{"mean":0.441667,"std":0.001155},{"mean":0.444667,"std":0.005508},{"mean":0.449333,"std":0.005859},{"mean":0.454,"std":0.003464},{"mean":0.461,"std":0.007211},{"mean":0.474,"std":0.003464},{"mean":0.493,"std":0.01044},{"mean":0.512,"std":0.016},{"mean":0.538333,"std":0.01701},{"mean":0.571333,"std":0.02203},{"mean":0.611667,"std":0.029366},{"mean":0.661333,"std":0.027429},{"mean":0.723667,"std":0.033126},{"mean":0.799,"std":0.028618},{"mean":0.875333,"std":0.028024},{"mean":0.951667,"std":0.035019}],"llm_call_rate":[{"mean":0.959394,"std":0.005575},{"mean":0.915636,"std":0.005961},{"mean":0.871091,"std":0.007805},{"mean":0.827152,"std":0.01076},{"mean":0.783152,"std":0.008311},{"mean":0.743879,"std":0.007863},{"mean":0.700364,"std":0.004367},{"mean":0.656424,"std":0.001377},{"mean":0.609636,"std":0.002321},{"mean":0.557758,"std":0.001238},{"mean":0.508303,"std":0.004055},{"mean":0.463333,"std":0.003789},{"mean":0.415576,"std":0.002068},{"mean":0.363939,"std":0.005339},{"mean":0.311515,"std":0.010674},{"mean":0.252909,"std":0.012141},{"mean":0.192848,"std":0.012668},{"mean":0.125333,"std":0.011229},{"mean":0.062121,"std":0.007719},{"mean":0.0,"std":0.0}]}},"msp_t":{"risk_coverage":[{"mean":0.002424,"std":0.004199},{"mean":0.010303,"std":0.005555},{"mean":0.013333,"std":0.005284},{"mean":0.016061,"std":0.00105},{"mean":0.020606,"std":0.001514},{"mean":0.025051,"std":0.003943},{"mean":0.029784,"std":0.001081},{"mean":0.033485,"std":0.001461},{"mean":0.039461,"std":0.002297},{"mean":0.04703,"std":0.002962},{"mean":0.055207,"std":0.002863},{"mean":0.06798,"std":0.003908},{"mean":0.081399,"std":0.002839},{"mean":0.097576,"std":0.002777},{"mean":0.11604,"std":0.005583},{"mean":0.139318,"std":0.006187},{"mean":0.165561,"std":0.006378},{"mean":0.189966,"std":0.007288},{"mean":0.215183,"std":0.007465},{"mean":0.244182,"std":0.008458}],"operating":{"tau":[{"mean":0.999975,"std":2e-06},{"mean":0.999816,"std":5.8e-05},{"mean":0.999412,"std":0.000195},{"mean":0.998493,"std":0.000369},{"mean":0.996878,"std":0.000517},{"mean":0.993895,"std":0.000717},{"mean":0.988497,"std":0.000769},{"mean":0.979873,"std":0.000266},{"mean":0.964701,"std":0.002279},{"mean":0.940497,"std":0.001682},{"mean":0.905818,"std":0.00156},{"mean":0.853208,"std":0.008136},{"mean":0.77825,"std":0.004285},{"mean":0.691706,"std":0.00443},{"mean":0.593573,"std":0.011021},{"mean":0.503206,"std":0.007917},{"mean":0.390503,"std":0.012514},{"mean":0.266881,"std":0.015915},{"mean":0.148324,"std":0.010556},{"mean":0.006623,"std":0.0}],"coverage":[{"mean":0.041333,"std":0.004187},{"mean":0.08697,"std":0.002637},{"mean":0.129212,"std":0.009802},{"mean":0.172303,"std":0.011259},{"mean":0.213212,"std":0.008321},{"mean":0.253818,"std":0.007119},{"mean":0.296303,"std":0.005634},{"mean":0.336909,"std":0.003107},{"mean":0.384545,"std":0.00619},{"mean":0.434303,"std":0.00516},{"mean":0.478788,"std":0.003422},{"mean":0.52697,"std":0.010453},{"mean":0.576667,"std":0.001377},{"mean":0.628788,"std":0.003203},{"mean":0.680606,"std":0.007697},{"mean":0.728606,"std":0.00258},{"mean":0.783515,"std":0.002068},{"mean":0.84697,"std":0.002169},{"mean":0.915455,"std":0.001273},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.001475,"std":0.002555},{"mean":0.007734,"std":0.003404},{"mean":0.01139,"std":0.004839},{"mean":0.015967,"std":0.003179},{"mean":0.01736,"std":0.001493},{"mean":0.020286,"std":0.001309},{"mean":0.02436,"std":0.002794},{"mean":0.028243,"std":0.001116},{"mean":0.032617,"std":0.001598},{"mean":0.036991,"std":0.001967},{"mean":0.043796,"std":0.00238},{"mean":0.050905,"std":0.004245},{"mean":0.062009,"std":0.004137},{"mean":0.075767,"std":0.002544},{"mean":0.090354,"std":0.0046},{"mean":0.106623,"std":0.006477},{"mean":0.131257,"std":0.006054},{"mean":0.163495,"std":0.007471},{"mean":0.197952,"std":0.007643},{"mean":0.244182,"std":0.008458}],"accuracy_8":[{"mean":0.822667,"std":0.001389},{"mean":0.826364,"std":0.001373},{"mean":0.831576,"std":0.002003},{"mean":0.834545,"std":0.001734},{"mean":0.837758,"std":0.001065},{"mean":0.841333,"std":0.001238},{"mean":0.843879,"std":0.002309},{"mean":0.847152,"std":0.003098},{"mean":0.851576,"std":0.003824},{"mean":0.855091,"std":0.00381},{"mean":0.856485,"std":0.004455},{"mean":0.857697,"std":0.003109},{"mean":0.857636,"std":0.003507},{"mean":0.856,"std":0.001818},{"mean":0.851515,"std":0.003109},{"mean":0.845394,"std":0.004777},{"mean":0.834182,"std":0.004502},{"mean":0.814,"std":0.004919},{"mean":0.792788,"std":0.006086},{"mean":0.755818,"std":0.008458}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.002667,"std":0.001155},{"mean":0.005667,"std":0.004041},{"mean":0.012,"std":0.003606},{"mean":0.015667,"std":0.002517},{"mean":0.02,"std":0.002},{"mean":0.028333,"std":0.004933},{"mean":0.036667,"std":0.003786},{"mean":0.048333,"std":0.005774},{"mean":0.063,"std":0.002646},{"mean":0.085333,"std":0.004726},{"mean":0.108,"std":0.007211},{"mean":0.141333,"std":0.01159},{"mean":0.186333,"std":0.009018},{"mean":0.241333,"std":0.017673},{"mean":0.298,"std":0.023065},{"mean":0.386667,"std":0.020793},{"mean":0.524,"std":0.031575},{"mean":0.693,"std":0.03651},{"mean":0.951667,"std":0.035019}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.433667,"std":0.000577},{"mean":0.435667,"std":0.002082},{"mean":0.438667,"std":0.002517},{"mean":0.44,"std":0.002},{"mean":0.442,"std":0.002646},{"mean":0.447,"std":0.003464},{"mean":0.451333,"std":0.004041},{"mean":0.459,"std":0.003606},{"mean":0.468,"std":0.003},{"mean":0.481,"std":0.004},{"mean":0.493667,"std":0.006429},{"mean":0.512667,"std":0.010116},{"mean":0.536667,"std":0.009074},{"mean":0.567667,"std":0.016289},{"mean":0.595333,"std":0.020033},{"mean":0.635667,"std":0.017243},{"mean":0.709667,"std":0.024906},{"mean":0.797333,"std":0.031005},{"mean":0.951667,"std":0.035019}],"llm_call_rate":[{"mean":0.958667,"std":0.004187},{"mean":0.91303,"std":0.002637},{"mean":0.870788,"std":0.009802},{"mean":0.827697,"std":0.011259},{"mean":0.786788,"std":0.008321},{"mean":0.746182,"std":0.007119},{"mean":0.703697,"std":0.005634},{"mean":0.663091,"std":0.003107},{"mean":0.615455,"std":0.00619},{"mean":0.565697,"std":0.00516},{"mean":0.521212,"std":0.003422},{"mean":0.47303,"std":0.010453},{"mean":0.423333,"std":0.001377},{"mean":0.371212,"std":0.003203},{"mean":0.319394,"std":0.007697},{"mean":0.271394,"std":0.00258},{"mean":0.216485,"std":0.002068},{"mean":0.15303,"std":0.002169},{"mean":0.084545,"std":0.001273},{"mean":0.0,"std":0.0}]}}},"summed":{"margin":{"risk_coverage":[{"mean":0.006061,"std":0.002099},{"mean":0.006667,"std":0.003785},{"mean":0.007273,"std":0.003207},{"mean":0.010909,"std":0.002405},{"mean":0.012364,"std":0.001924},{"mean":0.015354,"std":0.000926},{"mean":0.019567,"std":0.002666},{"mean":0.026515,"std":0.002288},{"mean":0.030572,"std":0.001419},{"mean":0.034545,"std":0.000727},{"mean":0.043306,"std":0.00303},{"mean":0.055253,"std":0.005466},{"mean":0.068252,"std":0.005814},{"mean":0.085022,"std":0.004862},{"mean":0.106586,"std":0.005423},{"mean":0.133712,"std":0.005706},{"mean":0.160927,"std":0.007386},{"mean":0.190774,"std":0.007296},{"mean":0.222648,"std":0.005825},{"mean":0.251273,"std":0.005178}],"operating":{"tau":[{"mean":12.910461,"std":0.239533},{"mean":11.037371,"std":0.209738},{"mean":9.789653,"std":0.318794},{"mean":8.884489,"std":0.270605},{"mean":8.09251,"std":0.184565},{"mean":7.429423,"std":0.179112},{"mean":6.824605,"std":0.141513},{"mean":6.238637,"std":0.064451},{"mean":5.659161,"std":0.010336},{"mean":5.129821,"std":0.028536},{"mean":4.504534,"std":0.017367},{"mean":3.921352,"std":0.023891},{"mean":3.412943,"std":0.049026},{"mean":2.864244,"std":0.046002},{"mean":2.28759,"std":0.043523},{"mean":1.73102,"std":0.082742},{"mean":1.172467,"std":0.096404},{"mean":0.694646,"std":0.062559},{"mean":0.262693,"std":0.018201},{"mean":0.00089,"std":0.000332}],"coverage":[{"mean":0.041879,"std":0.00469},{"mean":0.085455,"std":0.005844},{"mean":0.129879,"std":0.005937},{"mean":0.172364,"std":0.009609},{"mean":0.214121,"std":0.009542},{"mean":0.253939,"std":0.008095},{"mean":0.296727,"std":0.006786},{"mean":0.339697,"std":0.003854},{"mean":0.384667,"std":0.002044},{"mean":0.425879,"std":0.004686},{"mean":0.476848,"std":0.004565},{"mean":0.526061,"std":0.004622},{"mean":0.569152,"std":0.007654},{"mean":0.617636,"std":0.004194},{"mean":0.670545,"std":0.001135},{"mean":0.725636,"std":0.00411},{"mean":0.786303,"std":0.007637},{"mean":0.847576,"std":0.00854},{"mean":0.925152,"std":0.002469},{"mean":0.999697,"std":0.00021}],"selective_risk":[{"mean":0.007158,"std":0.002148},{"mean":0.005753,"std":0.002743},{"mean":0.007077,"std":0.003913},{"mean":0.00812,"std":0.002847},{"mean":0.010832,"std":0.002591},{"mean":0.012658,"std":0.000919},{"mean":0.015525,"std":0.000902},{"mean":0.018542,"std":0.002537},{"mean":0.023942,"std":0.003155},{"mean":0.028167,"std":0.001996},{"mean":0.032535,"std":0.001803},{"mean":0.039288,"std":0.000957},{"mean":0.047378,"std":0.003694},{"mean":0.059354,"std":0.005092},{"mean":0.074571,"std":0.004134},{"mean":0.095117,"std":0.006172},{"mean":0.126207,"std":0.00781},{"mean":0.160045,"std":0.010737},{"mean":0.20576,"std":0.006575},{"mean":0.251045,"std":0.005328}],"accuracy_8":[{"mean":0.822485,"std":0.001034},{"mean":0.825091,"std":0.001373},{"mean":0.828545,"std":0.001273},{"mean":0.833697,"std":0.003093},{"mean":0.836485,"std":0.002599},{"mean":0.839939,"std":0.002401},{"mean":0.844,"std":0.001012},{"mean":0.847636,"std":0.000793},{"mean":0.85103,"std":0.000555},{"mean":0.854242,"std":0.001794},{"mean":0.858,"std":0.003273},{"mean":0.859273,"std":0.003549},{"mean":0.859939,"std":0.004462},{"mean":0.859636,"std":0.004999},{"mean":0.857212,"std":0.004338},{"mean":0.850545,"std":0.005366},{"mean":0.837152,"std":0.004547},{"mean":0.819152,"std":0.00674},{"mean":0.787394,"std":0.006193},{"mean":0.74897,"std":0.005456}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.001155},{"mean":0.002333,"std":0.000577},{"mean":0.003667,"std":0.002887},{"mean":0.006,"std":0.003606},{"mean":0.010333,"std":0.002517},{"mean":0.014,"std":0.001},{"mean":0.02,"std":0.002},{"mean":0.027,"std":0.004359},{"mean":0.038667,"std":0.008327},{"mean":0.050667,"std":0.005033},{"mean":0.066,"std":0.00781},{"mean":0.086667,"std":0.004041},{"mean":0.114333,"std":0.010599},{"mean":0.158667,"std":0.016197},{"mean":0.216333,"std":0.018502},{"mean":0.292667,"std":0.029366},{"mean":0.417,"std":0.035791},{"mean":0.558333,"std":0.040501},{"mean":0.773333,"std":0.030436},{"mean":0.986333,"std":0.015011}],"oos_misroute_rate":[{"mean":0.433333,"std":0.001155},{"mean":0.434333,"std":0.000577},{"mean":0.435,"std":0.001732},{"mean":0.435667,"std":0.001528},{"mean":0.438,"std":0.001},{"mean":0.440333,"std":0.001155},{"mean":0.442333,"std":0.002082},{"mean":0.445667,"std":0.004509},{"mean":0.451333,"std":0.005132},{"mean":0.458,"std":0.004359},{"mean":0.464667,"std":0.007024},{"mean":0.475667,"std":0.007506},{"mean":0.49,"std":0.01179},{"mean":0.516333,"std":0.014224},{"mean":0.546667,"std":0.017616},{"mean":0.591333,"std":0.026388},{"mean":0.659667,"std":0.026727},{"mean":0.733333,"std":0.025423},{"mean":0.858333,"std":0.028937},{"mean":0.986667,"std":0.015308}],"llm_call_rate":[{"mean":0.958121,"std":0.00469},{"mean":0.914545,"std":0.005844},{"mean":0.870121,"std":0.005937},{"mean":0.827636,"std":0.009609},{"mean":0.785879,"std":0.009542},{"mean":0.746061,"std":0.008095},{"mean":0.703273,"std":0.006786},{"mean":0.660303,"std":0.003854},{"mean":0.615333,"std":0.002044},{"mean":0.574121,"std":0.004686},{"mean":0.523152,"std":0.004565},{"mean":0.473939,"std":0.004622},{"mean":0.430848,"std":0.007654},{"mean":0.382364,"std":0.004194},{"mean":0.329455,"std":0.001135},{"mean":0.274364,"std":0.00411},{"mean":0.213697,"std":0.007637},{"mean":0.152424,"std":0.00854},{"mean":0.074848,"std":0.002469},{"mean":0.000303,"std":0.00021}]}},"msp_t":{"risk_coverage":[{"mean":0.006061,"std":0.002099},{"mean":0.006667,"std":0.003785},{"mean":0.007273,"std":0.003207},{"mean":0.010606,"std":0.003442},{"mean":0.012848,"std":0.002338},{"mean":0.016768,"std":0.003338},{"mean":0.022338,"std":0.001039},{"mean":0.027121,"std":0.001144},{"mean":0.031111,"std":0.001069},{"mean":0.03697,"std":0.000555},{"mean":0.044738,"std":0.003},{"mean":0.055657,"std":0.005128},{"mean":0.068718,"std":0.005265},{"mean":0.087013,"std":0.004935},{"mean":0.107556,"std":0.005604},{"mean":0.132727,"std":0.006552},{"mean":0.160143,"std":0.006849},{"mean":0.189899,"std":0.007225},{"mean":0.219968,"std":0.006477},{"mean":0.251273,"std":0.005178}],"operating":{"tau":[{"mean":0.999993,"std":1e-06},{"mean":0.999956,"std":1e-05},{"mean":0.999854,"std":5e-05},{"mean":0.999644,"std":0.000115},{"mean":0.999263,"std":0.00014},{"mean":0.998517,"std":0.000285},{"mean":0.997262,"std":0.000413},{"mean":0.995057,"std":0.000525},{"mean":0.991209,"std":0.000355},{"mean":0.98536,"std":0.000442},{"mean":0.973768,"std":0.002086},{"mean":0.95435,"std":0.001959},{"mean":0.924968,"std":0.001094},{"mean":0.879021,"std":0.002181},{"mean":0.806306,"std":0.011506},{"mean":0.703577,"std":0.010804},{"mean":0.585589,"std":0.015479},{"mean":0.451465,"std":0.02725},{"mean":0.309966,"std":0.006752},{"mean":0.190164,"std":0.006058}],"coverage":[{"mean":0.041273,"std":0.004882},{"mean":0.088061,"std":0.004261},{"mean":0.130364,"std":0.006392},{"mean":0.170667,"std":0.009719},{"mean":0.212182,"std":0.008526},{"mean":0.25503,"std":0.007103},{"mean":0.300182,"std":0.007774},{"mean":0.341939,"std":0.007663},{"mean":0.386606,"std":0.005079},{"mean":0.426606,"std":0.003413},{"mean":0.475212,"std":0.010147},{"mean":0.522788,"std":0.007787},{"mean":0.569394,"std":0.006113},{"mean":0.616667,"std":0.005774},{"mean":0.668545,"std":0.004949},{"mean":0.72297,"std":0.003527},{"mean":0.779939,"std":0.004506},{"mean":0.845212,"std":0.008579},{"mean":0.923091,"std":0.004738},{"mean":0.999152,"std":0.000584}],"selective_risk":[{"mean":0.0,"std":0.0},{"mean":0.005563,"std":0.002604},{"mean":0.007967,"std":0.003113},{"mean":0.008526,"std":0.003278},{"mean":0.011754,"std":0.002155},{"mean":0.014042,"std":0.001369},{"mean":0.016761,"std":0.002827},{"mean":0.020888,"std":0.001972},{"mean":0.024915,"std":0.002165},{"mean":0.028555,"std":0.00033},{"mean":0.035055,"std":0.001903},{"mean":0.040337,"std":0.000723},{"mean":0.048236,"std":0.003244},{"mean":0.059967,"std":0.004456},{"mean":0.074418,"std":0.005915},{"mean":0.095651,"std":0.003451},{"mean":0.123607,"std":0.006692},{"mean":0.157424,"std":0.008391},{"mean":0.204464,"std":0.004162},{"mean":0.250639,"std":0.0048}],"accuracy_8":[{"mean":0.822485,"std":0.000688},{"mean":0.825636,"std":0.001616},{"mean":0.829394,"std":0.002003},{"mean":0.833576,"std":0.00147},{"mean":0.837939,"std":0.001994},{"mean":0.841394,"std":0.002011},{"mean":0.845212,"std":0.000458},{"mean":0.848182,"std":0.000481},{"mean":0.851576,"std":0.001785},{"mean":0.854667,"std":0.003},{"mean":0.857152,"std":0.003926},{"mean":0.859273,"std":0.00381},{"mean":0.859697,"std":0.004746},{"mean":0.858788,"std":0.004433},{"mean":0.857152,"std":0.004365},{"mean":0.85103,"std":0.003305},{"mean":0.838182,"std":0.005042},{"mean":0.820788,"std":0.004859},{"mean":0.788424,"std":0.004338},{"mean":0.749394,"std":0.004784}],"high_conf_oos_misroute_rate":[{"mean":0.0,"std":0.0},{"mean":0.002333,"std":0.000577},{"mean":0.004,"std":0.002646},{"mean":0.006,"std":0.004},{"mean":0.010667,"std":0.002887},{"mean":0.015333,"std":0.003055},{"mean":0.021667,"std":0.003215},{"mean":0.028333,"std":0.001528},{"mean":0.038667,"std":0.005508},{"mean":0.049333,"std":0.002517},{"mean":0.066333,"std":0.008505},{"mean":0.086,"std":0.005568},{"mean":0.114,"std":0.008888},{"mean":0.153,"std":0.009644},{"mean":0.206333,"std":0.020133},{"mean":0.283333,"std":0.013317},{"mean":0.387,"std":0.02512},{"mean":0.524,"std":0.036014},{"mean":0.741,"std":0.017578},{"mean":0.984,"std":0.013}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.434333,"std":0.000577},{"mean":0.434667,"std":0.001155},{"mean":0.436,"std":0.002},{"mean":0.437667,"std":0.002082},{"mean":0.440333,"std":0.002082},{"mean":0.442333,"std":0.001528},{"mean":0.445667,"std":0.002517},{"mean":0.450333,"std":0.004933},{"mean":0.456333,"std":0.003512},{"mean":0.465,"std":0.007937},{"mean":0.475667,"std":0.005033},{"mean":0.491,"std":0.009539},{"mean":0.511,"std":0.008185},{"mean":0.537333,"std":0.016258},{"mean":0.580333,"std":0.013429},{"mean":0.636333,"std":0.024007},{"mean":0.703,"std":0.021932},{"mean":0.824333,"std":0.018903},{"mean":0.984667,"std":0.013577}],"llm_call_rate":[{"mean":0.958727,"std":0.004882},{"mean":0.911939,"std":0.004261},{"mean":0.869636,"std":0.006392},{"mean":0.829333,"std":0.009719},{"mean":0.787818,"std":0.008526},{"mean":0.74497,"std":0.007103},{"mean":0.699818,"std":0.007774},{"mean":0.658061,"std":0.007663},{"mean":0.613394,"std":0.005079},{"mean":0.573394,"std":0.003413},{"mean":0.524788,"std":0.010147},{"mean":0.477212,"std":0.007787},{"mean":0.430606,"std":0.006113},{"mean":0.383333,"std":0.005774},{"mean":0.331455,"std":0.004949},{"mean":0.27703,"std":0.003527},{"mean":0.220061,"std":0.004506},{"mean":0.154788,"std":0.008579},{"mean":0.076909,"std":0.004738},{"mean":0.000848,"std":0.000584}]}}}}},"tfidf-centroid/k5":{"model":"tfidf-centroid","k":5,"oos_train":null,"kind":"tfidf","seeds":[42,43,44],"run_names":["tfidf-centroid-k5-seed42","tfidf-centroid-k5-seed43","tfidf-centroid-k5-seed44"],"logits_sha256":["cac96d18d9cad091f8dd3822554c9b94ae38e08d90688d161f52560791ccb63b","dbfdd8f7f5107d61cae3e48f2e7a99916ecdf6a094047b4053e65fe43bc3ec1d","e3e751ba605ba46a492f6f8ba0759c63b41239d46bf01f9513dc0d1d5db175ba"],"result":{"argmax":{"margin":{"risk_coverage":[{"mean":0.013333,"std":0.01277},{"mean":0.019394,"std":0.010014},{"mean":0.029899,"std":0.01152},{"mean":0.038788,"std":0.009549},{"mean":0.04897,"std":0.008429},{"mean":0.058384,"std":0.002864},{"mean":0.071169,"std":0.004617},{"mean":0.085455,"std":0.002405},{"mean":0.103973,"std":0.003644},{"mean":0.123879,"std":0.007092},{"mean":0.145895,"std":0.008934},{"mean":0.166566,"std":0.006101},{"mean":0.187692,"std":0.005841},{"mean":0.206234,"std":0.00625},{"mean":0.223111,"std":0.005551},{"mean":0.241212,"std":0.00537},{"mean":0.259109,"std":0.006447},{"mean":0.278923,"std":0.006079},{"mean":0.297416,"std":0.005266},{"mean":0.316,"std":0.00619}],"operating":{"tau":[{"mean":9.476936,"std":0.281615},{"mean":7.828503,"std":0.308272},{"mean":6.597745,"std":0.128038},{"mean":5.630449,"std":0.204228},{"mean":4.879982,"std":0.167705},{"mean":4.180986,"std":0.139706},{"mean":3.667457,"std":0.09171},{"mean":3.161865,"std":0.079424},{"mean":2.696816,"std":0.057182},{"mean":2.301112,"std":0.047962},{"mean":1.915237,"std":0.016498},{"mean":1.603004,"std":0.023222},{"mean":1.305323,"std":0.011629},{"mean":1.045545,"std":0.024398},{"mean":0.822835,"std":0.017148},{"mean":0.626004,"std":0.010488},{"mean":0.436718,"std":0.007007},{"mean":0.270309,"std":0.014379},{"mean":0.12733,"std":0.013393},{"mean":0.0,"std":0.0}],"coverage":[{"mean":0.040303,"std":0.00488},{"mean":0.078788,"std":0.009731},{"mean":0.120545,"std":0.005781},{"mean":0.171091,"std":0.012889},{"mean":0.218788,"std":0.007129},{"mean":0.266727,"std":0.003004},{"mean":0.307697,"std":0.005904},{"mean":0.356121,"std":0.003494},{"mean":0.403576,"std":0.003},{"mean":0.45097,"std":0.004565},{"mean":0.503212,"std":0.002044},{"mean":0.554424,"std":0.005437},{"mean":0.610727,"std":0.005827},{"mean":0.667091,"std":0.00756},{"mean":0.722545,"std":0.00426},{"mean":0.771212,"std":0.001568},{"mean":0.827939,"std":0.001377},{"mean":0.884727,"std":0.007494},{"mean":0.943636,"std":0.002886},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.010918,"std":0.014812},{"mean":0.012582,"std":0.010977},{"mean":0.021197,"std":0.007362},{"mean":0.034989,"std":0.009255},{"mean":0.041378,"std":0.008828},{"mean":0.048834,"std":0.005923},{"mean":0.060996,"std":0.005635},{"mean":0.07198,"std":0.00392},{"mean":0.086347,"std":0.002088},{"mean":0.104036,"std":0.002645},{"mean":0.12585,"std":0.008016},{"mean":0.147087,"std":0.00964},{"mean":0.170062,"std":0.00746},{"mean":0.193997,"std":0.008132},{"mean":0.213552,"std":0.005454},{"mean":0.231437,"std":0.005555},{"mean":0.251078,"std":0.006552},{"mean":0.27266,"std":0.008676},{"mean":0.295182,"std":0.005328},{"mean":0.316,"std":0.00619}],"accuracy_8":[{"mean":0.822727,"std":0.000481},{"mean":0.825212,"std":0.000933},{"mean":0.827818,"std":0.002065},{"mean":0.830667,"std":0.003213},{"mean":0.833394,"std":0.003714},{"mean":0.835818,"std":0.002278},{"mean":0.836,"std":0.002678},{"mean":0.835879,"std":0.003305},{"mean":0.834485,"std":0.004034},{"mean":0.83097,"std":0.00348},{"mean":0.825152,"std":0.002599},{"mean":0.817333,"std":0.000735},{"mean":0.806667,"std":0.000378},{"mean":0.793212,"std":0.00191},{"mean":0.781455,"std":0.001488},{"mean":0.766848,"std":0.002888},{"mean":0.750121,"std":0.003234},{"mean":0.731212,"std":0.005567},{"mean":0.708364,"std":0.005292},{"mean":0.684,"std":0.00619}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.001},{"mean":0.002333,"std":0.002082},{"mean":0.008667,"std":0.003055},{"mean":0.02,"std":0.003606},{"mean":0.031,"std":0.008},{"mean":0.043667,"std":0.006028},{"mean":0.063,"std":0.00755},{"mean":0.082667,"std":0.006658},{"mean":0.114,"std":0.005568},{"mean":0.152333,"std":0.003512},{"mean":0.207,"std":0.019053},{"mean":0.262667,"std":0.02409},{"mean":0.338333,"std":0.017673},{"mean":0.417333,"std":0.020526},{"mean":0.498,"std":0.007},{"mean":0.572667,"std":0.003786},{"mean":0.667,"std":0.01253},{"mean":0.770333,"std":0.015503},{"mean":0.888,"std":0.008},{"mean":0.990667,"std":0.009238}],"oos_misroute_rate":[{"mean":0.432333,"std":0.000577},{"mean":0.433333,"std":0.001155},{"mean":0.436,"std":0.002},{"mean":0.443333,"std":0.002082},{"mean":0.450667,"std":0.00611},{"mean":0.456333,"std":0.006658},{"mean":0.466,"std":0.004359},{"mean":0.476667,"std":0.002517},{"mean":0.494667,"std":0.003786},{"mean":0.516333,"std":0.006658},{"mean":0.545667,"std":0.000577},{"mean":0.577333,"std":0.003055},{"mean":0.62,"std":0.002},{"mean":0.664333,"std":0.004726},{"mean":0.705,"std":0.005292},{"mean":0.746667,"std":0.006429},{"mean":0.801333,"std":0.009452},{"mean":0.862333,"std":0.001155},{"mean":0.929,"std":0.005292},{"mean":0.990667,"std":0.009238}],"llm_call_rate":[{"mean":0.959697,"std":0.00488},{"mean":0.921212,"std":0.009731},{"mean":0.879455,"std":0.005781},{"mean":0.828909,"std":0.012889},{"mean":0.781212,"std":0.007129},{"mean":0.733273,"std":0.003004},{"mean":0.692303,"std":0.005904},{"mean":0.643879,"std":0.003494},{"mean":0.596424,"std":0.003},{"mean":0.54903,"std":0.004565},{"mean":0.496788,"std":0.002044},{"mean":0.445576,"std":0.005437},{"mean":0.389273,"std":0.005827},{"mean":0.332909,"std":0.00756},{"mean":0.277455,"std":0.00426},{"mean":0.228788,"std":0.001568},{"mean":0.172061,"std":0.001377},{"mean":0.115273,"std":0.007494},{"mean":0.056364,"std":0.002886},{"mean":0.0,"std":0.0}]}},"msp_t":{"risk_coverage":[{"mean":0.014545,"std":0.012597},{"mean":0.023636,"std":0.013111},{"mean":0.030707,"std":0.010309},{"mean":0.037879,"std":0.006701},{"mean":0.048485,"std":0.009035},{"mean":0.054343,"std":0.004936},{"mean":0.066147,"std":0.002163},{"mean":0.077424,"std":0.005007},{"mean":0.09064,"std":0.005888},{"mean":0.105091,"std":0.010583},{"mean":0.123747,"std":0.009246},{"mean":0.142121,"std":0.011286},{"mean":0.16,"std":0.009222},{"mean":0.18026,"std":0.009132},{"mean":0.200808,"std":0.009576},{"mean":0.220985,"std":0.008607},{"mean":0.243422,"std":0.009047},{"mean":0.266599,"std":0.007691},{"mean":0.290718,"std":0.00653},{"mean":0.316,"std":0.00619}],"operating":{"tau":[{"mean":0.999365,"std":0.000145},{"mean":0.99753,"std":0.000515},{"mean":0.992235,"std":0.000992},{"mean":0.981615,"std":0.002005},{"mean":0.9656,"std":0.003651},{"mean":0.940689,"std":0.001377},{"mean":0.903319,"std":0.004452},{"mean":0.848465,"std":0.000724},{"mean":0.785761,"std":0.005726},{"mean":0.711458,"std":0.008263},{"mean":0.63439,"std":0.010231},{"mean":0.556366,"std":0.01625},{"mean":0.478976,"std":0.015512},{"mean":0.404191,"std":0.007105},{"mean":0.335851,"std":0.005264},{"mean":0.272175,"std":0.006091},{"mean":0.214558,"std":0.004652},{"mean":0.155676,"std":0.006846},{"mean":0.099367,"std":0.005315},{"mean":0.006623,"std":0.0}],"coverage":[{"mean":0.041939,"std":0.004784},{"mean":0.077091,"std":0.008043},{"mean":0.124727,"std":0.005354},{"mean":0.174303,"std":0.005617},{"mean":0.21697,"std":0.004866},{"mean":0.258303,"std":0.002637},{"mean":0.301091,"std":0.006638},{"mean":0.348727,"std":0.00461},{"mean":0.395939,"std":0.001718},{"mean":0.447152,"std":0.001481},{"mean":0.498303,"std":0.001412},{"mean":0.546242,"std":0.006561},{"mean":0.599152,"std":0.010499},{"mean":0.648485,"std":0.009782},{"mean":0.699273,"std":0.007035},{"mean":0.750485,"std":0.003745},{"mean":0.803939,"std":0.003203},{"mean":0.86703,"std":0.009396},{"mean":0.935455,"std":0.008105},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.009979,"std":0.008655},{"mean":0.019623,"std":0.013975},{"mean":0.028122,"std":0.010673},{"mean":0.03332,"std":0.010262},{"mean":0.040538,"std":0.005448},{"mean":0.049693,"std":0.007401},{"mean":0.054708,"std":0.005171},{"mean":0.065705,"std":0.001374},{"mean":0.076239,"std":0.003916},{"mean":0.089043,"std":0.006709},{"mean":0.10364,"std":0.009831},{"mean":0.121735,"std":0.011321},{"mean":0.141246,"std":0.014204},{"mean":0.159527,"std":0.012111},{"mean":0.179623,"std":0.010811},{"mean":0.200967,"std":0.01108},{"mean":0.222767,"std":0.008999},{"mean":0.251108,"std":0.010341},{"mean":0.283409,"std":0.0088},{"mean":0.316,"std":0.00619}],"accuracy_8":[{"mean":0.822242,"std":0.000378},{"mean":0.824,"std":0.000833},{"mean":0.826667,"std":0.00134},{"mean":0.829697,"std":0.001568},{"mean":0.831576,"std":0.002586},{"mean":0.833333,"std":0.003728},{"mean":0.835636,"std":0.003107},{"mean":0.836485,"std":0.003135},{"mean":0.836061,"std":0.004514},{"mean":0.83497,"std":0.004163},{"mean":0.832121,"std":0.004676},{"mean":0.826121,"std":0.005498},{"mean":0.819636,"std":0.007014},{"mean":0.811333,"std":0.005699},{"mean":0.799697,"std":0.005552},{"mean":0.785333,"std":0.00631},{"mean":0.769576,"std":0.005446},{"mean":0.745152,"std":0.008134},{"mean":0.716182,"std":0.007948},{"mean":0.684,"std":0.00619}],"high_conf_oos_misroute_rate":[{"mean":0.001,"std":0.001},{"mean":0.003333,"std":0.003512},{"mean":0.009333,"std":0.004041},{"mean":0.019667,"std":0.007234},{"mean":0.029667,"std":0.005033},{"mean":0.039667,"std":0.008145},{"mean":0.051333,"std":0.004041},{"mean":0.07,"std":0.002646},{"mean":0.093,"std":0.003606},{"mean":0.124333,"std":0.009866},{"mean":0.166333,"std":0.018502},{"mean":0.211,"std":0.023643},{"mean":0.266,"std":0.029138},{"mean":0.325,"std":0.027495},{"mean":0.392,"std":0.023431},{"mean":0.469333,"std":0.024502},{"mean":0.564333,"std":0.017898},{"mean":0.683,"std":0.026058},{"mean":0.828667,"std":0.013429},{"mean":0.990667,"std":0.009238}],"oos_misroute_rate":[{"mean":0.432333,"std":0.000577},{"mean":0.433333,"std":0.001528},{"mean":0.436333,"std":0.001155},{"mean":0.443333,"std":0.004163},{"mean":0.449,"std":0.003},{"mean":0.453333,"std":0.004163},{"mean":0.459333,"std":0.002082},{"mean":0.47,"std":0.001},{"mean":0.484333,"std":0.004509},{"mean":0.499,"std":0.004583},{"mean":0.523333,"std":0.007024},{"mean":0.546,"std":0.01},{"mean":0.571333,"std":0.012858},{"mean":0.602,"std":0.013077},{"mean":0.638333,"std":0.012503},{"mean":0.682,"std":0.013892},{"mean":0.736333,"std":0.011719},{"mean":0.808,"std":0.023812},{"mean":0.894,"std":0.008888},{"mean":0.990667,"std":0.009238}],"llm_call_rate":[{"mean":0.958061,"std":0.004784},{"mean":0.922909,"std":0.008043},{"mean":0.875273,"std":0.005354},{"mean":0.825697,"std":0.005617},{"mean":0.78303,"std":0.004866},{"mean":0.741697,"std":0.002637},{"mean":0.698909,"std":0.006638},{"mean":0.651273,"std":0.00461},{"mean":0.604061,"std":0.001718},{"mean":0.552848,"std":0.001481},{"mean":0.501697,"std":0.001412},{"mean":0.453758,"std":0.006561},{"mean":0.400848,"std":0.010499},{"mean":0.351515,"std":0.009782},{"mean":0.300727,"std":0.007035},{"mean":0.249515,"std":0.003745},{"mean":0.196061,"std":0.003203},{"mean":0.13297,"std":0.009396},{"mean":0.064545,"std":0.008105},{"mean":0.0,"std":0.0}]}}},"summed":{"margin":{"risk_coverage":[{"mean":0.006061,"std":0.002099},{"mean":0.014545,"std":0.008332},{"mean":0.020202,"std":0.005728},{"mean":0.026061,"std":0.004665},{"mean":0.03297,"std":0.004135},{"mean":0.042222,"std":0.003551},{"mean":0.048312,"std":0.003746},{"mean":0.05803,"std":0.001144},{"mean":0.067205,"std":0.003484},{"mean":0.081455,"std":0.004542},{"mean":0.096639,"std":0.008267},{"mean":0.116162,"std":0.008312},{"mean":0.138741,"std":0.009572},{"mean":0.163636,"std":0.007433},{"mean":0.186747,"std":0.006253},{"mean":0.212879,"std":0.005234},{"mean":0.239287,"std":0.00589},{"mean":0.265993,"std":0.005519},{"mean":0.293546,"std":0.004587},{"mean":0.317879,"std":0.005437}],"operating":{"tau":[{"mean":9.458876,"std":0.05429},{"mean":8.011198,"std":0.131486},{"mean":6.985644,"std":0.167622},{"mean":6.111103,"std":0.122548},{"mean":5.411855,"std":0.10297},{"mean":4.823082,"std":0.110899},{"mean":4.241514,"std":0.079529},{"mean":3.740341,"std":0.074569},{"mean":3.256862,"std":0.065595},{"mean":2.848579,"std":0.064833},{"mean":2.42734,"std":0.092037},{"mean":2.064189,"std":0.080121},{"mean":1.689093,"std":0.055208},{"mean":1.37018,"std":0.066361},{"mean":1.073861,"std":0.080812},{"mean":0.786648,"std":0.068681},{"mean":0.516024,"std":0.069133},{"mean":0.308137,"std":0.032987},{"mean":0.143943,"std":0.016079},{"mean":0.001418,"std":0.000626}],"coverage":[{"mean":0.037515,"std":0.003833},{"mean":0.079455,"std":0.008387},{"mean":0.123636,"std":0.013619},{"mean":0.171636,"std":0.010746},{"mean":0.215212,"std":0.010019},{"mean":0.258727,"std":0.009012},{"mean":0.303515,"std":0.003574},{"mean":0.34697,"std":0.001315},{"mean":0.394545,"std":0.00426},{"mean":0.43703,"std":0.002345},{"mean":0.485455,"std":0.001488},{"mean":0.530182,"std":0.003623},{"mean":0.578848,"std":0.006599},{"mean":0.629333,"std":0.012608},{"mean":0.681455,"std":0.014088},{"mean":0.743818,"std":0.015021},{"mean":0.813091,"std":0.017338},{"mean":0.874303,"std":0.01013},{"mean":0.937636,"std":0.001106},{"mean":0.999394,"std":0.00021}],"selective_risk":[{"mean":0.006412,"std":0.002437},{"mean":0.013089,"std":0.010255},{"mean":0.017344,"std":0.005954},{"mean":0.023805,"std":0.0052},{"mean":0.026169,"std":0.003435},{"mean":0.033908,"std":0.004643},{"mean":0.042912,"std":0.004239},{"mean":0.048374,"std":0.004197},{"mean":0.056058,"std":0.00154},{"mean":0.065053,"std":0.004101},{"mean":0.077645,"std":0.004318},{"mean":0.091884,"std":0.007829},{"mean":0.105971,"std":0.011982},{"mean":0.127973,"std":0.015758},{"mean":0.154371,"std":0.014518},{"mean":0.183566,"std":0.012744},{"mean":0.219973,"std":0.014891},{"mean":0.252727,"std":0.010614},{"mean":0.286985,"std":0.005596},{"mean":0.317525,"std":0.005568}],"accuracy_8":[{"mean":0.822182,"std":0.000545},{"mean":0.824848,"std":0.001034},{"mean":0.828182,"std":0.000481},{"mean":0.830909,"std":0.001311},{"mean":0.833939,"std":0.001315},{"mean":0.836364,"std":0.002405},{"mean":0.837515,"std":0.002309},{"mean":0.839212,"std":0.002373},{"mean":0.840848,"std":0.003161},{"mean":0.840424,"std":0.005876},{"mean":0.839576,"std":0.005144},{"mean":0.836121,"std":0.006367},{"mean":0.832485,"std":0.007028},{"mean":0.824545,"std":0.00756},{"mean":0.812606,"std":0.006004},{"mean":0.794848,"std":0.007708},{"mean":0.768848,"std":0.009117},{"mean":0.743697,"std":0.007943},{"mean":0.714121,"std":0.004547},{"mean":0.682424,"std":0.005587}],"high_conf_oos_misroute_rate":[{"mean":0.000667,"std":0.000577},{"mean":0.001667,"std":0.001155},{"mean":0.005,"std":0.001},{"mean":0.012667,"std":0.001155},{"mean":0.017667,"std":0.001155},{"mean":0.029667,"std":0.003215},{"mean":0.045,"std":0.006},{"mean":0.056667,"std":0.00611},{"mean":0.076,"std":0.002646},{"mean":0.093333,"std":0.002517},{"mean":0.126,"std":0.00755},{"mean":0.166333,"std":0.016258},{"mean":0.211333,"std":0.022679},{"mean":0.281,"std":0.040447},{"mean":0.368,"std":0.042036},{"mean":0.466333,"std":0.036364},{"mean":0.599,"std":0.041797},{"mean":0.728333,"std":0.023965},{"mean":0.858333,"std":0.005508},{"mean":0.995,"std":0.004583}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.432333,"std":0.000577},{"mean":0.435,"std":0.001},{"mean":0.438,"std":0.002},{"mean":0.440333,"std":0.003215},{"mean":0.448333,"std":0.000577},{"mean":0.454667,"std":0.003055},{"mean":0.459667,"std":0.001528},{"mean":0.468667,"std":0.003215},{"mean":0.477667,"std":0.007095},{"mean":0.490667,"std":0.010066},{"mean":0.513667,"std":0.015275},{"mean":0.538,"std":0.017692},{"mean":0.575667,"std":0.026006},{"mean":0.622667,"std":0.022143},{"mean":0.680333,"std":0.01823},{"mean":0.761,"std":0.021},{"mean":0.838,"std":0.019287},{"mean":0.912333,"std":0.011504},{"mean":0.996,"std":0.003606}],"llm_call_rate":[{"mean":0.962485,"std":0.003833},{"mean":0.920545,"std":0.008387},{"mean":0.876364,"std":0.013619},{"mean":0.828364,"std":0.010746},{"mean":0.784788,"std":0.010019},{"mean":0.741273,"std":0.009012},{"mean":0.696485,"std":0.003574},{"mean":0.65303,"std":0.001315},{"mean":0.605455,"std":0.00426},{"mean":0.56297,"std":0.002345},{"mean":0.514545,"std":0.001488},{"mean":0.469818,"std":0.003623},{"mean":0.421152,"std":0.006599},{"mean":0.370667,"std":0.012608},{"mean":0.318545,"std":0.014088},{"mean":0.256182,"std":0.015021},{"mean":0.186909,"std":0.017338},{"mean":0.125697,"std":0.01013},{"mean":0.062364,"std":0.001106},{"mean":0.000606,"std":0.00021}]}},"msp_t":{"risk_coverage":[{"mean":0.010909,"std":0.003636},{"mean":0.01697,"std":0.005555},{"mean":0.022626,"std":0.005979},{"mean":0.028485,"std":0.005845},{"mean":0.035394,"std":0.004135},{"mean":0.045253,"std":0.004035},{"mean":0.052814,"std":0.005976},{"mean":0.059848,"std":0.001721},{"mean":0.071246,"std":0.004854},{"mean":0.086303,"std":0.006826},{"mean":0.099174,"std":0.008913},{"mean":0.12,"std":0.009754},{"mean":0.140513,"std":0.010407},{"mean":0.163203,"std":0.009373},{"mean":0.186909,"std":0.005923},{"mean":0.212348,"std":0.005647},{"mean":0.237148,"std":0.005081},{"mean":0.262694,"std":0.006532},{"mean":0.289442,"std":0.004782},{"mean":0.317879,"std":0.005437}],"operating":{"tau":[{"mean":0.999781,"std":2.3e-05},{"mean":0.999058,"std":0.000118},{"mean":0.997439,"std":0.000393},{"mean":0.993987,"std":0.000917},{"mean":0.988013,"std":0.001047},{"mean":0.978741,"std":0.001513},{"mean":0.963787,"std":0.002723},{"mean":0.941679,"std":0.002851},{"mean":0.909033,"std":0.007509},{"mean":0.868397,"std":0.005033},{"mean":0.817326,"std":0.010888},{"mean":0.749726,"std":0.011236},{"mean":0.678811,"std":0.010381},{"mean":0.60651,"std":0.012142},{"mean":0.52959,"std":0.015637},{"mean":0.465112,"std":0.010742},{"mean":0.39415,"std":0.009046},{"mean":0.327803,"std":0.008806},{"mean":0.269453,"std":0.008742},{"mean":0.182926,"std":0.013392}],"coverage":[{"mean":0.037152,"std":0.005195},{"mean":0.080061,"std":0.007903},{"mean":0.121818,"std":0.011529},{"mean":0.170121,"std":0.013333},{"mean":0.215697,"std":0.007368},{"mean":0.257576,"std":0.006403},{"mean":0.301333,"std":0.003442},{"mean":0.345394,"std":0.005558},{"mean":0.392424,"std":0.000897},{"mean":0.436061,"std":0.004665},{"mean":0.481818,"std":0.003154},{"mean":0.532424,"std":0.003354},{"mean":0.581212,"std":0.004732},{"mean":0.629152,"std":0.012055},{"mean":0.688364,"std":0.010432},{"mean":0.740364,"std":0.010776},{"mean":0.803152,"std":0.008949},{"mean":0.871091,"std":0.009718},{"mean":0.935939,"std":0.008301},{"mean":0.999576,"std":0.000458}],"selective_risk":[{"mean":0.008266,"std":0.003184},{"mean":0.015511,"std":0.008869},{"mean":0.020214,"std":0.00821},{"mean":0.025033,"std":0.007919},{"mean":0.031167,"std":0.007669},{"mean":0.037172,"std":0.002746},{"mean":0.045654,"std":0.003893},{"mean":0.052749,"std":0.006264},{"mean":0.059151,"std":0.000218},{"mean":0.068682,"std":0.003253},{"mean":0.081739,"std":0.007929},{"mean":0.094214,"std":0.010213},{"mean":0.110266,"std":0.010922},{"mean":0.132081,"std":0.014098},{"mean":0.157896,"std":0.014437},{"mean":0.180798,"std":0.011695},{"mean":0.213795,"std":0.008929},{"mean":0.247963,"std":0.010179},{"mean":0.28176,"std":0.008493},{"mean":0.317588,"std":0.005752}],"accuracy_8":[{"mean":0.822424,"std":0.000555},{"mean":0.825394,"std":0.000859},{"mean":0.828121,"std":0.00021},{"mean":0.831576,"std":0.001001},{"mean":0.833455,"std":0.001373},{"mean":0.836,"std":0.001135},{"mean":0.837212,"std":0.002573},{"mean":0.838,"std":0.002578},{"mean":0.839576,"std":0.002917},{"mean":0.839394,"std":0.004746},{"mean":0.837818,"std":0.006462},{"mean":0.836424,"std":0.005699},{"mean":0.831212,"std":0.006995},{"mean":0.822182,"std":0.008346},{"mean":0.809455,"std":0.008931},{"mean":0.797394,"std":0.006876},{"mean":0.775394,"std":0.005932},{"mean":0.748424,"std":0.007196},{"mean":0.718727,"std":0.006008},{"mean":0.682364,"std":0.005715}],"high_conf_oos_misroute_rate":[{"mean":0.000667,"std":0.000577},{"mean":0.002333,"std":0.001155},{"mean":0.005667,"std":0.002309},{"mean":0.011667,"std":0.003215},{"mean":0.022333,"std":0.005132},{"mean":0.032333,"std":0.003215},{"mean":0.045333,"std":0.005033},{"mean":0.058,"std":0.008185},{"mean":0.076333,"std":0.001528},{"mean":0.095,"std":0.003464},{"mean":0.128,"std":0.011533},{"mean":0.167667,"std":0.019035},{"mean":0.213333,"std":0.014742},{"mean":0.278,"std":0.027875},{"mean":0.357333,"std":0.031086},{"mean":0.434667,"std":0.030534},{"mean":0.558,"std":0.021},{"mean":0.696333,"std":0.029263},{"mean":0.848,"std":0.021166},{"mean":0.995667,"std":0.003512}],"oos_misroute_rate":[{"mean":0.432,"std":0.0},{"mean":0.433,"std":0.001},{"mean":0.435,"std":0.0},{"mean":0.436333,"std":0.000577},{"mean":0.444667,"std":0.002082},{"mean":0.450667,"std":0.000577},{"mean":0.454333,"std":0.001528},{"mean":0.458667,"std":0.000577},{"mean":0.468667,"std":0.005508},{"mean":0.479333,"std":0.006429},{"mean":0.495333,"std":0.012014},{"mean":0.51,"std":0.01044},{"mean":0.538,"std":0.01249},{"mean":0.573,"std":0.015},{"mean":0.616,"std":0.01852},{"mean":0.656,"std":0.01562},{"mean":0.726,"std":0.01044},{"mean":0.804667,"std":0.012662},{"mean":0.899333,"std":0.004509},{"mean":0.996333,"std":0.004041}],"llm_call_rate":[{"mean":0.962848,"std":0.005195},{"mean":0.919939,"std":0.007903},{"mean":0.878182,"std":0.011529},{"mean":0.829879,"std":0.013333},{"mean":0.784303,"std":0.007368},{"mean":0.742424,"std":0.006403},{"mean":0.698667,"std":0.003442},{"mean":0.654606,"std":0.005558},{"mean":0.607576,"std":0.000897},{"mean":0.563939,"std":0.004665},{"mean":0.518182,"std":0.003154},{"mean":0.467576,"std":0.003354},{"mean":0.418788,"std":0.004732},{"mean":0.370848,"std":0.012055},{"mean":0.311636,"std":0.010432},{"mean":0.259636,"std":0.010776},{"mean":0.196848,"std":0.008949},{"mean":0.128909,"std":0.009718},{"mean":0.064061,"std":0.008301},{"mean":0.000424,"std":0.000458}]}}}}},"tfidf-centroid/k50":{"model":"tfidf-centroid","k":50,"oos_train":null,"kind":"tfidf","seeds":[42,43,44],"run_names":["tfidf-centroid-k50-seed42","tfidf-centroid-k50-seed43","tfidf-centroid-k50-seed44"],"logits_sha256":["27608c5f570bcb13a37d9cff167a0c2980f2915bc0fbce190c0027b0e52c3b93","9f8eed2728260b4b58e05b7e7d70b79b4b5f74d6b9f7a01decc8cff13e42aa2f","3fa2c12a4a881fe0c9148c86acc9be6edd315d968621aecf6f7936c0927d79ab"],"result":{"argmax":{"margin":{"risk_coverage":[{"mean":0.002424,"std":0.002099},{"mean":0.010909,"std":0.001818},{"mean":0.014545,"std":0.002099},{"mean":0.016364,"std":0.004166},{"mean":0.019636,"std":0.002622},{"mean":0.024646,"std":0.00035},{"mean":0.028398,"std":0.000794},{"mean":0.035152,"std":0.002663},{"mean":0.041481,"std":0.001682},{"mean":0.048364,"std":0.002622},{"mean":0.057851,"std":0.002624},{"mean":0.071515,"std":0.003207},{"mean":0.087273,"std":0.001453},{"mean":0.10381,"std":0.002469},{"mean":0.122586,"std":0.003465},{"mean":0.144924,"std":0.004553},{"mean":0.167415,"std":0.003218},{"mean":0.190842,"std":0.002888},{"mean":0.213461,"std":0.002393},{"mean":0.235455,"std":0.003933}],"operating":{"tau":[{"mean":13.214111,"std":0.181127},{"mean":11.019483,"std":0.137581},{"mean":9.621628,"std":0.168232},{"mean":8.644848,"std":0.084069},{"mean":7.73627,"std":0.048491},{"mean":6.920814,"std":0.070508},{"mean":6.18389,"std":0.041303},{"mean":5.527582,"std":0.016148},{"mean":4.913488,"std":0.017564},{"mean":4.306695,"std":0.021805},{"mean":3.729762,"std":0.029492},{"mean":3.16667,"std":0.047189},{"mean":2.63752,"std":0.016253},{"mean":2.143918,"std":0.022608},{"mean":1.689021,"std":0.032005},{"mean":1.281918,"std":0.021813},{"mean":0.894167,"std":0.047873},{"mean":0.561143,"std":0.039171},{"mean":0.259734,"std":0.030052},{"mean":0.0,"std":0.0}],"coverage":[{"mean":0.040727,"std":0.00246},{"mean":0.083576,"std":0.001065},{"mean":0.130424,"std":0.006099},{"mean":0.172,"std":0.00769},{"mean":0.216788,"std":0.006656},{"mean":0.261758,"std":0.009325},{"mean":0.307333,"std":0.001514},{"mean":0.349212,"std":0.004087},{"mean":0.391758,"std":0.004801},{"mean":0.437758,"std":0.005575},{"mean":0.484727,"std":0.007197},{"mean":0.53497,"std":0.007273},{"mean":0.583939,"std":0.002699},{"mean":0.633515,"std":0.00129},{"mean":0.684182,"std":0.005357},{"mean":0.741152,"std":0.002123},{"mean":0.797576,"std":0.00305},{"mean":0.863636,"std":0.003102},{"mean":0.93,"std":0.003938},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.001587,"std":0.002749},{"mean":0.005806,"std":0.001281},{"mean":0.012458,"std":0.003674},{"mean":0.014888,"std":0.003752},{"mean":0.017405,"std":0.003896},{"mean":0.020621,"std":0.001203},{"mean":0.025834,"std":0.000397},{"mean":0.028637,"std":0.000524},{"mean":0.033744,"std":0.002571},{"mean":0.039321,"std":0.00148},{"mean":0.045393,"std":0.000958},{"mean":0.054269,"std":0.001043},{"mean":0.068403,"std":0.002096},{"mean":0.082084,"std":0.001905},{"mean":0.097445,"std":0.001432},{"mean":0.118564,"std":0.003596},{"mean":0.143836,"std":0.005833},{"mean":0.174234,"std":0.004878},{"mean":0.204484,"std":0.004412},{"mean":0.235455,"std":0.003933}],"accuracy_8":[{"mean":0.822788,"std":0.000584},{"mean":0.82503,"std":0.000378},{"mean":0.828909,"std":0.001012},{"mean":0.832303,"std":0.000639},{"mean":0.835455,"std":0.001192},{"mean":0.84,"std":0.000727},{"mean":0.843576,"std":0.001389},{"mean":0.847697,"std":0.002003},{"mean":0.852424,"std":0.00258},{"mean":0.856727,"std":0.000481},{"mean":0.859455,"std":0.001012},{"mean":0.860727,"std":0.001106},{"mean":0.85897,"std":0.001689},{"mean":0.856788,"std":0.002469},{"mean":0.852545,"std":0.003091},{"mean":0.843576,"std":0.004026},{"mean":0.830848,"std":0.005095},{"mean":0.812727,"std":0.005292},{"mean":0.790848,"std":0.004055},{"mean":0.764545,"std":0.003933}],"high_conf_oos_misroute_rate":[{"mean":0.000333,"std":0.000577},{"mean":0.002333,"std":0.000577},{"mean":0.006667,"std":0.002082},{"mean":0.009667,"std":0.001528},{"mean":0.014667,"std":0.003786},{"mean":0.022333,"std":0.001528},{"mean":0.033333,"std":0.001528},{"mean":0.043,"std":0.001732},{"mean":0.055667,"std":0.005686},{"mean":0.072667,"std":0.000577},{"mean":0.093667,"std":0.004619},{"mean":0.124,"std":0.003606},{"mean":0.170333,"std":0.008327},{"mean":0.222333,"std":0.009866},{"mean":0.28,"std":0.008185},{"mean":0.362333,"std":0.021548},{"mean":0.464,"std":0.027622},{"mean":0.603,"std":0.016093},{"mean":0.759333,"std":0.01601},{"mean":0.938333,"std":0.018583}],"oos_misroute_rate":[{"mean":0.432333,"std":0.000577},{"mean":0.433667,"std":0.000577},{"mean":0.435333,"std":0.000577},{"mean":0.436333,"std":0.001155},{"mean":0.439333,"std":0.001528},{"mean":0.443,"std":0.001},{"mean":0.451,"std":0.002646},{"mean":0.456333,"std":0.001528},{"mean":0.460333,"std":0.003512},{"mean":0.469667,"std":0.002309},{"mean":0.482667,"std":0.006028},{"mean":0.501333,"std":0.005508},{"mean":0.526667,"std":0.005774},{"mean":0.554667,"std":0.006658},{"mean":0.584333,"std":0.004041},{"mean":0.627667,"std":0.01601},{"mean":0.681333,"std":0.023352},{"mean":0.753333,"std":0.022301},{"mean":0.836333,"std":0.014154},{"mean":0.938333,"std":0.018583}],"llm_call_rate":[{"mean":0.959273,"std":0.00246},{"mean":0.916424,"std":0.001065},{"mean":0.869576,"std":0.006099},{"mean":0.828,"std":0.00769},{"mean":0.783212,"std":0.006656},{"mean":0.738242,"std":0.009325},{"mean":0.692667,"std":0.001514},{"mean":0.650788,"std":0.004087},{"mean":0.608242,"std":0.004801},{"mean":0.562242,"std":0.005575},{"mean":0.515273,"std":0.007197},{"mean":0.46503,"std":0.007273},{"mean":0.416061,"std":0.002699},{"mean":0.366485,"std":0.00129},{"mean":0.315818,"std":0.005357},{"mean":0.258848,"std":0.002123},{"mean":0.202424,"std":0.00305},{"mean":0.136364,"std":0.003102},{"mean":0.07,"std":0.003938},{"mean":0.0,"std":0.0}]}},"msp_t":{"risk_coverage":[{"mean":0.004848,"std":0.002099},{"mean":0.009091,"std":0.003149},{"mean":0.016162,"std":0.00305},{"mean":0.019091,"std":0.002405},{"mean":0.021091,"std":0.002909},{"mean":0.024242,"std":0.000606},{"mean":0.029091,"std":0.002078},{"mean":0.033788,"std":0.002242},{"mean":0.036902,"std":0.001419},{"mean":0.042667,"std":0.001514},{"mean":0.051791,"std":0.001631},{"mean":0.061212,"std":0.002891},{"mean":0.074872,"std":0.000855},{"mean":0.088918,"std":0.00167},{"mean":0.107879,"std":0.002566},{"mean":0.128106,"std":0.003653},{"mean":0.152941,"std":0.004492},{"mean":0.178451,"std":0.003093},{"mean":0.207464,"std":0.003386},{"mean":0.235455,"std":0.003933}],"operating":{"tau":[{"mean":0.99999,"std":1e-06},{"mean":0.999928,"std":8e-06},{"mean":0.999742,"std":4e-05},{"mean":0.999261,"std":3.8e-05},{"mean":0.998304,"std":6.9e-05},{"mean":0.996357,"std":0.000187},{"mean":0.992894,"std":0.000206},{"mean":0.98666,"std":0.000305},{"mean":0.975485,"std":0.001371},{"mean":0.960215,"std":0.002838},{"mean":0.932927,"std":0.003589},{"mean":0.887162,"std":0.003314},{"mean":0.822129,"std":0.001382},{"mean":0.74635,"std":0.006873},{"mean":0.647343,"std":0.013936},{"mean":0.550685,"std":0.011921},{"mean":0.447806,"std":0.00683},{"mean":0.301095,"std":0.011032},{"mean":0.159147,"std":0.014611},{"mean":0.006623,"std":0.0}],"coverage":[{"mean":0.042,"std":0.001734},{"mean":0.083152,"std":0.001169},{"mean":0.127758,"std":0.006531},{"mean":0.174,"std":0.006158},{"mean":0.215636,"std":0.004639},{"mean":0.256,"std":0.004287},{"mean":0.297333,"std":0.002482},{"mean":0.340485,"std":0.002723},{"mean":0.387697,"std":0.004832},{"mean":0.429152,"std":0.007043},{"mean":0.479758,"std":0.007876},{"mean":0.531697,"std":0.005558},{"mean":0.582424,"std":0.007187},{"mean":0.628182,"std":0.004919},{"mean":0.678424,"std":0.005277},{"mean":0.724,"std":0.004166},{"mean":0.775273,"std":0.002278},{"mean":0.843939,"std":0.001766},{"mean":0.916242,"std":0.005327},{"mean":1.0,"std":0.0}],"selective_risk":[{"mean":0.002956,"std":0.00256},{"mean":0.006561,"std":9.3e-05},{"mean":0.013735,"std":0.003126},{"mean":0.018184,"std":0.003271},{"mean":0.019153,"std":0.002875},{"mean":0.021559,"std":0.002007},{"mean":0.024254,"std":0.000847},{"mean":0.028651,"std":0.001714},{"mean":0.0319,"std":0.002094},{"mean":0.036162,"std":0.001314},{"mean":0.039798,"std":0.000776},{"mean":0.047997,"std":0.001434},{"mean":0.058688,"std":0.002408},{"mean":0.068596,"std":0.000469},{"mean":0.082452,"std":0.001414},{"mean":0.097859,"std":0.001494},{"mean":0.118043,"std":0.003411},{"mean":0.150165,"std":0.003782},{"mean":0.187901,"std":0.005713},{"mean":0.235455,"std":0.003933}],"accuracy_8":[{"mean":0.823091,"std":0.000182},{"mean":0.826,"std":0.000909},{"mean":0.828848,"std":0.000639},{"mean":0.831455,"std":0.000364},{"mean":0.834848,"std":0.000378},{"mean":0.83897,"std":0.00129},{"mean":0.843152,"std":0.002068},{"mean":0.846485,"std":0.001689},{"mean":0.849939,"std":0.001211},{"mean":0.853879,"std":0.000897},{"mean":0.858364,"std":0.0},{"mean":0.860424,"std":0.001183},{"mean":0.860606,"std":0.001728},{"mean":0.859758,"std":0.002169},{"mean":0.856848,"std":0.001821},{"mean":0.851515,"std":0.002548},{"mean":0.842848,"std":0.004261},{"mean":0.826061,"std":0.003824},{"mean":0.80097,"std":0.00537},{"mean":0.764545,"std":0.003933}],"high_conf_oos_misroute_rate":[{"mean":0.000667,"std":0.000577},{"mean":0.002667,"std":0.000577},{"mean":0.006667,"std":0.001528},{"mean":0.011333,"std":0.001528},{"mean":0.015333,"std":0.001528},{"mean":0.021333,"std":0.002082},{"mean":0.028667,"std":0.002082},{"mean":0.039667,"std":0.002887},{"mean":0.049667,"std":0.002309},{"mean":0.062333,"std":0.004041},{"mean":0.077,"std":0.003},{"mean":0.104,"std":0.00781},{"mean":0.141333,"std":0.008505},{"mean":0.176,"std":0.003606},{"mean":0.224667,"std":0.005033},{"mean":0.278,"std":0.009165},{"mean":0.350333,"std":0.004041},{"mean":0.486,"std":0.013077},{"mean":0.671333,"std":0.023692},{"mean":0.938333,"std":0.018583}],"oos_misroute_rate":[{"mean":0.432667,"std":0.000577},{"mean":0.433,"std":0.0},{"mean":0.434333,"std":0.000577},{"mean":0.436667,"std":0.001155},{"mean":0.439667,"std":0.001528},{"mean":0.442333,"std":0.002082},{"mean":0.446667,"std":0.002887},{"mean":0.453,"std":0.001732},{"mean":0.460333,"std":0.003512},{"mean":0.466667,"std":0.003786},{"mean":0.474667,"std":0.002517},{"mean":0.489667,"std":0.005859},{"mean":0.509667,"std":0.005508},{"mean":0.528333,"std":0.004619},{"mean":0.554,"std":0.002646},{"mean":0.579667,"std":0.007638},{"mean":0.615667,"std":0.006658},{"mean":0.684667,"std":0.012702},{"mean":0.786,"std":0.017692},{"mean":0.938333,"std":0.018583}],"llm_call_rate":[{"mean":0.958,"std":0.001734},{"mean":0.916848,"std":0.001169},{"mean":0.872242,"std":0.006531},{"mean":0.826,"std":0.006158},{"mean":0.784364,"std":0.004639},{"mean":0.744,"std":0.004287},{"mean":0.702667,"std":0.002482},{"mean":0.659515,"std":0.002723},{"mean":0.612303,"std":0.004832},{"mean":0.570848,"std":0.007043},{"mean":0.520242,"std":0.007876},{"mean":0.468303,"std":0.005558},{"mean":0.417576,"std":0.007187},{"mean":0.371818,"std":0.004919},{"mean":0.321576,"std":0.005277},{"mean":0.276,"std":0.004166},{"mean":0.224727,"std":0.002278},{"mean":0.156061,"std":0.001766},{"mean":0.083758,"std":0.005327},{"mean":0.0,"std":0.0}]}}},"summed":{"margin":{"risk_coverage":[{"mean":0.007273,"std":0.003636},{"mean":0.004242,"std":0.002777},{"mean":0.009697,"std":0.002099},{"mean":0.010909,"std":0.002727},{"mean":0.014788,"std":0.003279},{"mean":0.017172,"std":0.003653},{"mean":0.019221,"std":0.0018},{"mean":0.024242,"std":0.002922},{"mean":0.029899,"std":0.000404},{"mean":0.034545,"std":0.00126},{"mean":0.041212,"std":0.001562},{"mean":0.049394,"std":0.001515},{"mean":0.061911,"std":0.001435},{"mean":0.076797,"std":0.001522},{"mean":0.097778,"std":0.003031},{"mean":0.122424,"std":0.003086},{"mean":0.150018,"std":0.003005},{"mean":0.180471,"std":0.0044},{"mean":0.212823,"std":0.005103},{"mean":0.243333,"std":0.003772}],"operating":{"tau":[{"mean":13.984111,"std":0.074044},{"mean":12.026864,"std":0.12496},{"mean":10.756923,"std":0.146607},{"mean":9.752848,"std":0.150806},{"mean":8.892921,"std":0.150834},{"mean":8.131847,"std":0.108867},{"mean":7.469945,"std":0.085271},{"mean":6.792286,"std":0.052884},{"mean":6.155945,"std":0.029536},{"mean":5.520859,"std":0.039495},{"mean":4.898309,"std":0.035296},{"mean":4.324751,"std":0.028207},{"mean":3.685692,"std":0.054537},{"mean":3.06034,"std":0.036915},{"mean":2.526878,"std":0.043684},{"mean":1.890073,"std":0.012256},{"mean":1.307265,"std":0.014074},{"mean":0.784486,"std":0.005397},{"mean":0.313498,"std":0.030447},{"mean":0.001515,"std":0.001438}],"coverage":[{"mean":0.042727,"std":0.001273},{"mean":0.086364,"std":0.001845},{"mean":0.126788,"std":0.001111},{"mean":0.168182,"std":0.00365},{"mean":0.211394,"std":0.003772},{"mean":0.254182,"std":0.003406},{"mean":0.296788,"std":0.005092},{"mean":0.341273,"std":0.005781},{"mean":0.388909,"std":0.007818},{"mean":0.433879,"std":0.003611},{"mean":0.482,"std":0.00629},{"mean":0.525939,"std":0.006948},{"mean":0.577636,"std":0.005464},{"mean":0.625939,"std":0.005095},{"mean":0.671091,"std":0.001273},{"mean":0.72703,"std":0.00444},{"mean":0.785818,"std":0.004098},{"mean":0.849636,"std":0.002065},{"mean":0.920667,"std":0.006511},{"mean":0.999576,"std":0.000584}],"selective_risk":[{"mean":0.008557,"std":0.004321},{"mean":0.004193,"std":0.002031},{"mean":0.005266,"std":0.001697},{"mean":0.009715,"std":0.002031},{"mean":0.012025,"std":0.002435},{"mean":0.014808,"std":0.003072},{"mean":0.016768,"std":0.003227},{"mean":0.018665,"std":0.001704},{"mean":0.023095,"std":0.002336},{"mean":0.028498,"std":0.001177},{"mean":0.033199,"std":0.000433},{"mean":0.038377,"std":0.000506},{"mean":0.045228,"std":0.001292},{"mean":0.055768,"std":0.000694},{"mean":0.067645,"std":0.002397},{"mean":0.087199,"std":0.001753},{"mean":0.114914,"std":0.001674},{"mean":0.149861,"std":0.004504},{"mean":0.19344,"std":0.007311},{"mean":0.243134,"std":0.003676}],"accuracy_8":[{"mean":0.822242,"std":0.000378},{"mean":0.824848,"std":0.000555},{"mean":0.827273,"std":0.001135},{"mean":0.830606,"std":0.000735},{"mean":0.833758,"std":0.000933},{"mean":0.837273,"std":0.000833},{"mean":0.841939,"std":0.001994},{"mean":0.846303,"std":0.00147},{"mean":0.850303,"std":0.002027},{"mean":0.852848,"std":0.002003},{"mean":0.856364,"std":0.000962},{"mean":0.859576,"std":0.000915},{"mean":0.862727,"std":0.000793},{"mean":0.862848,"std":0.000555},{"mean":0.861455,"std":0.001924},{"mean":0.856848,"std":0.002624},{"mean":0.846545,"std":0.003154},{"mean":0.82897,"std":0.004846},{"mean":0.798848,"std":0.005144},{"mean":0.756848,"std":0.003661}],"high_conf_oos_misroute_rate":[{"mean":0.002,"std":0.001},{"mean":0.002,"std":0.001},{"mean":0.003333,"std":0.000577},{"mean":0.006667,"std":0.001528},{"mean":0.010667,"std":0.003055},{"mean":0.015333,"std":0.003055},{"mean":0.020333,"std":0.003786},{"mean":0.026,"std":0.001},{"mean":0.038667,"std":0.001528},{"mean":0.054,"std":0.002646},{"mean":0.070333,"std":0.002082},{"mean":0.085667,"std":0.003215},{"mean":0.113,"std":0.002646},{"mean":0.150667,"std":0.002082},{"mean":0.196667,"std":0.011846},{"mean":0.273667,"std":0.008083},{"mean":0.388,"std":0.002},{"mean":0.536667,"std":0.015044},{"mean":0.733333,"std":0.031565},{"mean":0.980667,"std":0.01106}],"oos_misroute_rate":[{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.435333,"std":0.001155},{"mean":0.437667,"std":0.002082},{"mean":0.439667,"std":0.001528},{"mean":0.442,"std":0.001732},{"mean":0.445333,"std":0.001155},{"mean":0.451667,"std":0.001528},{"mean":0.461333,"std":0.000577},{"mean":0.467667,"std":0.002887},{"mean":0.475667,"std":0.003055},{"mean":0.491,"std":0.004359},{"mean":0.509667,"std":0.004509},{"mean":0.536,"std":0.011358},{"mean":0.574333,"std":0.009504},{"mean":0.636333,"std":0.005686},{"mean":0.715333,"std":0.01124},{"mean":0.828,"std":0.020421},{"mean":0.981,"std":0.011136}],"llm_call_rate":[{"mean":0.957273,"std":0.001273},{"mean":0.913636,"std":0.001845},{"mean":0.873212,"std":0.001111},{"mean":0.831818,"std":0.00365},{"mean":0.788606,"std":0.003772},{"mean":0.745818,"std":0.003406},{"mean":0.703212,"std":0.005092},{"mean":0.658727,"std":0.005781},{"mean":0.611091,"std":0.007818},{"mean":0.566121,"std":0.003611},{"mean":0.518,"std":0.00629},{"mean":0.474061,"std":0.006948},{"mean":0.422364,"std":0.005464},{"mean":0.374061,"std":0.005095},{"mean":0.328909,"std":0.001273},{"mean":0.27297,"std":0.00444},{"mean":0.214182,"std":0.004098},{"mean":0.150364,"std":0.002065},{"mean":0.079333,"std":0.006511},{"mean":0.000424,"std":0.000584}]}},"msp_t":{"risk_coverage":[{"mean":0.007273,"std":0.003636},{"mean":0.004848,"std":0.002777},{"mean":0.008081,"std":0.001852},{"mean":0.013636,"std":0.004166},{"mean":0.016,"std":0.002909},{"mean":0.018384,"std":0.00311},{"mean":0.020606,"std":0.002954},{"mean":0.024242,"std":0.002099},{"mean":0.030707,"std":0.001069},{"mean":0.036848,"std":0.002099},{"mean":0.041873,"std":0.00101},{"mean":0.049697,"std":0.000802},{"mean":0.063124,"std":0.002816},{"mean":0.077662,"std":0.003192},{"mean":0.09802,"std":0.003397},{"mean":0.123485,"std":0.003193},{"mean":0.150303,"std":0.004509},{"mean":0.179125,"std":0.003241},{"mean":0.210271,"std":0.004061},{"mean":0.243333,"std":0.003772}],"operating":{"tau":[{"mean":0.999998,"std":0.0},{"mean":0.999985,"std":3e-06},{"mean":0.999945,"std":1e-05},{"mean":0.999859,"std":2.1e-05},{"mean":0.999666,"std":3.9e-05},{"mean":0.999299,"std":6.9e-05},{"mean":0.998606,"std":0.000155},{"mean":0.997185,"std":0.000242},{"mean":0.994758,"std":0.000298},{"mean":0.990533,"std":0.000562},{"mean":0.982147,"std":0.000432},{"mean":0.968312,"std":0.001245},{"mean":0.944425,"std":0.002077},{"mean":0.902893,"std":0.002603},{"mean":0.837956,"std":0.003718},{"mean":0.748036,"std":0.008353},{"mean":0.623656,"std":0.012973},{"mean":0.475874,"std":0.010479},{"mean":0.334922,"std":0.002101},{"mean":0.185285,"std":0.00748}],"coverage":[{"mean":0.04297,"std":0.001994},{"mean":0.085939,"std":0.002759},{"mean":0.128667,"std":0.001001},{"mean":0.168667,"std":0.005675},{"mean":0.212606,"std":0.001747},{"mean":0.253273,"std":0.003677},{"mean":0.297515,"std":0.00147},{"mean":0.343636,"std":0.004643},{"mean":0.389152,"std":0.005534},{"mean":0.432848,"std":0.004246},{"mean":0.48103,"std":0.004967},{"mean":0.528303,"std":0.005937},{"mean":0.575091,"std":0.007875},{"mean":0.623636,"std":0.007825},{"mean":0.671818,"std":0.004643},{"mean":0.720909,"std":0.005645},{"mean":0.77897,"std":0.005611},{"mean":0.845394,"std":0.0029},{"mean":0.915636,"std":0.002378},{"mean":0.999152,"std":0.000735}],"selective_risk":[{"mean":0.005726,"std":0.002774},{"mean":0.004888,"std":0.003061},{"mean":0.005659,"std":0.00145},{"mean":0.010032,"std":0.003116},{"mean":0.014837,"std":0.002725},{"mean":0.017239,"std":0.002587},{"mean":0.018138,"std":0.002559},{"mean":0.02013,"std":0.002708},{"mean":0.023376,"std":0.001557},{"mean":0.029402,"std":0.00069},{"mean":0.034409,"std":0.002558},{"mean":0.040387,"std":0.001589},{"mean":0.046045,"std":0.000986},{"mean":0.055483,"std":0.001326},{"mean":0.069289,"std":0.001805},{"mean":0.085499,"std":0.000503},{"mean":0.112034,"std":0.002346},{"mean":0.147467,"std":0.003751},{"mean":0.188844,"std":0.002363},{"mean":0.242691,"std":0.003523}],"accuracy_8":[{"mean":0.822364,"std":0.00063},{"mean":0.82503,"std":0.000915},{"mean":0.828424,"std":0.001315},{"mean":0.832,"std":0.001135},{"mean":0.835091,"std":0.000793},{"mean":0.838182,"std":0.001311},{"mean":0.843273,"std":0.00126},{"mean":0.847455,"std":0.000793},{"mean":0.85103,"std":0.000897},{"mean":0.853636,"std":0.002271},{"mean":0.856303,"std":0.001315},{"mean":0.859273,"std":0.000656},{"mean":0.862182,"std":0.000315},{"mean":0.862788,"std":0.000688},{"mean":0.861212,"std":0.002107},{"mean":0.856545,"std":0.002065},{"mean":0.847152,"std":0.003},{"mean":0.829394,"std":0.003951},{"mean":0.802485,"std":0.002373},{"mean":0.757394,"std":0.003546}],"high_conf_oos_misroute_rate":[{"mean":0.001333,"std":0.000577},{"mean":0.002,"std":0.001},{"mean":0.003667,"std":0.000577},{"mean":0.006667,"std":0.002517},{"mean":0.012,"std":0.002646},{"mean":0.017667,"std":0.001528},{"mean":0.021667,"std":0.002517},{"mean":0.029,"std":0.002646},{"mean":0.038,"std":0.001732},{"mean":0.053667,"std":0.002082},{"mean":0.068667,"std":0.004041},{"mean":0.088667,"std":0.002887},{"mean":0.11,"std":0.003606},{"mean":0.146333,"std":0.005132},{"mean":0.197333,"std":0.007506},{"mean":0.257333,"std":0.002887},{"mean":0.353333,"std":0.013051},{"mean":0.498,"std":0.009539},{"mean":0.692,"std":0.003},{"mean":0.978667,"std":0.010504}],"oos_misroute_rate":[{"mean":0.433333,"std":0.000577},{"mean":0.434,"std":0.001},{"mean":0.434,"std":0.001},{"mean":0.435667,"std":0.001528},{"mean":0.437,"std":0.001732},{"mean":0.440667,"std":0.001528},{"mean":0.442333,"std":0.002082},{"mean":0.446,"std":0.002646},{"mean":0.450333,"std":0.002082},{"mean":0.458333,"std":0.002082},{"mean":0.467667,"std":0.002082},{"mean":0.478,"std":0.002646},{"mean":0.487333,"std":0.003786},{"mean":0.506667,"std":0.005686},{"mean":0.529667,"std":0.008737},{"mean":0.561667,"std":0.002309},{"mean":0.610333,"std":0.008963},{"mean":0.687333,"std":0.008505},{"mean":0.795333,"std":0.003215},{"mean":0.979333,"std":0.011015}],"llm_call_rate":[{"mean":0.95703,"std":0.001994},{"mean":0.914061,"std":0.002759},{"mean":0.871333,"std":0.001001},{"mean":0.831333,"std":0.005675},{"mean":0.787394,"std":0.001747},{"mean":0.746727,"std":0.003677},{"mean":0.702485,"std":0.00147},{"mean":0.656364,"std":0.004643},{"mean":0.610848,"std":0.005534},{"mean":0.567152,"std":0.004246},{"mean":0.51897,"std":0.004967},{"mean":0.471697,"std":0.005937},{"mean":0.424909,"std":0.007875},{"mean":0.376364,"std":0.007825},{"mean":0.328182,"std":0.004643},{"mean":0.279091,"std":0.005645},{"mean":0.22103,"std":0.005611},{"mean":0.154606,"std":0.0029},{"mean":0.084364,"std":0.002378},{"mean":0.000848,"std":0.000735}]}}}}}}} diff --git a/results/analysis/haiku.json b/results/analysis/haiku.json new file mode 100644 index 0000000..c191c57 --- /dev/null +++ b/results/analysis/haiku.json @@ -0,0 +1,112 @@ +{ + "format_version":1, + "haiku_predictions_sha256":"cac660f25f06bad317573c14aa55c0d7d977619c3cd6471c97654d94ec3432a0", + "splits":{ + "validation":{ + "new_parser":{ + "accuracy_8":0.875161, + "oos":{ + "tp":68, + "gold_oos":100, + "predicted_oos":211, + "recall":0.68, + "precision":0.322275, + "f1":0.437299 + }, + "oos_recall_wilson95":[ + 0.583372, + 0.76331 + ], + "oos_misroute_rate":0.32, + "n":3100 + }, + "legacy_parser":{ + "accuracy_8":0.875161, + "oos":{ + "tp":68, + "gold_oos":100, + "predicted_oos":211, + "recall":0.68, + "precision":0.322275, + "f1":0.437299 + }, + "oos_recall_wilson95":[ + 0.583372, + 0.76331 + ], + "oos_misroute_rate":0.32, + "n":3100 + }, + "parsers_disagree_rows":0, + "parsers_disagree_examples":[], + "legacy_order_dependent_rows":0, + "parse_failed_rows":0, + "accuracy_8_parse_failed_as_wrong":0.875161, + "raw_reply_counts":{ + "auto_agent":296, + "device_agent":304, + "finance_agent":797, + "kitchen_agent":243, + "meta_agent":431, + "oos":211, + "productivity_agent":548, + "travel_agent":270 + }, + "cost_usd_total":1.146542, + "cost_usd_per_1k_queries":0.369852 + }, + "test":{ + "new_parser":{ + "accuracy_8":0.820727, + "oos":{ + "tp":568, + "gold_oos":1000, + "predicted_oos":759, + "recall":0.568, + "precision":0.748353, + "f1":0.645821 + }, + "oos_recall_wilson95":[ + 0.537095, + 0.598384 + ], + "oos_misroute_rate":0.432, + "n":5500 + }, + "legacy_parser":{ + "accuracy_8":0.820727, + "oos":{ + "tp":568, + "gold_oos":1000, + "predicted_oos":759, + "recall":0.568, + "precision":0.748353, + "f1":0.645821 + }, + "oos_recall_wilson95":[ + 0.537095, + 0.598384 + ], + "oos_misroute_rate":0.432, + "n":5500 + }, + "parsers_disagree_rows":0, + "parsers_disagree_examples":[], + "legacy_order_dependent_rows":0, + "parse_failed_rows":0, + "accuracy_8_parse_failed_as_wrong":0.820727, + "raw_reply_counts":{ + "auto_agent":493, + "device_agent":532, + "finance_agent":1268, + "kitchen_agent":425, + "meta_agent":686, + "oos":759, + "productivity_agent":915, + "travel_agent":422 + }, + "cost_usd_total":2.032204, + "cost_usd_per_1k_queries":0.369492 + } + } +} diff --git a/results/analysis/summary.json b/results/analysis/summary.json new file mode 100644 index 0000000..04164ab --- /dev/null +++ b/results/analysis/summary.json @@ -0,0 +1,64051 @@ +{ + "format_version":1, + "inputs":{ + "archives":75, + "logits_manifest_sha256":"2635659e9e95348b7b0de34d73f91fc3957e228b33491e6a41708265260b08db", + "curve_index_sha256":{ + "bert.json":"35b46d514e6b47728501678dd6e029fb6a670e9b57d4514d6a2e73827aa987a5", + "modernbert.json":"515c153596bfb0115b39b2a6166999135014ef2ca68361a35764aa1866e9ffc4", + "oos-ablation.json":"4d8bb6aa6e4cf64d17a15e8b0453825b946da948bcd6cddbe256afa7099e081b", + "baselines.json":"f6a483c306e2ea10c35c3515e39428e5654b8d1e2303fbadd5f0b56d4b7177d3" + }, + "haiku_predictions_sha256":"cac660f25f06bad317573c14aa55c0d7d977619c3cd6471c97654d94ec3432a0", + "llm_rows":8600 + }, + "protocol":{ + "aggregations":[ + "argmax", + "summed" + ], + "aggregation_choice":"higher validation 8-way accuracy; ties go to argmax", + "signals":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "signals_by_kind":{ + "encoder":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "tfidf":[ + "margin", + "msp_t" + ], + "majority":[] + }, + "target_risks":[ + 0.02, + 0.05 + ], + "threshold_choice":"largest validation coverage whose one-sided 95% Wilson upper bound on selective risk (8-way routing error among kept rows) is <= the target; no such threshold means every query goes to the LLM", + "one_sided_z":1.644854, + "signal_choice":"largest validation coverage at the target, then lower validation AURC", + "oracle":"defers exactly the queries the small model gets wrong (8-way)", + "error_recovery_rate":"small-model errors deferred and fixed by Haiku / small errors", + "recoverable_caught":"errors deferred and fixed by Haiku / errors Haiku would fix", + "high_conf_oos_misroute_rate":"gold OOS kept by the small model (confidence >= tau) and routed to an in-scope agent / gold OOS", + "coverage_grid":[ + 0.05, + 0.1, + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.55, + 0.6, + 0.65, + 0.7, + 0.75, + 0.8, + 0.85, + 0.9, + 0.95, + 1.0 + ], + "operating_curve_tau":"the validation score at each coverage of the grid", + "std":"sample standard deviation (ddof=1) over seeds 42, 43, 44; no other randomness", + "oos_precision_when_none_predicted":0.0 + }, + "llm_only_test":{ + "accuracy_8":0.820727, + "oos":{ + "tp":568, + "gold_oos":1000, + "predicted_oos":759, + "recall":0.568, + "precision":0.748353, + "f1":0.645821 + }, + "oos_recall_wilson95":[ + 0.537095, + 0.598384 + ], + "oos_misroute_rate":0.432, + "n":5500, + "coverage":0.0, + "llm_call_rate":1.0, + "llm_cost_usd_per_1k":0.369492, + "high_conf_oos_misroute_rate":null, + "note":"no confidence score, so no high-confidence misroute; see oos_misroute_rate" + }, + "groups":{ + "bert/k1":{ + "model":"bert", + "k":1, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "bert-base-uncased-k1-seed42", + "bert-base-uncased-k1-seed43", + "bert-base-uncased-k1-seed44" + ], + "logits_sha256":[ + "33464fd83ba26e6de38f2df4a347659dc396c57d98ce9fca16b78ab31101da2d", + "de95f070e64b443b4761f3ca786da624343fe8a5dc20e3f341220333e586cff3", + "db5595875d86d5b3476689bb5932f72bdb7651effb3686d8636cb308e8279881" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":0.468018, + "std":0.031603, + "values":[ + 0.503419, + 0.457986, + 0.44265 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.227692, + "std":0.015501, + "values":[ + 0.212948, + 0.243853, + 0.226276 + ] + }, + "ece_8_summed":{ + "mean":0.154649, + "std":0.014172, + "values":[ + 0.168764, + 0.154761, + 0.140421 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.030893, + "std":0.011144, + "values":[ + 0.04302, + 0.028557, + 0.021102 + ] + }, + "ece_8_summed":{ + "mean":0.103411, + "std":0.02005, + "values":[ + 0.12568, + 0.097759, + 0.086793 + ] + } + } + } + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.605591, + "std":0.00985, + "values":[ + 0.595484, + 0.615161, + 0.606129 + ] + }, + "summed":{ + "mean":0.638065, + "std":0.014573, + "values":[ + 0.621613, + 0.649355, + 0.643226 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.605591, + "std":0.00985 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.466121, + "std":0.012365 + }, + "accuracy_8":{ + "mean":0.533879, + "std":0.012365 + }, + "oos_recall":{ + "mean":0.172333, + "std":0.086743 + }, + "oos_recall_wilson95":[ + { + "mean":0.151041, + "std":0.080951 + }, + { + "mean":0.196133, + "std":0.091873 + } + ], + "oos_precision":{ + "mean":0.421341, + "std":0.127195 + }, + "oos_f1":{ + "mean":0.242421, + "std":0.109774 + }, + "oos_misroute_rate":{ + "mean":0.827667, + "std":0.086743 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.827667, + "std":0.086743 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.533879, + "std":0.012365 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.878788, + "std":0.007458 + }, + "oos_recall":{ + "mean":0.618333, + "std":0.035642 + }, + "oos_recall_wilson95":[ + { + "mean":0.587882, + "std":0.036029 + }, + { + "mean":0.647879, + "std":0.034982 + } + ], + "oos_precision":{ + "mean":0.868399, + "std":0.02026 + }, + "oos_f1":{ + "mean":0.722192, + "std":0.031247 + }, + "oos_misroute_rate":{ + "mean":0.381667, + "std":0.035642 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.466121, + "std":0.012365 + }, + "llm_cost_usd_per_1k":{ + "mean":0.172141, + "std":0.004518 + }, + "error_recovery_rate":{ + "mean":0.73996, + "std":0.01419 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1897.0, + "std":61.147363 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.25534, + "std":0.024804 + }, + "oos_auroc":{ + "mean":0.622838, + "std":0.06622 + }, + "oos_auprc":{ + "mean":0.055272, + "std":0.018815 + } + }, + "test":{ + "aurc":{ + "mean":0.309025, + "std":0.023236 + }, + "oos_auroc":{ + "mean":0.609204, + "std":0.027812 + }, + "oos_auprc":{ + "mean":0.249495, + "std":0.010942 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.290948, + "std":0.037781 + }, + "oos_auroc":{ + "mean":0.611054, + "std":0.069299 + }, + "oos_auprc":{ + "mean":0.059045, + "std":0.023319 + } + }, + "test":{ + "aurc":{ + "mean":0.343552, + "std":0.033053 + }, + "oos_auroc":{ + "mean":0.596144, + "std":0.022553 + }, + "oos_auprc":{ + "mean":0.245533, + "std":0.00831 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.257864, + "std":0.008528 + }, + "oos_auroc":{ + "mean":0.552332, + "std":0.036129 + }, + "oos_auprc":{ + "mean":0.037216, + "std":0.005221 + } + }, + "test":{ + "aurc":{ + "mean":0.318362, + "std":0.01276 + }, + "oos_auroc":{ + "mean":0.562609, + "std":0.034167 + }, + "oos_auprc":{ + "mean":0.206477, + "std":0.01507 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.250497, + "std":0.020088 + }, + "oos_auroc":{ + "mean":0.617644, + "std":0.064318 + }, + "oos_auprc":{ + "mean":0.053554, + "std":0.016235 + } + }, + "test":{ + "aurc":{ + "mean":0.306085, + "std":0.019628 + }, + "oos_auroc":{ + "mean":0.605054, + "std":0.031305 + }, + "oos_auprc":{ + "mean":0.245368, + "std":0.014602 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "margin", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.73996, + "std":0.01419 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1897.0, + "std":61.147363 + } + } + }, + "by_signal":{ + "msp":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.73996, + "std":0.01419 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1897.0, + "std":61.147363 + } + } + }, + "entropy":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.73996, + "std":0.01419 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1897.0, + "std":61.147363 + } + } + }, + "margin":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.73996, + "std":0.01419 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1897.0, + "std":61.147363 + } + } + }, + "msp_t":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.73996, + "std":0.01419 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1897.0, + "std":61.147363 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "margin", + "msp", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.954367, + "std":2.481873, + "n":2 + }, + "feasible":[ + false, + true, + true + ], + "validation_coverage":{ + "mean":0.029785, + "std":0.02616 + }, + "validation_selective_risk":{ + "mean":0.017868, + "std":0.002642, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.047778, + "std":0.000819, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.027333, + "std":0.025075 + }, + "selective_risk":{ + "mean":0.066523, + "std":0.015511, + "n":2 + }, + "accuracy_8":{ + "mean":0.821394, + "std":0.000584 + }, + "oos_recall":{ + "mean":0.568333, + "std":0.003512 + }, + "oos_recall_wilson95":[ + { + "mean":0.53743, + "std":0.003528 + }, + { + "mean":0.598713, + "std":0.003468 + } + ], + "oos_precision":{ + "mean":0.748131, + "std":0.002644 + }, + "oos_f1":{ + "mean":0.645953, + "std":0.003245 + }, + "oos_misroute_rate":{ + "mean":0.431667, + "std":0.003512 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.003333, + "std":0.002887 + }, + "llm_call_rate":{ + "mean":0.972667, + "std":0.025075 + }, + "llm_cost_usd_per_1k":{ + "mean":0.359401, + "std":0.009261 + }, + "error_recovery_rate":{ + "mean":0.737265, + "std":0.016708 + }, + "recoverable_caught":{ + "mean":0.996316, + "std":0.004238 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1890.0, + "std":61.024585 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.177369, + "std":0.031182, + "n":2 + }, + "feasible":[ + false, + true, + true + ], + "validation_coverage":{ + "mean":0.026237, + "std":0.022742 + }, + "validation_selective_risk":{ + "mean":0.016403, + "std":0.00057, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.048359, + "std":0.001641, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.022667, + "std":0.019671 + }, + "selective_risk":{ + "mean":0.066438, + "std":0.015389, + "n":2 + }, + "accuracy_8":{ + "mean":0.821212, + "std":0.000458 + }, + "oos_recall":{ + "mean":0.568667, + "std":0.003055 + }, + "oos_recall_wilson95":[ + { + "mean":0.537765, + "std":0.00307 + }, + { + "mean":0.599043, + "std":0.003017 + } + ], + "oos_precision":{ + "mean":0.747915, + "std":0.002984 + }, + "oos_f1":{ + "mean":0.646089, + "std":0.003049 + }, + "oos_misroute_rate":{ + "mean":0.431333, + "std":0.003055 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002667, + "std":0.002517 + }, + "llm_call_rate":{ + "mean":0.977333, + "std":0.019671 + }, + "llm_cost_usd_per_1k":{ + "mean":0.361122, + "std":0.007264 + }, + "error_recovery_rate":{ + "mean":0.737645, + "std":0.016304 + }, + "recoverable_caught":{ + "mean":0.996836, + "std":0.003399 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1891.0, + "std":61.489837 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.075108, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.006129, + "std":0.010616 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.045315, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.004485, + "std":0.007768 + }, + "selective_risk":{ + "mean":0.013514, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820788, + "std":0.000105 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748682, + "std":0.00057 + }, + "oos_f1":{ + "mean":0.645944, + "std":0.000212 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.000333, + "std":0.000577 + }, + "llm_call_rate":{ + "mean":0.995515, + "std":0.007768 + }, + "llm_cost_usd_per_1k":{ + "mean":0.367839, + "std":0.002863 + }, + "error_recovery_rate":{ + "mean":0.739827, + "std":0.0143 + }, + "recoverable_caught":{ + "mean":0.999818, + "std":0.000316 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1896.666667, + "std":61.719797 + } + } + }, + "margin":{ + "tau":{ + "mean":3.709316, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + false, + true + ], + "validation_coverage":{ + "mean":0.016344, + "std":0.028309 + }, + "validation_selective_risk":{ + "mean":0.019737, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.048357, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.016424, + "std":0.028448 + }, + "selective_risk":{ + "mean":0.077491, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.821091, + "std":0.00063 + }, + "oos_recall":{ + "mean":0.567, + "std":0.001732 + }, + "oos_recall_wilson95":[ + { + "mean":0.536091, + "std":0.00174 + }, + { + "mean":0.597397, + "std":0.001711 + } + ], + "oos_precision":{ + "mean":0.747363, + "std":0.001715 + }, + "oos_f1":{ + "mean":0.644806, + "std":0.001758 + }, + "oos_misroute_rate":{ + "mean":0.433, + "std":0.001732 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.001667, + "std":0.002887 + }, + "llm_call_rate":{ + "mean":0.983576, + "std":0.028448 + }, + "llm_cost_usd_per_1k":{ + "mean":0.363425, + "std":0.010508 + }, + "error_recovery_rate":{ + "mean":0.737933, + "std":0.016324 + }, + "recoverable_caught":{ + "mean":0.997228, + "std":0.004801 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1891.666667, + "std":58.243741 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.888534, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + false, + true + ], + "validation_coverage":{ + "mean":0.010215, + "std":0.017693 + }, + "validation_selective_risk":{ + "mean":0.010526, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.045809, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.009455, + "std":0.016376 + }, + "selective_risk":{ + "mean":0.070513, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820909, + "std":0.000315 + }, + "oos_recall":{ + "mean":0.567333, + "std":0.001155 + }, + "oos_recall_wilson95":[ + { + "mean":0.536425, + "std":0.00116 + }, + { + "mean":0.597726, + "std":0.001141 + } + ], + "oos_precision":{ + "mean":0.747475, + "std":0.001521 + }, + "oos_f1":{ + "mean":0.645063, + "std":0.001313 + }, + "oos_misroute_rate":{ + "mean":0.432667, + "std":0.001155 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.001, + "std":0.001732 + }, + "llm_call_rate":{ + "mean":0.990545, + "std":0.016376 + }, + "llm_cost_usd_per_1k":{ + "mean":0.365996, + "std":0.006054 + }, + "error_recovery_rate":{ + "mean":0.73882, + "std":0.015328 + }, + "recoverable_caught":{ + "mean":0.998441, + "std":0.002701 + }, + "small_errors":{ + "mean":2563.666667, + "std":68.009803 + }, + "recoverable_errors":{ + "mean":1897.0, + "std":61.147363 + }, + "recovered_errors":{ + "mean":1894.0, + "std":59.354865 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.638065, + "std":0.014573 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.450727, + "std":0.013235 + }, + "accuracy_8":{ + "mean":0.549273, + "std":0.013235 + }, + "oos_recall":{ + "mean":0.099, + "std":0.061733 + }, + "oos_recall_wilson95":[ + { + "mean":0.082982, + "std":0.055264 + }, + { + "mean":0.118087, + "std":0.06773 + } + ], + "oos_precision":{ + "mean":0.435564, + "std":0.164565 + }, + "oos_f1":{ + "mean":0.159567, + "std":0.095152 + }, + "oos_misroute_rate":{ + "mean":0.901, + "std":0.061733 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.901, + "std":0.061733 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.549273, + "std":0.013235 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.874242, + "std":0.005079 + }, + "oos_recall":{ + "mean":0.598667, + "std":0.023587 + }, + "oos_recall_wilson95":[ + { + "mean":0.567988, + "std":0.023784 + }, + { + "mean":0.62859, + "std":0.023209 + } + ], + "oos_precision":{ + "mean":0.872051, + "std":0.013942 + }, + "oos_f1":{ + "mean":0.709886, + "std":0.021217 + }, + "oos_misroute_rate":{ + "mean":0.401333, + "std":0.023587 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.450727, + "std":0.013235 + }, + "llm_cost_usd_per_1k":{ + "mean":0.166446, + "std":0.004844 + }, + "error_recovery_rate":{ + "mean":0.720832, + "std":0.01365 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1787.333333, + "std":77.500538 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.230525, + "std":0.005079 + }, + "oos_auroc":{ + "mean":0.650803, + "std":0.097357 + }, + "oos_auprc":{ + "mean":0.055299, + "std":0.024601 + } + }, + "test":{ + "aurc":{ + "mean":0.318472, + "std":0.002753 + }, + "oos_auroc":{ + "mean":0.573087, + "std":0.064199 + }, + "oos_auprc":{ + "mean":0.214768, + "std":0.046802 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.225938, + "std":0.017797 + }, + "oos_auroc":{ + "mean":0.732789, + "std":0.062741 + }, + "oos_auprc":{ + "mean":0.091195, + "std":0.021984 + } + }, + "test":{ + "aurc":{ + "mean":0.305952, + "std":0.016071 + }, + "oos_auroc":{ + "mean":0.669386, + "std":0.02434 + }, + "oos_auprc":{ + "mean":0.316954, + "std":0.029279 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.179271, + "std":0.014356 + }, + "oos_auroc":{ + "mean":0.651744, + "std":0.05103 + }, + "oos_auprc":{ + "mean":0.058596, + "std":0.017885 + } + }, + "test":{ + "aurc":{ + "mean":0.252626, + "std":0.011671 + }, + "oos_auroc":{ + "mean":0.620342, + "std":0.030324 + }, + "oos_auprc":{ + "mean":0.237372, + "std":0.016182 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.180977, + "std":0.013485 + }, + "oos_auroc":{ + "mean":0.692213, + "std":0.05174 + }, + "oos_auprc":{ + "mean":0.084555, + "std":0.025366 + } + }, + "test":{ + "aurc":{ + "mean":0.251916, + "std":0.012028 + }, + "oos_auroc":{ + "mean":0.651049, + "std":0.033847 + }, + "oos_auprc":{ + "mean":0.282136, + "std":0.033454 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "msp", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":0.401006, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.022581, + "std":0.039111 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.01272, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.015636, + "std":0.027083 + }, + "selective_risk":{ + "mean":0.023256, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820667, + "std":0.000105 + }, + "oos_recall":{ + "mean":0.567667, + "std":0.000577 + }, + "oos_recall_wilson95":[ + { + "mean":0.53676, + "std":0.00058 + }, + { + "mean":0.598055, + "std":0.00057 + } + ], + "oos_precision":{ + "mean":0.748242, + "std":0.000192 + }, + "oos_f1":{ + "mean":0.645565, + "std":0.000445 + }, + "oos_misroute_rate":{ + "mean":0.432333, + "std":0.000577 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002, + "std":0.003464 + }, + "llm_call_rate":{ + "mean":0.984364, + "std":0.027083 + }, + "llm_cost_usd_per_1k":{ + "mean":0.363704, + "std":0.010025 + }, + "error_recovery_rate":{ + "mean":0.720554, + "std":0.013875 + }, + "recoverable_caught":{ + "mean":0.99961, + "std":0.000675 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1786.666667, + "std":78.500531 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.401006, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.022581, + "std":0.039111 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.01272, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.015636, + "std":0.027083 + }, + "selective_risk":{ + "mean":0.023256, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820667, + "std":0.000105 + }, + "oos_recall":{ + "mean":0.567667, + "std":0.000577 + }, + "oos_recall_wilson95":[ + { + "mean":0.53676, + "std":0.00058 + }, + { + "mean":0.598055, + "std":0.00057 + } + ], + "oos_precision":{ + "mean":0.748242, + "std":0.000192 + }, + "oos_f1":{ + "mean":0.645565, + "std":0.000445 + }, + "oos_misroute_rate":{ + "mean":0.432333, + "std":0.000577 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002, + "std":0.003464 + }, + "llm_call_rate":{ + "mean":0.984364, + "std":0.027083 + }, + "llm_cost_usd_per_1k":{ + "mean":0.363704, + "std":0.010025 + }, + "error_recovery_rate":{ + "mean":0.720554, + "std":0.013875 + }, + "recoverable_caught":{ + "mean":0.99961, + "std":0.000675 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1786.666667, + "std":78.500531 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.700069, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.016237, + "std":0.028123 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.017602, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.011212, + "std":0.01942 + }, + "selective_risk":{ + "mean":0.027027, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.001333, + "std":0.002309 + }, + "llm_call_rate":{ + "mean":0.988788, + "std":0.01942 + }, + "llm_cost_usd_per_1k":{ + "mean":0.365341, + "std":0.00719 + }, + "error_recovery_rate":{ + "mean":0.720693, + "std":0.013761 + }, + "recoverable_caught":{ + "mean":0.999805, + "std":0.000338 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1787.0, + "std":78.0 + } + } + }, + "margin":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.720832, + "std":0.01365 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1787.333333, + "std":77.500538 + } + } + }, + "msp_t":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.720832, + "std":0.01365 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1787.333333, + "std":77.500538 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "msp", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":-0.340128, + "std":1.222357 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.101613, + "std":0.024446 + }, + "validation_selective_risk":{ + "mean":0.02828, + "std":0.001818 + }, + "validation_risk_upper95":{ + "mean":0.04855, + "std":0.001142 + }, + "test":{ + "coverage":{ + "mean":0.08503, + "std":0.026823 + }, + "selective_risk":{ + "mean":0.058689, + "std":0.020595 + }, + "accuracy_8":{ + "mean":0.820424, + "std":0.000525 + }, + "oos_recall":{ + "mean":0.566, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.535086, + "std":0.0 + }, + { + "mean":0.596409, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748018, + "std":0.000571 + }, + "oos_f1":{ + "mean":0.644402, + "std":0.000212 + }, + "oos_misroute_rate":{ + "mean":0.434, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.017, + "std":0.009644 + }, + "llm_call_rate":{ + "mean":0.91497, + "std":0.026823 + }, + "llm_cost_usd_per_1k":{ + "mean":0.337973, + "std":0.009941 + }, + "error_recovery_rate":{ + "mean":0.717761, + "std":0.013713 + }, + "recoverable_caught":{ + "mean":0.995739, + "std":0.001908 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1779.666667, + "std":75.566748 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.369819, + "std":0.008775 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.096129, + "std":0.033826 + }, + "validation_selective_risk":{ + "mean":0.027037, + "std":0.003962 + }, + "validation_risk_upper95":{ + "mean":0.048302, + "std":0.000718 + }, + "test":{ + "coverage":{ + "mean":0.080727, + "std":0.033978 + }, + "selective_risk":{ + "mean":0.048682, + "std":0.028614 + }, + "accuracy_8":{ + "mean":0.82103, + "std":0.000525 + }, + "oos_recall":{ + "mean":0.567, + "std":0.001732 + }, + "oos_recall_wilson95":[ + { + "mean":0.536091, + "std":0.00174 + }, + { + "mean":0.597397, + "std":0.001711 + } + ], + "oos_precision":{ + "mean":0.749341, + "std":0.002862 + }, + "oos_f1":{ + "mean":0.645541, + "std":0.002185 + }, + "oos_misroute_rate":{ + "mean":0.433, + "std":0.001732 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.015333, + "std":0.011676 + }, + "llm_call_rate":{ + "mean":0.919273, + "std":0.033978 + }, + "llm_cost_usd_per_1k":{ + "mean":0.339565, + "std":0.012587 + }, + "error_recovery_rate":{ + "mean":0.718815, + "std":0.015519 + }, + "recoverable_caught":{ + "mean":0.997168, + "std":0.003107 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1782.333333, + "std":79.688979 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.74118, + "std":0.014699, + "n":2 + }, + "feasible":[ + true, + true, + false + ], + "validation_coverage":{ + "mean":0.05086, + "std":0.044113 + }, + "validation_selective_risk":{ + "mean":0.025396, + "std":0.001139, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.048349, + "std":0.002135, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.037818, + "std":0.03277 + }, + "selective_risk":{ + "mean":0.048218, + "std":0.010377, + "n":2 + }, + "accuracy_8":{ + "mean":0.820242, + "std":0.000458 + }, + "oos_recall":{ + "mean":0.567, + "std":0.001 + }, + "oos_recall_wilson95":[ + { + "mean":0.536091, + "std":0.001005 + }, + { + "mean":0.597397, + "std":0.000988 + } + ], + "oos_precision":{ + "mean":0.74835, + "std":0.000328 + }, + "oos_f1":{ + "mean":0.645173, + "std":0.000597 + }, + "oos_misroute_rate":{ + "mean":0.433, + "std":0.001 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.005667, + "std":0.005132 + }, + "llm_call_rate":{ + "mean":0.962182, + "std":0.03277 + }, + "llm_cost_usd_per_1k":{ + "mean":0.355477, + "std":0.012143 + }, + "error_recovery_rate":{ + "mean":0.719223, + "std":0.012289 + }, + "recoverable_caught":{ + "mean":0.99779, + "std":0.002148 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1783.333333, + "std":75.566748 + } + } + }, + "margin":{ + "tau":{ + "mean":2.944128, + "std":0.165557, + "n":2 + }, + "feasible":[ + false, + true, + true + ], + "validation_coverage":{ + "mean":0.044839, + "std":0.04211 + }, + "validation_selective_risk":{ + "mean":0.023007, + "std":0.005685, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.047836, + "std":0.001808, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.039455, + "std":0.03643 + }, + "selective_risk":{ + "mean":0.049441, + "std":0.001895, + "n":2 + }, + "accuracy_8":{ + "mean":0.821939, + "std":0.00105 + }, + "oos_recall":{ + "mean":0.568333, + "std":0.002517 + }, + "oos_recall_wilson95":[ + { + "mean":0.53743, + "std":0.002528 + }, + { + "mean":0.598713, + "std":0.002486 + } + ], + "oos_precision":{ + "mean":0.749783, + "std":0.003993 + }, + "oos_f1":{ + "mean":0.646569, + "std":0.003097 + }, + "oos_misroute_rate":{ + "mean":0.431667, + "std":0.002517 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.004333, + "std":0.004509 + }, + "llm_call_rate":{ + "mean":0.960545, + "std":0.03643 + }, + "llm_cost_usd_per_1k":{ + "mean":0.354925, + "std":0.013445 + }, + "error_recovery_rate":{ + "mean":0.718391, + "std":0.01578 + }, + "recoverable_caught":{ + "mean":0.996575, + "std":0.003001 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1781.333333, + "std":81.586355 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.874498, + "std":0.014032, + "n":2 + }, + "feasible":[ + false, + true, + true + ], + "validation_coverage":{ + "mean":0.042258, + "std":0.03908 + }, + "validation_selective_risk":{ + "mean":0.022293, + "std":0.003977, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.047773, + "std":4.4e-05, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.035697, + "std":0.033064 + }, + "selective_risk":{ + "mean":0.061075, + "std":0.000291, + "n":2 + }, + "accuracy_8":{ + "mean":0.821515, + "std":0.00082 + }, + "oos_recall":{ + "mean":0.568, + "std":0.002 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.002009 + }, + { + "mean":0.598384, + "std":0.001975 + } + ], + "oos_precision":{ + "mean":0.748352, + "std":0.001649 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.001907 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.002 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.005333, + "std":0.00611 + }, + "llm_call_rate":{ + "mean":0.964303, + "std":0.033064 + }, + "llm_cost_usd_per_1k":{ + "mean":0.356309, + "std":0.012205 + }, + "error_recovery_rate":{ + "mean":0.71798, + "std":0.016123 + }, + "recoverable_caught":{ + "mean":0.995999, + "std":0.003469 + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231 + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538 + }, + "recovered_errors":{ + "mean":1780.333333, + "std":82.585309 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.450727, + "std":0.013235, + "values":[ + 0.460364, + 0.435636, + 0.456182 + ] + }, + "accuracy_8":{ + "mean":0.549273, + "std":0.013235, + "values":[ + 0.539636, + 0.564364, + 0.543818 + ] + }, + "oos_recall":{ + "mean":0.099, + "std":0.061733, + "values":[ + 0.129, + 0.14, + 0.028 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.082982, + "std":0.055264, + "values":[ + 0.109635, + 0.119868, + 0.019442 + ] + }, + { + "mean":0.118087, + "std":0.06773, + "values":[ + 0.151204, + 0.162887, + 0.04017 + ] + } + ], + "oos_precision":{ + "mean":0.435564, + "std":0.164565, + "values":[ + 0.476015, + 0.576132, + 0.254545 + ] + }, + "oos_f1":{ + "mean":0.159567, + "std":0.095152, + "values":[ + 0.20299, + 0.225261, + 0.05045 + ] + }, + "oos_misroute_rate":{ + "mean":0.901, + "std":0.061733, + "values":[ + 0.871, + 0.86, + 0.972 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.901, + "std":0.061733, + "values":[ + 0.871, + 0.86, + 0.972 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231, + "values":[ + 2532, + 2396, + 2509 + ] + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538, + "values":[ + 1865, + 1710, + 1787 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.549273, + "std":0.013235, + "values":[ + 0.539636, + 0.564364, + 0.543818 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.874242, + "std":0.005079, + "values":[ + 0.878727, + 0.875273, + 0.868727 + ] + }, + "oos_recall":{ + "mean":0.598667, + "std":0.023587, + "values":[ + 0.621, + 0.601, + 0.574 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.567988, + "std":0.023784, + "values":[ + 0.590522, + 0.570318, + 0.543125 + ] + }, + { + "mean":0.62859, + "std":0.023209, + "values":[ + 0.650552, + 0.630909, + 0.604308 + ] + } + ], + "oos_precision":{ + "mean":0.872051, + "std":0.013942, + "values":[ + 0.885877, + 0.872279, + 0.857997 + ] + }, + "oos_f1":{ + "mean":0.709886, + "std":0.021217, + "values":[ + 0.730159, + 0.711664, + 0.687837 + ] + }, + "oos_misroute_rate":{ + "mean":0.401333, + "std":0.023587, + "values":[ + 0.379, + 0.399, + 0.426 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.450727, + "std":0.013235, + "values":[ + 0.460364, + 0.435636, + 0.456182 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.166446, + "std":0.004844, + "values":[ + 0.169994, + 0.160928, + 0.168417 + ] + }, + "error_recovery_rate":{ + "mean":0.720832, + "std":0.01365, + "values":[ + 0.736572, + 0.713689, + 0.712236 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231, + "values":[ + 2532, + 2396, + 2509 + ] + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538, + "values":[ + 1865, + 1710, + 1787 + ] + }, + "recovered_errors":{ + "mean":1787.333333, + "std":77.500538, + "values":[ + 1865, + 1710, + 1787 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.230525, + "std":0.005079, + "values":[ + 0.233486, + 0.22466, + 0.233429 + ] + }, + "oos_auroc":{ + "mean":0.650803, + "std":0.097357, + "values":[ + 0.65666, + 0.55065, + 0.7451 + ] + }, + "oos_auprc":{ + "mean":0.055299, + "std":0.024601, + "values":[ + 0.048834, + 0.034576, + 0.082486 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.318472, + "std":0.002753, + "values":[ + 0.31564, + 0.321138, + 0.318639 + ] + }, + "oos_auroc":{ + "mean":0.573087, + "std":0.064199, + "values":[ + 0.579978, + 0.505721, + 0.633562 + ] + }, + "oos_auprc":{ + "mean":0.214768, + "std":0.046802, + "values":[ + 0.210275, + 0.170374, + 0.263654 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.225938, + "std":0.017797, + "values":[ + 0.219943, + 0.211912, + 0.245959 + ] + }, + "oos_auroc":{ + "mean":0.732789, + "std":0.062741, + "values":[ + 0.790407, + 0.665947, + 0.742013 + ] + }, + "oos_auprc":{ + "mean":0.091195, + "std":0.021984, + "values":[ + 0.114801, + 0.071308, + 0.087475 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.305952, + "std":0.016071, + "values":[ + 0.296562, + 0.296786, + 0.324509 + ] + }, + "oos_auroc":{ + "mean":0.669386, + "std":0.02434, + "values":[ + 0.697358, + 0.653026, + 0.657774 + ] + }, + "oos_auprc":{ + "mean":0.316954, + "std":0.029279, + "values":[ + 0.346065, + 0.317287, + 0.287509 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.179271, + "std":0.014356, + "values":[ + 0.195268, + 0.167508, + 0.175038 + ] + }, + "oos_auroc":{ + "mean":0.651744, + "std":0.05103, + "values":[ + 0.615493, + 0.62964, + 0.7101 + ] + }, + "oos_auprc":{ + "mean":0.058596, + "std":0.017885, + "values":[ + 0.047709, + 0.048842, + 0.079237 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.252626, + "std":0.011671, + "values":[ + 0.263706, + 0.240442, + 0.253732 + ] + }, + "oos_auroc":{ + "mean":0.620342, + "std":0.030324, + "values":[ + 0.622226, + 0.58912, + 0.64968 + ] + }, + "oos_auprc":{ + "mean":0.237372, + "std":0.016182, + "values":[ + 0.241203, + 0.219618, + 0.251295 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.180977, + "std":0.013485, + "values":[ + 0.194955, + 0.168047, + 0.179929 + ] + }, + "oos_auroc":{ + "mean":0.692213, + "std":0.05174, + "values":[ + 0.655173, + 0.670137, + 0.75133 + ] + }, + "oos_auprc":{ + "mean":0.084555, + "std":0.025366, + "values":[ + 0.064125, + 0.076594, + 0.112947 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.251916, + "std":0.012028, + "values":[ + 0.264079, + 0.240029, + 0.251639 + ] + }, + "oos_auroc":{ + "mean":0.651049, + "std":0.033847, + "values":[ + 0.649398, + 0.618058, + 0.685692 + ] + }, + "oos_auprc":{ + "mean":0.282136, + "std":0.033454, + "values":[ + 0.285187, + 0.247261, + 0.313961 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "msp", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":0.401006, + "std":0.0, + "n":1, + "values":[ + null, + 0.401006, + null + ] + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.022581, + "std":0.039111, + "values":[ + 0.0, + 0.067742, + 0.0 + ] + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0, + "n":1, + "values":[ + null, + 0.0, + null + ] + }, + "validation_risk_upper95":{ + "mean":0.01272, + "std":0.0, + "n":1, + "values":[ + null, + 0.01272, + null + ] + }, + "test":{ + "coverage":{ + "mean":0.015636, + "std":0.027083, + "values":[ + 0.0, + 0.046909, + 0.0 + ] + }, + "selective_risk":{ + "mean":0.023256, + "std":0.0, + "n":1, + "values":[ + null, + 0.023256, + null + ] + }, + "accuracy_8":{ + "mean":0.820667, + "std":0.000105, + "values":[ + 0.820727, + 0.820545, + 0.820727 + ] + }, + "oos_recall":{ + "mean":0.567667, + "std":0.000577, + "values":[ + 0.568, + 0.567, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.53676, + "std":0.00058, + "values":[ + 0.537095, + 0.536091, + 0.537095 + ] + }, + { + "mean":0.598055, + "std":0.00057, + "values":[ + 0.598384, + 0.597397, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.748242, + "std":0.000192, + "values":[ + 0.748353, + 0.748021, + 0.748353 + ] + }, + "oos_f1":{ + "mean":0.645565, + "std":0.000445, + "values":[ + 0.645821, + 0.645051, + 0.645821 + ] + }, + "oos_misroute_rate":{ + "mean":0.432333, + "std":0.000577, + "values":[ + 0.432, + 0.433, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002, + "std":0.003464, + "values":[ + 0.0, + 0.006, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.984364, + "std":0.027083, + "values":[ + 1.0, + 0.953091, + 1.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.363704, + "std":0.010025, + "values":[ + 0.369492, + 0.352128, + 0.369492 + ] + }, + "error_recovery_rate":{ + "mean":0.720554, + "std":0.013875, + "values":[ + 0.736572, + 0.712855, + 0.712236 + ] + }, + "recoverable_caught":{ + "mean":0.99961, + "std":0.000675, + "values":[ + 1.0, + 0.99883, + 1.0 + ] + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231, + "values":[ + 2532, + 2396, + 2509 + ] + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538, + "values":[ + 1865, + 1710, + 1787 + ] + }, + "recovered_errors":{ + "mean":1786.666667, + "std":78.500531, + "values":[ + 1865, + 1708, + 1787 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "msp", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":-0.340128, + "std":1.222357, + "values":[ + -1.751573, + 0.370436, + 0.360752 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.101613, + "std":0.024446, + "values":[ + 0.073871, + 0.110968, + 0.12 + ] + }, + "validation_selective_risk":{ + "mean":0.02828, + "std":0.001818, + "values":[ + 0.026201, + 0.02907, + 0.02957 + ] + }, + "validation_risk_upper95":{ + "mean":0.04855, + "std":0.001142, + "values":[ + 0.049859, + 0.048034, + 0.047756 + ] + }, + "test":{ + "coverage":{ + "mean":0.08503, + "std":0.026823, + "values":[ + 0.055636, + 0.091273, + 0.108182 + ] + }, + "selective_risk":{ + "mean":0.058689, + "std":0.020595, + "values":[ + 0.055556, + 0.039841, + 0.080672 + ] + }, + "accuracy_8":{ + "mean":0.820424, + "std":0.000525, + "values":[ + 0.819818, + 0.820727, + 0.820727 + ] + }, + "oos_recall":{ + "mean":0.566, + "std":0.0, + "values":[ + 0.566, + 0.566, + 0.566 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.535086, + "std":0.0, + "values":[ + 0.535086, + 0.535086, + 0.535086 + ] + }, + { + "mean":0.596409, + "std":0.0, + "values":[ + 0.596409, + 0.596409, + 0.596409 + ] + } + ], + "oos_precision":{ + "mean":0.748018, + "std":0.000571, + "values":[ + 0.748677, + 0.747688, + 0.747688 + ] + }, + "oos_f1":{ + "mean":0.644402, + "std":0.000212, + "values":[ + 0.644647, + 0.64428, + 0.64428 + ] + }, + "oos_misroute_rate":{ + "mean":0.434, + "std":0.0, + "values":[ + 0.434, + 0.434, + 0.434 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.017, + "std":0.009644, + "values":[ + 0.01, + 0.013, + 0.028 + ] + }, + "llm_call_rate":{ + "mean":0.91497, + "std":0.026823, + "values":[ + 0.944364, + 0.908727, + 0.891818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.337973, + "std":0.009941, + "values":[ + 0.348861, + 0.335679, + 0.32938 + ] + }, + "error_recovery_rate":{ + "mean":0.717761, + "std":0.013713, + "values":[ + 0.733412, + 0.71202, + 0.707852 + ] + }, + "recoverable_caught":{ + "mean":0.995739, + "std":0.001908, + "values":[ + 0.99571, + 0.997661, + 0.993844 + ] + }, + "small_errors":{ + "mean":2479.0, + "std":72.794231, + "values":[ + 2532, + 2396, + 2509 + ] + }, + "recoverable_errors":{ + "mean":1787.333333, + "std":77.500538, + "values":[ + 1865, + 1710, + 1787 + ] + }, + "recovered_errors":{ + "mean":1779.666667, + "std":75.566748, + "values":[ + 1857, + 1706, + 1776 + ] + } + } + } + } + } + } + } + }, + "bert/k10":{ + "model":"bert", + "k":10, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "bert-base-uncased-k10-seed42", + "bert-base-uncased-k10-seed43", + "bert-base-uncased-k10-seed44" + ], + "logits_sha256":[ + "51df91b493b56ae7d49fdd3195bc1d53d55099ec5b1cf011dfa9579a8a6320a3", + "113c11c8b72a8f20ac46e70a1c8492ee1a4d0571b1b86caba1b583cc85d0faf8", + "b60ca5253fc8ff28108fcfa32d4aa79139357e9f5639d187b2270a266c594adb" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":0.260398, + "std":0.007523, + "values":[ + 0.268719, + 0.258395, + 0.254079 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.614176, + "std":0.013099, + "values":[ + 0.628556, + 0.611047, + 0.602926 + ] + }, + "ece_8_summed":{ + "mean":0.342029, + "std":0.016509, + "values":[ + 0.330985, + 0.334094, + 0.361006 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.030988, + "std":0.007199, + "values":[ + 0.023451, + 0.031723, + 0.037791 + ] + }, + "ece_8_summed":{ + "mean":0.05643, + "std":0.016417, + "values":[ + 0.037663, + 0.063496, + 0.068131 + ] + } + } + } + }, + "selected_aggregation":[ + "summed", + "argmax", + "argmax" + ], + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.930108, + "std":0.000931, + "values":[ + 0.929032, + 0.930645, + 0.930645 + ] + }, + "summed":{ + "mean":0.92957, + "std":0.002196, + "values":[ + 0.93129, + 0.927097, + 0.930323 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.930108, + "std":0.000931 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.177697, + "std":0.017591 + }, + "accuracy_8":{ + "mean":0.822303, + "std":0.017591 + }, + "oos_recall":{ + "mean":0.26, + "std":0.074666 + }, + "oos_recall_wilson95":[ + { + "mean":0.234096, + "std":0.07208 + }, + { + "mean":0.287741, + "std":0.076681 + } + ], + "oos_precision":{ + "mean":0.934768, + "std":0.017075 + }, + "oos_f1":{ + "mean":0.403388, + "std":0.089838 + }, + "oos_misroute_rate":{ + "mean":0.74, + "std":0.074666 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.74, + "std":0.074666 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.822303, + "std":0.017591 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.916364, + "std":0.004098 + }, + "oos_recall":{ + "mean":0.609667, + "std":0.022502 + }, + "oos_recall_wilson95":[ + { + "mean":0.579089, + "std":0.022743 + }, + { + "mean":0.639405, + "std":0.022088 + } + ], + "oos_precision":{ + "mean":0.954533, + "std":0.002827 + }, + "oos_f1":{ + "mean":0.743946, + "std":0.017479 + }, + "oos_misroute_rate":{ + "mean":0.390333, + "std":0.022502 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.177697, + "std":0.017591 + }, + "llm_cost_usd_per_1k":{ + "mean":0.065477, + "std":0.006481 + }, + "error_recovery_rate":{ + "mean":0.527665, + "std":0.025364 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":517.333333, + "std":74.446849 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.015605, + "std":0.003211 + }, + "oos_auroc":{ + "mean":0.862107, + "std":0.005422 + }, + "oos_auprc":{ + "mean":0.244896, + "std":0.038654 + } + }, + "test":{ + "aurc":{ + "mean":0.045024, + "std":0.006318 + }, + "oos_auroc":{ + "mean":0.883595, + "std":0.003798 + }, + "oos_auprc":{ + "mean":0.648989, + "std":0.027077 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.022787, + "std":0.005519 + }, + "oos_auroc":{ + "mean":0.852954, + "std":0.038391 + }, + "oos_auprc":{ + "mean":0.223315, + "std":0.033168 + } + }, + "test":{ + "aurc":{ + "mean":0.056177, + "std":0.006721 + }, + "oos_auroc":{ + "mean":0.873052, + "std":0.02414 + }, + "oos_auprc":{ + "mean":0.621534, + "std":0.057028 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.019672, + "std":0.002335 + }, + "oos_auroc":{ + "mean":0.741519, + "std":0.02309 + }, + "oos_auprc":{ + "mean":0.077122, + "std":0.011242 + } + }, + "test":{ + "aurc":{ + "mean":0.063674, + "std":0.007358 + }, + "oos_auroc":{ + "mean":0.742799, + "std":0.008037 + }, + "oos_auprc":{ + "mean":0.341904, + "std":0.002831 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.015831, + "std":0.002575 + }, + "oos_auroc":{ + "mean":0.805342, + "std":0.018995 + }, + "oos_auprc":{ + "mean":0.177737, + "std":0.040579 + } + }, + "test":{ + "aurc":{ + "mean":0.051075, + "std":0.00662 + }, + "oos_auroc":{ + "mean":0.818427, + "std":0.007592 + }, + "oos_auprc":{ + "mean":0.536633, + "std":0.038092 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":0.539077, + "std":0.411834 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.657312, + "std":0.075286 + }, + "validation_selective_risk":{ + "mean":0.014703, + "std":0.000325 + }, + "validation_risk_upper95":{ + "mean":0.019796, + "std":0.000241 + }, + "test":{ + "coverage":{ + "mean":0.565758, + "std":0.066003 + }, + "selective_risk":{ + "mean":0.034856, + "std":0.005714 + }, + "accuracy_8":{ + "mean":0.871152, + "std":0.006289 + }, + "oos_recall":{ + "mean":0.549667, + "std":0.005033 + }, + "oos_recall_wilson95":[ + { + "mean":0.518699, + "std":0.005045 + }, + { + "mean":0.580254, + "std":0.004983 + } + ], + "oos_precision":{ + "mean":0.84859, + "std":0.024164 + }, + "oos_f1":{ + "mean":0.667128, + "std":0.010839 + }, + "oos_misroute_rate":{ + "mean":0.450333, + "std":0.005033 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.076667, + "std":0.017214 + }, + "llm_call_rate":{ + "mean":0.434242, + "std":0.066003 + }, + "llm_cost_usd_per_1k":{ + "mean":0.160206, + "std":0.024395 + }, + "error_recovery_rate":{ + "mean":0.480687, + "std":0.03575 + }, + "recoverable_caught":{ + "mean":0.91023, + "std":0.026262 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":472.0, + "std":79.037966 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.08416, + "std":0.01722 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.522688, + "std":0.191219 + }, + "validation_selective_risk":{ + "mean":0.013933, + "std":0.001073 + }, + "validation_risk_upper95":{ + "mean":0.019825, + "std":0.000186 + }, + "test":{ + "coverage":{ + "mean":0.44703, + "std":0.170286 + }, + "selective_risk":{ + "mean":0.023528, + "std":0.006649 + }, + "accuracy_8":{ + "mean":0.860242, + "std":0.0154 + }, + "oos_recall":{ + "mean":0.559333, + "std":0.004509 + }, + "oos_recall_wilson95":[ + { + "mean":0.528394, + "std":0.004525 + }, + { + "mean":0.589819, + "std":0.004459 + } + ], + "oos_precision":{ + "mean":0.810991, + "std":0.04643 + }, + "oos_f1":{ + "mean":0.661619, + "std":0.012474 + }, + "oos_misroute_rate":{ + "mean":0.440667, + "std":0.004509 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.037333, + "std":0.024583 + }, + "llm_call_rate":{ + "mean":0.55297, + "std":0.170286 + }, + "llm_cost_usd_per_1k":{ + "mean":0.204049, + "std":0.062925 + }, + "error_recovery_rate":{ + "mean":0.498133, + "std":0.049214 + }, + "recoverable_caught":{ + "mean":0.942457, + "std":0.049139 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":490.0, + "std":93.642939 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.490902, + "std":0.08277 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.273011, + "std":0.271631 + }, + "validation_selective_risk":{ + "mean":0.008897, + "std":0.005127 + }, + "validation_risk_upper95":{ + "mean":0.018795, + "std":0.000968 + }, + "test":{ + "coverage":{ + "mean":0.233697, + "std":0.23014 + }, + "selective_risk":{ + "mean":0.025093, + "std":0.006239 + }, + "accuracy_8":{ + "mean":0.834909, + "std":0.017895 + }, + "oos_recall":{ + "mean":0.565667, + "std":0.002309 + }, + "oos_recall_wilson95":[ + { + "mean":0.534752, + "std":0.002319 + }, + { + "mean":0.596079, + "std":0.002282 + } + ], + "oos_precision":{ + "mean":0.786343, + "std":0.05783 + }, + "oos_f1":{ + "mean":0.657366, + "std":0.018174 + }, + "oos_misroute_rate":{ + "mean":0.434333, + "std":0.002309 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.019, + "std":0.030315 + }, + "llm_call_rate":{ + "mean":0.766303, + "std":0.23014 + }, + "llm_cost_usd_per_1k":{ + "mean":0.283009, + "std":0.085152 + }, + "error_recovery_rate":{ + "mean":0.511384, + "std":0.039668 + }, + "recoverable_caught":{ + "mean":0.968206, + "std":0.029326 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":502.333333, + "std":86.071676 + } + } + }, + "margin":{ + "tau":{ + "mean":2.248631, + "std":0.304739 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.589677, + "std":0.034107 + }, + "validation_selective_risk":{ + "mean":0.014394, + "std":0.000332 + }, + "validation_risk_upper95":{ + "mean":0.019751, + "std":0.000222 + }, + "test":{ + "coverage":{ + "mean":0.519394, + "std":0.042494 + }, + "selective_risk":{ + "mean":0.042635, + "std":0.005851 + }, + "accuracy_8":{ + "mean":0.862848, + "std":0.003642 + }, + "oos_recall":{ + "mean":0.548667, + "std":0.009074 + }, + "oos_recall_wilson95":[ + { + "mean":0.517699, + "std":0.009093 + }, + { + "mean":0.579262, + "std":0.008985 + } + ], + "oos_precision":{ + "mean":0.827583, + "std":0.01402 + }, + "oos_f1":{ + "mean":0.659851, + "std":0.010528 + }, + "oos_misroute_rate":{ + "mean":0.451333, + "std":0.009074 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.090667, + "std":0.010214 + }, + "llm_call_rate":{ + "mean":0.480606, + "std":0.042494 + }, + "llm_cost_usd_per_1k":{ + "mean":0.177384, + "std":0.015673 + }, + "error_recovery_rate":{ + "mean":0.474484, + "std":0.045935 + }, + "recoverable_caught":{ + "mean":0.897775, + "std":0.044944 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":466.666667, + "std":88.296848 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.73761, + "std":0.074767 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.646667, + "std":0.061392 + }, + "validation_selective_risk":{ + "mean":0.014624, + "std":0.000189 + }, + "validation_risk_upper95":{ + "mean":0.019748, + "std":0.000218 + }, + "test":{ + "coverage":{ + "mean":0.565152, + "std":0.065138 + }, + "selective_risk":{ + "mean":0.039526, + "std":0.00626 + }, + "accuracy_8":{ + "mean":0.870061, + "std":0.00455 + }, + "oos_recall":{ + "mean":0.548667, + "std":0.003512 + }, + "oos_recall_wilson95":[ + { + "mean":0.517696, + "std":0.00352 + }, + { + "mean":0.579265, + "std":0.003477 + } + ], + "oos_precision":{ + "mean":0.84748, + "std":0.023122 + }, + "oos_f1":{ + "mean":0.666046, + "std":0.009602 + }, + "oos_misroute_rate":{ + "mean":0.451333, + "std":0.003512 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.090667, + "std":0.017616 + }, + "llm_call_rate":{ + "mean":0.434848, + "std":0.065138 + }, + "llm_cost_usd_per_1k":{ + "mean":0.160461, + "std":0.024032 + }, + "error_recovery_rate":{ + "mean":0.473759, + "std":0.047125 + }, + "recoverable_caught":{ + "mean":0.896341, + "std":0.047477 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":466.0, + "std":89.202018 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.035917, + "std":0.000849 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.945376, + "std":0.006891 + }, + "validation_selective_risk":{ + "mean":0.043221, + "std":9.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.049832, + "std":8.7e-05 + }, + "test":{ + "coverage":{ + "mean":0.877939, + "std":0.016963 + }, + "selective_risk":{ + "mean":0.103684, + "std":0.005884 + }, + "accuracy_8":{ + "mean":0.866848, + "std":0.009073 + }, + "oos_recall":{ + "mean":0.462333, + "std":0.040104 + }, + "oos_recall_wilson95":[ + { + "mean":0.431701, + "std":0.039776 + }, + { + "mean":0.493254, + "std":0.040126 + } + ], + "oos_precision":{ + "mean":0.937819, + "std":0.013175 + }, + "oos_f1":{ + "mean":0.618851, + "std":0.038742 + }, + "oos_misroute_rate":{ + "mean":0.537667, + "std":0.040104 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.341667, + "std":0.019348 + }, + "llm_call_rate":{ + "mean":0.122061, + "std":0.016963 + }, + "llm_cost_usd_per_1k":{ + "mean":0.044936, + "std":0.006251 + }, + "error_recovery_rate":{ + "mean":0.279989, + "std":0.021878 + }, + "recoverable_caught":{ + "mean":0.53018, + "std":0.020236 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":275.0, + "std":46.808119 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.035917, + "std":0.000849 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.945376, + "std":0.006891 + }, + "validation_selective_risk":{ + "mean":0.043221, + "std":9.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.049832, + "std":8.7e-05 + }, + "test":{ + "coverage":{ + "mean":0.877939, + "std":0.016963 + }, + "selective_risk":{ + "mean":0.103684, + "std":0.005884 + }, + "accuracy_8":{ + "mean":0.866848, + "std":0.009073 + }, + "oos_recall":{ + "mean":0.462333, + "std":0.040104 + }, + "oos_recall_wilson95":[ + { + "mean":0.431701, + "std":0.039776 + }, + { + "mean":0.493254, + "std":0.040126 + } + ], + "oos_precision":{ + "mean":0.937819, + "std":0.013175 + }, + "oos_f1":{ + "mean":0.618851, + "std":0.038742 + }, + "oos_misroute_rate":{ + "mean":0.537667, + "std":0.040104 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.341667, + "std":0.019348 + }, + "llm_call_rate":{ + "mean":0.122061, + "std":0.016963 + }, + "llm_cost_usd_per_1k":{ + "mean":0.044936, + "std":0.006251 + }, + "error_recovery_rate":{ + "mean":0.279989, + "std":0.021878 + }, + "recoverable_caught":{ + "mean":0.53018, + "std":0.020236 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":275.0, + "std":46.808119 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.762971, + "std":0.012689 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.897527, + "std":0.033833 + }, + "validation_selective_risk":{ + "mean":0.043004, + "std":0.000228 + }, + "validation_risk_upper95":{ + "mean":0.049789, + "std":0.000108 + }, + "test":{ + "coverage":{ + "mean":0.812545, + "std":0.030076 + }, + "selective_risk":{ + "mean":0.096204, + "std":0.010415 + }, + "accuracy_8":{ + "mean":0.864364, + "std":0.011189 + }, + "oos_recall":{ + "mean":0.475333, + "std":0.034312 + }, + "oos_recall_wilson95":[ + { + "mean":0.444583, + "std":0.034093 + }, + { + "mean":0.506273, + "std":0.034269 + } + ], + "oos_precision":{ + "mean":0.908304, + "std":0.040601 + }, + "oos_f1":{ + "mean":0.623597, + "std":0.034221 + }, + "oos_misroute_rate":{ + "mean":0.524667, + "std":0.034312 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.291667, + "std":0.031021 + }, + "llm_call_rate":{ + "mean":0.187455, + "std":0.030076 + }, + "llm_cost_usd_per_1k":{ + "mean":0.069037, + "std":0.011137 + }, + "error_recovery_rate":{ + "mean":0.314491, + "std":0.031791 + }, + "recoverable_caught":{ + "mean":0.596178, + "std":0.054261 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":308.0, + "std":50.764161 + } + } + }, + "margin":{ + "tau":{ + "mean":0.667398, + "std":0.126649 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.847634, + "std":0.026777 + }, + "validation_selective_risk":{ + "mean":0.042877, + "std":9.3e-05 + }, + "validation_risk_upper95":{ + "mean":0.049864, + "std":0.000169 + }, + "test":{ + "coverage":{ + "mean":0.804303, + "std":0.020874 + }, + "selective_risk":{ + "mean":0.119625, + "std":0.017638 + }, + "accuracy_8":{ + "mean":0.852303, + "std":0.012676 + }, + "oos_recall":{ + "mean":0.418, + "std":0.058966 + }, + "oos_recall_wilson95":[ + { + "mean":0.387948, + "std":0.05826 + }, + { + "mean":0.44868, + "std":0.059221 + } + ], + "oos_precision":{ + "mean":0.89106, + "std":0.023485 + }, + "oos_f1":{ + "mean":0.567739, + "std":0.056387 + }, + "oos_misroute_rate":{ + "mean":0.582, + "std":0.058966 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.397667, + "std":0.071431 + }, + "llm_call_rate":{ + "mean":0.195697, + "std":0.020874 + }, + "llm_cost_usd_per_1k":{ + "mean":0.072199, + "std":0.007709 + }, + "error_recovery_rate":{ + "mean":0.250701, + "std":0.019456 + }, + "recoverable_caught":{ + "mean":0.475783, + "std":0.041776 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":245.0, + "std":31.480152 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.364381, + "std":0.021389 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.918172, + "std":0.022974 + }, + "validation_selective_risk":{ + "mean":0.043094, + "std":0.000169 + }, + "validation_risk_upper95":{ + "mean":0.049802, + "std":0.000107 + }, + "test":{ + "coverage":{ + "mean":0.86, + "std":0.015904 + }, + "selective_risk":{ + "mean":0.10525, + "std":0.010326 + }, + "accuracy_8":{ + "mean":0.863818, + "std":0.009696 + }, + "oos_recall":{ + "mean":0.452667, + "std":0.050332 + }, + "oos_recall_wilson95":[ + { + "mean":0.422161, + "std":0.049874 + }, + { + "mean":0.483535, + "std":0.050405 + } + ], + "oos_precision":{ + "mean":0.926268, + "std":0.00127 + }, + "oos_f1":{ + "mean":0.607056, + "std":0.045344 + }, + "oos_misroute_rate":{ + "mean":0.547333, + "std":0.050332 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.348333, + "std":0.038631 + }, + "llm_call_rate":{ + "mean":0.14, + "std":0.015904 + }, + "llm_cost_usd_per_1k":{ + "mean":0.051597, + "std":0.005876 + }, + "error_recovery_rate":{ + "mean":0.270709, + "std":0.018159 + }, + "recoverable_caught":{ + "mean":0.512823, + "std":0.016433 + }, + "small_errors":{ + "mean":977.333333, + "std":96.748816 + }, + "recoverable_errors":{ + "mean":517.333333, + "std":74.446849 + }, + "recovered_errors":{ + "mean":265.666667, + "std":42.253205 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.92957, + "std":0.002196 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.185273, + "std":0.015458 + }, + "accuracy_8":{ + "mean":0.814727, + "std":0.015458 + }, + "oos_recall":{ + "mean":0.211667, + "std":0.067174 + }, + "oos_recall_wilson95":[ + { + "mean":0.187805, + "std":0.064166 + }, + { + "mean":0.237735, + "std":0.069668 + } + ], + "oos_precision":{ + "mean":0.954948, + "std":0.01456 + }, + "oos_f1":{ + "mean":0.34346, + "std":0.089187 + }, + "oos_misroute_rate":{ + "mean":0.788333, + "std":0.067174 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.788333, + "std":0.067174 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.814727, + "std":0.015458 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.914424, + "std":0.002819 + }, + "oos_recall":{ + "mean":0.598, + "std":0.017578 + }, + "oos_recall_wilson95":[ + { + "mean":0.567305, + "std":0.017738 + }, + { + "mean":0.627945, + "std":0.017284 + } + ], + "oos_precision":{ + "mean":0.956701, + "std":0.006664 + }, + "oos_f1":{ + "mean":0.735913, + "std":0.015232 + }, + "oos_misroute_rate":{ + "mean":0.402, + "std":0.017578 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.185273, + "std":0.015458 + }, + "llm_cost_usd_per_1k":{ + "mean":0.068254, + "std":0.005686 + }, + "error_recovery_rate":{ + "mean":0.536721, + "std":0.025144 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":548.333333, + "std":69.973805 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.028175, + "std":0.002406 + }, + "oos_auroc":{ + "mean":0.776151, + "std":0.033207 + }, + "oos_auprc":{ + "mean":0.142726, + "std":0.051875 + } + }, + "test":{ + "aurc":{ + "mean":0.090512, + "std":0.006399 + }, + "oos_auroc":{ + "mean":0.779515, + "std":0.020031 + }, + "oos_auprc":{ + "mean":0.465774, + "std":0.062751 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.028467, + "std":0.001716 + }, + "oos_auroc":{ + "mean":0.848998, + "std":0.023241 + }, + "oos_auprc":{ + "mean":0.351424, + "std":0.036746 + } + }, + "test":{ + "aurc":{ + "mean":0.086774, + "std":0.007992 + }, + "oos_auroc":{ + "mean":0.825359, + "std":0.013883 + }, + "oos_auprc":{ + "mean":0.633858, + "std":0.030452 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.013299, + "std":0.001198 + }, + "oos_auroc":{ + "mean":0.85413, + "std":0.015746 + }, + "oos_auprc":{ + "mean":0.184672, + "std":0.020311 + } + }, + "test":{ + "aurc":{ + "mean":0.047152, + "std":0.004731 + }, + "oos_auroc":{ + "mean":0.86542, + "std":0.010891 + }, + "oos_auprc":{ + "mean":0.555919, + "std":0.028234 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.012768, + "std":0.001389 + }, + "oos_auroc":{ + "mean":0.867222, + "std":0.011622 + }, + "oos_auprc":{ + "mean":0.280261, + "std":0.035889 + } + }, + "test":{ + "aurc":{ + "mean":0.044777, + "std":0.00462 + }, + "oos_auroc":{ + "mean":0.883748, + "std":0.010075 + }, + "oos_auprc":{ + "mean":0.654698, + "std":0.021902 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "msp_t", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.621489, + "std":1.281221 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.79086, + "std":0.037665 + }, + "validation_selective_risk":{ + "mean":0.015224, + "std":0.000154 + }, + "validation_risk_upper95":{ + "mean":0.019861, + "std":0.000103 + }, + "test":{ + "coverage":{ + "mean":0.678848, + "std":0.038491 + }, + "selective_risk":{ + "mean":0.044193, + "std":0.004392 + }, + "accuracy_8":{ + "mean":0.873697, + "std":0.005753 + }, + "oos_recall":{ + "mean":0.539667, + "std":0.010693 + }, + "oos_recall_wilson95":[ + { + "mean":0.508686, + "std":0.010708 + }, + { + "mean":0.570344, + "std":0.010595 + } + ], + "oos_precision":{ + "mean":0.866207, + "std":0.028573 + }, + "oos_f1":{ + "mean":0.664761, + "std":0.006015 + }, + "oos_misroute_rate":{ + "mean":0.460333, + "std":0.010693 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.133667, + "std":0.022591 + }, + "llm_call_rate":{ + "mean":0.321152, + "std":0.038491 + }, + "llm_cost_usd_per_1k":{ + "mean":0.118335, + "std":0.014205 + }, + "error_recovery_rate":{ + "mean":0.485619, + "std":0.027982 + }, + "recoverable_caught":{ + "mean":0.904528, + "std":0.014438 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":496.333333, + "std":67.67816 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.482985, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.028925, + "std":0.050099 + }, + "validation_selective_risk":{ + "mean":0.003717, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.016489, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.022848, + "std":0.039575 + }, + "selective_risk":{ + "mean":0.034483, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820788, + "std":0.000105 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.004, + "std":0.006928 + }, + "llm_call_rate":{ + "mean":0.977152, + "std":0.039575 + }, + "llm_cost_usd_per_1k":{ + "mean":0.361018, + "std":0.014677 + }, + "error_recovery_rate":{ + "mean":0.536407, + "std":0.024965 + }, + "recoverable_caught":{ + "mean":0.999426, + "std":0.000994 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":548.0, + "std":69.742383 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.599557, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.028172, + "std":0.048795 + }, + "validation_selective_risk":{ + "mean":0.003817, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.016925, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.022424, + "std":0.03884 + }, + "selective_risk":{ + "mean":0.035135, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820788, + "std":0.000105 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.004, + "std":0.006928 + }, + "llm_call_rate":{ + "mean":0.977576, + "std":0.03884 + }, + "llm_cost_usd_per_1k":{ + "mean":0.361177, + "std":0.014402 + }, + "error_recovery_rate":{ + "mean":0.536407, + "std":0.024965 + }, + "recoverable_caught":{ + "mean":0.999426, + "std":0.000994 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":548.0, + "std":69.742383 + } + } + }, + "margin":{ + "tau":{ + "mean":2.895226, + "std":0.178649 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.77, + "std":0.017428 + }, + "validation_selective_risk":{ + "mean":0.015081, + "std":0.000171 + }, + "validation_risk_upper95":{ + "mean":0.019768, + "std":0.000188 + }, + "test":{ + "coverage":{ + "mean":0.664, + "std":0.023645 + }, + "selective_risk":{ + "mean":0.043842, + "std":0.003598 + }, + "accuracy_8":{ + "mean":0.869333, + "std":0.000639 + }, + "oos_recall":{ + "mean":0.537, + "std":0.009644 + }, + "oos_recall_wilson95":[ + { + "mean":0.506016, + "std":0.009654 + }, + { + "mean":0.567701, + "std":0.00956 + } + ], + "oos_precision":{ + "mean":0.862647, + "std":0.02378 + }, + "oos_f1":{ + "mean":0.661736, + "std":0.003218 + }, + "oos_misroute_rate":{ + "mean":0.463, + "std":0.009644 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.135, + "std":0.014526 + }, + "llm_call_rate":{ + "mean":0.336, + "std":0.023645 + }, + "llm_cost_usd_per_1k":{ + "mean":0.123842, + "std":0.008705 + }, + "error_recovery_rate":{ + "mean":0.487429, + "std":0.025143 + }, + "recoverable_caught":{ + "mean":0.90809, + "std":0.013066 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":498.0, + "std":64.861391 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.898791, + "std":0.031529 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.778817, + "std":0.057706 + }, + "validation_selective_risk":{ + "mean":0.015177, + "std":0.000234 + }, + "validation_risk_upper95":{ + "mean":0.019853, + "std":0.000111 + }, + "test":{ + "coverage":{ + "mean":0.66897, + "std":0.05488 + }, + "selective_risk":{ + "mean":0.042247, + "std":0.004518 + }, + "accuracy_8":{ + "mean":0.873515, + "std":0.005993 + }, + "oos_recall":{ + "mean":0.541333, + "std":0.009452 + }, + "oos_recall_wilson95":[ + { + "mean":0.510353, + "std":0.009466 + }, + { + "mean":0.571997, + "std":0.009365 + } + ], + "oos_precision":{ + "mean":0.862062, + "std":0.030808 + }, + "oos_f1":{ + "mean":0.664788, + "std":0.00597 + }, + "oos_misroute_rate":{ + "mean":0.458667, + "std":0.009452 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.124333, + "std":0.026577 + }, + "llm_call_rate":{ + "mean":0.33103, + "std":0.05488 + }, + "llm_cost_usd_per_1k":{ + "mean":0.121973, + "std":0.020242 + }, + "error_recovery_rate":{ + "mean":0.486551, + "std":0.029298 + }, + "recoverable_caught":{ + "mean":0.906206, + "std":0.017335 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":497.333333, + "std":68.85734 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.544558, + "std":0.016827 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.95129, + "std":0.00264 + }, + "validation_selective_risk":{ + "mean":0.043178, + "std":8.9e-05 + }, + "validation_risk_upper95":{ + "mean":0.049765, + "std":8.8e-05 + }, + "test":{ + "coverage":{ + "mean":0.894364, + "std":0.00284 + }, + "selective_risk":{ + "mean":0.118052, + "std":0.013847 + }, + "accuracy_8":{ + "mean":0.858667, + "std":0.009518 + }, + "oos_recall":{ + "mean":0.420667, + "std":0.047248 + }, + "oos_recall_wilson95":[ + { + "mean":0.390525, + "std":0.046639 + }, + { + "mean":0.451416, + "std":0.047495 + } + ], + "oos_precision":{ + "mean":0.936088, + "std":0.002154 + }, + "oos_f1":{ + "mean":0.579471, + "std":0.044915 + }, + "oos_misroute_rate":{ + "mean":0.579333, + "std":0.047248 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.431333, + "std":0.055896 + }, + "llm_call_rate":{ + "mean":0.105636, + "std":0.00284 + }, + "llm_cost_usd_per_1k":{ + "mean":0.038861, + "std":0.001048 + }, + "error_recovery_rate":{ + "mean":0.270836, + "std":0.001786 + }, + "recoverable_caught":{ + "mean":0.505363, + "std":0.024184 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":276.0, + "std":23.388031 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.258529, + "std":0.005355 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.819677, + "std":0.044299 + }, + "validation_selective_risk":{ + "mean":0.042762, + "std":0.000156 + }, + "validation_risk_upper95":{ + "mean":0.04987, + "std":0.000166 + }, + "test":{ + "coverage":{ + "mean":0.755818, + "std":0.042435 + }, + "selective_risk":{ + "mean":0.118159, + "std":0.006185 + }, + "accuracy_8":{ + "mean":0.848788, + "std":0.004514 + }, + "oos_recall":{ + "mean":0.435333, + "std":0.027209 + }, + "oos_recall_wilson95":[ + { + "mean":0.40494, + "std":0.02686 + }, + { + "mean":0.466222, + "std":0.02735 + } + ], + "oos_precision":{ + "mean":0.889832, + "std":0.032782 + }, + "oos_f1":{ + "mean":0.584019, + "std":0.023088 + }, + "oos_misroute_rate":{ + "mean":0.564667, + "std":0.027209 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.376333, + "std":0.032532 + }, + "llm_call_rate":{ + "mean":0.244182, + "std":0.042435 + }, + "llm_cost_usd_per_1k":{ + "mean":0.090079, + "std":0.015655 + }, + "error_recovery_rate":{ + "mean":0.315875, + "std":0.047688 + }, + "recoverable_caught":{ + "mean":0.587003, + "std":0.066153 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":324.0, + "std":71.0 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.864353, + "std":0.012458 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.822903, + "std":0.026972 + }, + "validation_selective_risk":{ + "mean":0.042725, + "std":0.000187 + }, + "validation_risk_upper95":{ + "mean":0.049812, + "std":0.000104 + }, + "test":{ + "coverage":{ + "mean":0.743212, + "std":0.032942 + }, + "selective_risk":{ + "mean":0.106862, + "std":0.005519 + }, + "accuracy_8":{ + "mean":0.858788, + "std":0.004444 + }, + "oos_recall":{ + "mean":0.474667, + "std":0.01159 + }, + "oos_recall_wilson95":[ + { + "mean":0.443878, + "std":0.011514 + }, + { + "mean":0.505649, + "std":0.011577 + } + ], + "oos_precision":{ + "mean":0.885536, + "std":0.008003 + }, + "oos_f1":{ + "mean":0.61801, + "std":0.011172 + }, + "oos_misroute_rate":{ + "mean":0.525333, + "std":0.01159 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.315667, + "std":0.013051 + }, + "llm_call_rate":{ + "mean":0.256788, + "std":0.032942 + }, + "llm_cost_usd_per_1k":{ + "mean":0.09465, + "std":0.012167 + }, + "error_recovery_rate":{ + "mean":0.353365, + "std":0.022143 + }, + "recoverable_caught":{ + "mean":0.658048, + "std":0.011166 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":361.333333, + "std":51.539629 + } + } + }, + "margin":{ + "tau":{ + "mean":0.761537, + "std":0.056423 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.947957, + "std":0.004701 + }, + "validation_selective_risk":{ + "mean":0.043216, + "std":0.000129 + }, + "validation_risk_upper95":{ + "mean":0.049818, + "std":0.000121 + }, + "test":{ + "coverage":{ + "mean":0.899152, + "std":0.000859 + }, + "selective_risk":{ + "mean":0.124568, + "std":0.015192 + }, + "accuracy_8":{ + "mean":0.852727, + "std":0.008358 + }, + "oos_recall":{ + "mean":0.382, + "std":0.054443 + }, + "oos_recall_wilson95":[ + { + "mean":0.352522, + "std":0.053516 + }, + { + "mean":0.412381, + "std":0.054953 + } + ], + "oos_precision":{ + "mean":0.928193, + "std":0.002801 + }, + "oos_f1":{ + "mean":0.539702, + "std":0.053086 + }, + "oos_misroute_rate":{ + "mean":0.618, + "std":0.054443 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.485, + "std":0.06844 + }, + "llm_call_rate":{ + "mean":0.100848, + "std":0.000859 + }, + "llm_cost_usd_per_1k":{ + "mean":0.037118, + "std":0.000317 + }, + "error_recovery_rate":{ + "mean":0.244455, + "std":0.00558 + }, + "recoverable_caught":{ + "mean":0.455875, + "std":0.014617 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":249.333333, + "std":25.146239 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.544558, + "std":0.016827 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.95129, + "std":0.00264 + }, + "validation_selective_risk":{ + "mean":0.043178, + "std":8.9e-05 + }, + "validation_risk_upper95":{ + "mean":0.049765, + "std":8.8e-05 + }, + "test":{ + "coverage":{ + "mean":0.894364, + "std":0.00284 + }, + "selective_risk":{ + "mean":0.118052, + "std":0.013847 + }, + "accuracy_8":{ + "mean":0.858667, + "std":0.009518 + }, + "oos_recall":{ + "mean":0.420667, + "std":0.047248 + }, + "oos_recall_wilson95":[ + { + "mean":0.390525, + "std":0.046639 + }, + { + "mean":0.451416, + "std":0.047495 + } + ], + "oos_precision":{ + "mean":0.936088, + "std":0.002154 + }, + "oos_f1":{ + "mean":0.579471, + "std":0.044915 + }, + "oos_misroute_rate":{ + "mean":0.579333, + "std":0.047248 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.431333, + "std":0.055896 + }, + "llm_call_rate":{ + "mean":0.105636, + "std":0.00284 + }, + "llm_cost_usd_per_1k":{ + "mean":0.038861, + "std":0.001048 + }, + "error_recovery_rate":{ + "mean":0.270836, + "std":0.001786 + }, + "recoverable_caught":{ + "mean":0.505363, + "std":0.024184 + }, + "small_errors":{ + "mean":1019.0, + "std":85.017645 + }, + "recoverable_errors":{ + "mean":548.333333, + "std":69.973805 + }, + "recovered_errors":{ + "mean":276.0, + "std":23.388031 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.18103, + "std":0.011847, + "values":[ + 0.167455, + 0.186364, + 0.189273 + ] + }, + "accuracy_8":{ + "mean":0.81897, + "std":0.011847, + "values":[ + 0.832545, + 0.813636, + 0.810727 + ] + }, + "oos_recall":{ + "mean":0.240667, + "std":0.042028, + "values":[ + 0.287, + 0.23, + 0.205 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.215309, + "std":0.040349, + "values":[ + 0.259819, + 0.204979, + 0.18113 + ] + }, + { + "mean":0.268009, + "std":0.043385, + "values":[ + 0.315811, + 0.257087, + 0.231128 + ] + } + ], + "oos_precision":{ + "mean":0.943758, + "std":0.027312, + "values":[ + 0.969595, + 0.946502, + 0.915179 + ] + }, + "oos_f1":{ + "mean":0.382647, + "std":0.055055, + "values":[ + 0.442901, + 0.370072, + 0.334967 + ] + }, + "oos_misroute_rate":{ + "mean":0.759333, + "std":0.042028, + "values":[ + 0.713, + 0.77, + 0.795 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.759333, + "std":0.042028, + "values":[ + 0.713, + 0.77, + 0.795 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":995.666667, + "std":65.156223, + "values":[ + 921, + 1025, + 1041 + ] + }, + "recoverable_errors":{ + "mean":529.333333, + "std":53.873308, + "values":[ + 468, + 551, + 569 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.81897, + "std":0.011847, + "values":[ + 0.832545, + 0.813636, + 0.810727 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.915212, + "std":0.002107, + "values":[ + 0.917636, + 0.913818, + 0.914182 + ] + }, + "oos_recall":{ + "mean":0.604, + "std":0.013115, + "values":[ + 0.618, + 0.602, + 0.592 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.573352, + "std":0.013239, + "values":[ + 0.587488, + 0.571327, + 0.561243 + ] + }, + { + "mean":0.633852, + "std":0.01289, + "values":[ + 0.647609, + 0.631893, + 0.622053 + ] + } + ], + "oos_precision":{ + "mean":0.95665, + "std":0.006479, + "values":[ + 0.964119, + 0.952532, + 0.953301 + ] + }, + "oos_f1":{ + "mean":0.740453, + "std":0.011632, + "values":[ + 0.753199, + 0.737745, + 0.730413 + ] + }, + "oos_misroute_rate":{ + "mean":0.396, + "std":0.013115, + "values":[ + 0.382, + 0.398, + 0.408 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.18103, + "std":0.011847, + "values":[ + 0.167455, + 0.186364, + 0.189273 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.066704, + "std":0.004366, + "values":[ + 0.0617, + 0.068674, + 0.069738 + ] + }, + "error_recovery_rate":{ + "mean":0.530765, + "std":0.020104, + "values":[ + 0.508143, + 0.537561, + 0.54659 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":995.666667, + "std":65.156223, + "values":[ + 921, + 1025, + 1041 + ] + }, + "recoverable_errors":{ + "mean":529.333333, + "std":53.873308, + "values":[ + 468, + 551, + 569 + ] + }, + "recovered_errors":{ + "mean":529.333333, + "std":53.873308, + "values":[ + 468, + 551, + 569 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.020026, + "std":0.005483, + "values":[ + 0.025936, + 0.015106, + 0.019036 + ] + }, + "oos_auroc":{ + "mean":0.846936, + "std":0.028984, + "values":[ + 0.813953, + 0.868343, + 0.85851 + ] + }, + "oos_auprc":{ + "mean":0.240876, + "std":0.044285, + "values":[ + 0.198127, + 0.286552, + 0.237948 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.060064, + "std":0.020623, + "values":[ + 0.083645, + 0.045401, + 0.051145 + ] + }, + "oos_auroc":{ + "mean":0.855925, + "std":0.046613, + "values":[ + 0.802254, + 0.886272, + 0.879248 + ] + }, + "oos_auprc":{ + "mean":0.610146, + "std":0.068165, + "values":[ + 0.53772, + 0.67305, + 0.619668 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.026055, + "std":0.001665, + "values":[ + 0.0265, + 0.024213, + 0.027453 + ] + }, + "oos_auroc":{ + "mean":0.851973, + "std":0.03747, + "values":[ + 0.87473, + 0.872463, + 0.808727 + ] + }, + "oos_auprc":{ + "mean":0.284817, + "std":0.098527, + "values":[ + 0.39297, + 0.261314, + 0.200166 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.066564, + "std":0.011642, + "values":[ + 0.079852, + 0.058151, + 0.061689 + ] + }, + "oos_auroc":{ + "mean":0.858125, + "std":0.029933, + "values":[ + 0.836202, + 0.892228, + 0.845944 + ] + }, + "oos_auprc":{ + "mean":0.633965, + "std":0.063945, + "values":[ + 0.668833, + 0.672896, + 0.560165 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.017781, + "std":0.004998, + "values":[ + 0.012448, + 0.018537, + 0.022358 + ] + }, + "oos_auroc":{ + "mean":0.781606, + "std":0.061981, + "values":[ + 0.849203, + 0.768167, + 0.727447 + ] + }, + "oos_auprc":{ + "mean":0.121176, + "std":0.075405, + "values":[ + 0.207296, + 0.089225, + 0.067007 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.059188, + "std":0.014919, + "values":[ + 0.042187, + 0.065283, + 0.070094 + ] + }, + "oos_auroc":{ + "mean":0.788104, + "std":0.073236, + "values":[ + 0.872396, + 0.740071, + 0.751844 + ] + }, + "oos_auprc":{ + "mean":0.419488, + "std":0.13718, + "values":[ + 0.577888, + 0.33987, + 0.340705 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.014939, + "std":0.00361, + "values":[ + 0.011655, + 0.014357, + 0.018804 + ] + }, + "oos_auroc":{ + "mean":0.832149, + "std":0.033535, + "values":[ + 0.868453, + 0.825663, + 0.80233 + ] + }, + "oos_auprc":{ + "mean":0.212908, + "std":0.053324, + "values":[ + 0.259422, + 0.224591, + 0.154712 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.049364, + "std":0.00914, + "values":[ + 0.039915, + 0.050019, + 0.05816 + ] + }, + "oos_auroc":{ + "mean":0.845836, + "std":0.039915, + "values":[ + 0.89192, + 0.823438, + 0.822151 + ] + }, + "oos_auprc":{ + "mean":0.586375, + "std":0.087009, + "values":[ + 0.677125, + 0.578334, + 0.503666 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.81507, + "std":0.075711, + "values":[ + 0.893141, + 0.741964, + 0.810105 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.680968, + "std":0.111005, + "values":[ + 0.798387, + 0.666774, + 0.577742 + ] + }, + "validation_selective_risk":{ + "mean":0.014795, + "std":0.000484, + "values":[ + 0.015354, + 0.014514, + 0.014517 + ] + }, + "validation_risk_upper95":{ + "mean":0.019817, + "std":0.000259, + "values":[ + 0.01998, + 0.019519, + 0.019952 + ] + }, + "test":{ + "coverage":{ + "mean":0.58503, + "std":0.095454, + "values":[ + 0.686545, + 0.571455, + 0.497091 + ] + }, + "selective_risk":{ + "mean":0.037703, + "std":0.00459, + "values":[ + 0.039195, + 0.041362, + 0.032553 + ] + }, + "accuracy_8":{ + "mean":0.869273, + "std":0.003397, + "values":[ + 0.872364, + 0.869818, + 0.865636 + ] + }, + "oos_recall":{ + "mean":0.548667, + "std":0.003512, + "values":[ + 0.552, + 0.549, + 0.545 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.517696, + "std":0.00352, + "values":[ + 0.521037, + 0.51803, + 0.514022 + ] + }, + { + "mean":0.579265, + "std":0.003477, + "values":[ + 0.582565, + 0.579595, + 0.575634 + ] + } + ], + "oos_precision":{ + "mean":0.840916, + "std":0.019865, + "values":[ + 0.841463, + 0.860502, + 0.820783 + ] + }, + "oos_f1":{ + "mean":0.664015, + "std":0.007978, + "values":[ + 0.666667, + 0.67033, + 0.655048 + ] + }, + "oos_misroute_rate":{ + "mean":0.451333, + "std":0.003512, + "values":[ + 0.448, + 0.451, + 0.455 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.092333, + "std":0.019757, + "values":[ + 0.11, + 0.096, + 0.071 + ] + }, + "llm_call_rate":{ + "mean":0.41497, + "std":0.095454, + "values":[ + 0.313455, + 0.428545, + 0.502909 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.153061, + "std":0.03532, + "values":[ + 0.115483, + 0.158123, + 0.185577 + ] + }, + "error_recovery_rate":{ + "mean":0.485275, + "std":0.028503, + "values":[ + 0.456026, + 0.486829, + 0.512968 + ] + }, + "recoverable_caught":{ + "mean":0.91385, + "std":0.021727, + "values":[ + 0.897436, + 0.905626, + 0.938489 + ] + }, + "small_errors":{ + "mean":995.666667, + "std":65.156223, + "values":[ + 921, + 1025, + 1041 + ] + }, + "recoverable_errors":{ + "mean":529.333333, + "std":53.873308, + "values":[ + 468, + 551, + 569 + ] + }, + "recovered_errors":{ + "mean":484.333333, + "std":58.398059, + "values":[ + 420, + 499, + 534 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp_t", + "msp", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":0.20164, + "std":0.286486, + "values":[ + 0.532444, + 0.03688, + 0.035594 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.945699, + "std":0.007443, + "values":[ + 0.954194, + 0.942581, + 0.940323 + ] + }, + "validation_selective_risk":{ + "mean":0.043206, + "std":7.7e-05, + "values":[ + 0.043272, + 0.043121, + 0.043225 + ] + }, + "validation_risk_upper95":{ + "mean":0.049816, + "std":6.9e-05, + "values":[ + 0.049855, + 0.049736, + 0.049855 + ] + }, + "test":{ + "coverage":{ + "mean":0.877394, + "std":0.016022, + "values":[ + 0.895818, + 0.866727, + 0.869636 + ] + }, + "selective_risk":{ + "mean":0.105367, + "std":0.00341, + "values":[ + 0.102496, + 0.104468, + 0.109137 + ] + }, + "accuracy_8":{ + "mean":0.864303, + "std":0.004853, + "values":[ + 0.869455, + 0.863636, + 0.859818 + ] + }, + "oos_recall":{ + "mean":0.451667, + "std":0.024826, + "values":[ + 0.472, + 0.459, + 0.424 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.421091, + "std":0.024567, + "values":[ + 0.441225, + 0.42833, + 0.393718 + ] + }, + { + "mean":0.482613, + "std":0.024895, + "values":[ + 0.50299, + 0.489984, + 0.454864 + ] + } + ], + "oos_precision":{ + "mean":0.933029, + "std":0.005291, + "values":[ + 0.93837, + 0.932927, + 0.92779 + ] + }, + "oos_f1":{ + "mean":0.608459, + "std":0.023776, + "values":[ + 0.628077, + 0.615282, + 0.582018 + ] + }, + "oos_misroute_rate":{ + "mean":0.548333, + "std":0.024826, + "values":[ + 0.528, + 0.541, + 0.576 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.355333, + "std":0.022301, + "values":[ + 0.372, + 0.33, + 0.364 + ] + }, + "llm_call_rate":{ + "mean":0.122606, + "std":0.016022, + "values":[ + 0.104182, + 0.133273, + 0.130364 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.045127, + "std":0.005923, + "values":[ + 0.038314, + 0.049061, + 0.048005 + ] + }, + "error_recovery_rate":{ + "mean":0.285043, + "std":0.013393, + "values":[ + 0.270358, + 0.296585, + 0.288184 + ] + }, + "recoverable_caught":{ + "mean":0.537005, + "std":0.012972, + "values":[ + 0.532051, + 0.551724, + 0.527241 + ] + }, + "small_errors":{ + "mean":995.666667, + "std":65.156223, + "values":[ + 921, + 1025, + 1041 + ] + }, + "recoverable_errors":{ + "mean":529.333333, + "std":53.873308, + "values":[ + 468, + 551, + 569 + ] + }, + "recovered_errors":{ + "mean":284.333333, + "std":30.664855, + "values":[ + 249, + 304, + 300 + ] + } + } + } + } + } + } + } + }, + "bert/k100":{ + "model":"bert", + "k":100, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "bert-base-uncased-full-seed42", + "bert-base-uncased-full-seed43", + "bert-base-uncased-full-seed44" + ], + "logits_sha256":[ + "673a4c11200dd61ddd6310f174ab9fb27580cf5b512c7ba3b5db19088958bab2", + "b6963da29f5886b915d2c5626391721fbd18c7f54a68b7720bf2d10d6cc2c58f", + "0324b20d4736ca9ad913b97269100b398475ab06a2bd130f653af55de239c277" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":1.00518, + "std":0.005215, + "values":[ + 1.010961, + 1.000828, + 1.003751 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.038959, + "std":0.003381, + "values":[ + 0.035211, + 0.041778, + 0.039889 + ] + }, + "ece_8_summed":{ + "mean":0.044193, + "std":0.003876, + "values":[ + 0.03976, + 0.046942, + 0.045877 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.03796, + "std":0.004424, + "values":[ + 0.033049, + 0.041634, + 0.039198 + ] + }, + "ece_8_summed":{ + "mean":0.043522, + "std":0.004692, + "values":[ + 0.038187, + 0.04701, + 0.045368 + ] + } + } + } + }, + "selected_aggregation":[ + "argmax", + "summed", + "argmax" + ], + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.977742, + "std":0.001478, + "values":[ + 0.978065, + 0.976129, + 0.979032 + ] + }, + "summed":{ + "mean":0.977097, + "std":0.001117, + "values":[ + 0.976452, + 0.976452, + 0.978387 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.977742, + "std":0.001478 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.087939, + "std":0.003274 + }, + "accuracy_8":{ + "mean":0.912061, + "std":0.003274 + }, + "oos_recall":{ + "mean":0.569333, + "std":0.012662 + }, + "oos_recall_wilson95":[ + { + "mean":0.538441, + "std":0.012726 + }, + { + "mean":0.599695, + "std":0.012502 + } + ], + "oos_precision":{ + "mean":0.978299, + "std":0.00379 + }, + "oos_f1":{ + "mean":0.719706, + "std":0.009127 + }, + "oos_misroute_rate":{ + "mean":0.430667, + "std":0.012662 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.430667, + "std":0.012662 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.912061, + "std":0.003274 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.946121, + "std":0.000915 + }, + "oos_recall":{ + "mean":0.725333, + "std":0.002517 + }, + "oos_recall_wilson95":[ + { + "mean":0.696846, + "std":0.002585 + }, + { + "mean":0.752096, + "std":0.002429 + } + ], + "oos_precision":{ + "mean":0.986847, + "std":0.000816 + }, + "oos_f1":{ + "mean":0.836118, + "std":0.001946 + }, + "oos_misroute_rate":{ + "mean":0.274667, + "std":0.002517 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.087939, + "std":0.003274 + }, + "llm_cost_usd_per_1k":{ + "mean":0.032447, + "std":0.001205 + }, + "error_recovery_rate":{ + "mean":0.38697, + "std":0.014899 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":187.333333, + "std":13.868429 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.002528, + "std":0.000388 + }, + "oos_auroc":{ + "mean":0.650947, + "std":0.027838 + }, + "oos_auprc":{ + "mean":0.152444, + "std":0.028922 + } + }, + "test":{ + "aurc":{ + "mean":0.011015, + "std":0.000457 + }, + "oos_auroc":{ + "mean":0.767273, + "std":0.003236 + }, + "oos_auprc":{ + "mean":0.608839, + "std":0.00264 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.002477, + "std":0.000459 + }, + "oos_auroc":{ + "mean":0.654403, + "std":0.028686 + }, + "oos_auprc":{ + "mean":0.161638, + "std":0.027709 + } + }, + "test":{ + "aurc":{ + "mean":0.010774, + "std":0.000488 + }, + "oos_auroc":{ + "mean":0.770376, + "std":0.003411 + }, + "oos_auprc":{ + "mean":0.632029, + "std":0.001985 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.002863, + "std":0.000521 + }, + "oos_auroc":{ + "mean":0.64135, + "std":0.018587 + }, + "oos_auprc":{ + "mean":0.13683, + "std":0.017569 + } + }, + "test":{ + "aurc":{ + "mean":0.013789, + "std":0.001335 + }, + "oos_auroc":{ + "mean":0.754398, + "std":0.005547 + }, + "oos_auprc":{ + "mean":0.533315, + "std":0.009152 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.002522, + "std":0.000388 + }, + "oos_auroc":{ + "mean":0.65105, + "std":0.027977 + }, + "oos_auprc":{ + "mean":0.152366, + "std":0.028825 + } + }, + "test":{ + "aurc":{ + "mean":0.011008, + "std":0.000461 + }, + "oos_auroc":{ + "mean":0.767316, + "std":0.003229 + }, + "oos_auprc":{ + "mean":0.60914, + "std":0.002861 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"entropy", + "hybrid":{ + "tau":{ + "mean":-2.315738, + "std":0.129475 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.987957, + "std":0.001343 + }, + "validation_selective_risk":{ + "mean":0.015673, + "std":2.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.019815, + "std":2.7e-05 + }, + "test":{ + "coverage":{ + "mean":0.955576, + "std":0.004947 + }, + "selective_risk":{ + "mean":0.06076, + "std":0.002276 + }, + "accuracy_8":{ + "mean":0.919636, + "std":0.003623 + }, + "oos_recall":{ + "mean":0.608333, + "std":0.017474 + }, + "oos_recall_wilson95":[ + { + "mean":0.577733, + "std":0.017639 + }, + { + "mean":0.638104, + "std":0.017175 + } + ], + "oos_precision":{ + "mean":0.974944, + "std":0.004441 + }, + "oos_f1":{ + "mean":0.74908, + "std":0.012812 + }, + "oos_misroute_rate":{ + "mean":0.391667, + "std":0.017474 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.276, + "std":0.013077 + }, + "llm_call_rate":{ + "mean":0.044424, + "std":0.004947 + }, + "llm_cost_usd_per_1k":{ + "mean":0.016393, + "std":0.001819 + }, + "error_recovery_rate":{ + "mean":0.146165, + "std":0.032207 + }, + "recoverable_caught":{ + "mean":0.376894, + "std":0.077407 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":71.0, + "std":17.521415 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.469849, + "std":0.029246 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.986344, + "std":0.000672 + }, + "validation_selective_risk":{ + "mean":0.015698, + "std":1.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.019847, + "std":1.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.952121, + "std":0.004987 + }, + "selective_risk":{ + "mean":0.058172, + "std":0.004038 + }, + "accuracy_8":{ + "mean":0.920545, + "std":0.003333 + }, + "oos_recall":{ + "mean":0.613333, + "std":0.015948 + }, + "oos_recall_wilson95":[ + { + "mean":0.582782, + "std":0.016106 + }, + { + "mean":0.643017, + "std":0.015667 + } + ], + "oos_precision":{ + "mean":0.972596, + "std":0.004088 + }, + "oos_f1":{ + "mean":0.752164, + "std":0.010911 + }, + "oos_misroute_rate":{ + "mean":0.386667, + "std":0.015948 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.265667, + "std":0.020599 + }, + "llm_call_rate":{ + "mean":0.047879, + "std":0.004987 + }, + "llm_cost_usd_per_1k":{ + "mean":0.017673, + "std":0.00183 + }, + "error_recovery_rate":{ + "mean":0.154798, + "std":0.019379 + }, + "recoverable_caught":{ + "mean":0.40008, + "std":0.048312 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":75.0, + "std":11.135529 + } + } + }, + "entropy":{ + "tau":{ + "mean":-2.315738, + "std":0.129475 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.987957, + "std":0.001343 + }, + "validation_selective_risk":{ + "mean":0.015673, + "std":2.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.019815, + "std":2.7e-05 + }, + "test":{ + "coverage":{ + "mean":0.955576, + "std":0.004947 + }, + "selective_risk":{ + "mean":0.06076, + "std":0.002276 + }, + "accuracy_8":{ + "mean":0.919636, + "std":0.003623 + }, + "oos_recall":{ + "mean":0.608333, + "std":0.017474 + }, + "oos_recall_wilson95":[ + { + "mean":0.577733, + "std":0.017639 + }, + { + "mean":0.638104, + "std":0.017175 + } + ], + "oos_precision":{ + "mean":0.974944, + "std":0.004441 + }, + "oos_f1":{ + "mean":0.74908, + "std":0.012812 + }, + "oos_misroute_rate":{ + "mean":0.391667, + "std":0.017474 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.276, + "std":0.013077 + }, + "llm_call_rate":{ + "mean":0.044424, + "std":0.004947 + }, + "llm_cost_usd_per_1k":{ + "mean":0.016393, + "std":0.001819 + }, + "error_recovery_rate":{ + "mean":0.146165, + "std":0.032207 + }, + "recoverable_caught":{ + "mean":0.376894, + "std":0.077407 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":71.0, + "std":17.521415 + } + } + }, + "margin":{ + "tau":{ + "mean":1.002529, + "std":0.143212 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.979355, + "std":0.002957 + }, + "validation_selective_risk":{ + "mean":0.0157, + "std":0.000146 + }, + "validation_risk_upper95":{ + "mean":0.019866, + "std":0.000157 + }, + "test":{ + "coverage":{ + "mean":0.951879, + "std":0.003745 + }, + "selective_risk":{ + "mean":0.060934, + "std":0.002274 + }, + "accuracy_8":{ + "mean":0.919636, + "std":0.0025 + }, + "oos_recall":{ + "mean":0.609, + "std":0.008544 + }, + "oos_recall_wilson95":[ + { + "mean":0.578396, + "std":0.00863 + }, + { + "mean":0.63877, + "std":0.008393 + } + ], + "oos_precision":{ + "mean":0.973393, + "std":0.003335 + }, + "oos_f1":{ + "mean":0.749207, + "std":0.005737 + }, + "oos_misroute_rate":{ + "mean":0.391, + "std":0.008544 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.282, + "std":0.00755 + }, + "llm_call_rate":{ + "mean":0.048121, + "std":0.003745 + }, + "llm_cost_usd_per_1k":{ + "mean":0.017766, + "std":0.00138 + }, + "error_recovery_rate":{ + "mean":0.142364, + "std":0.013956 + }, + "recoverable_caught":{ + "mean":0.367351, + "std":0.022198 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":69.0, + "std":9.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.467533, + "std":0.030148 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.986344, + "std":0.000672 + }, + "validation_selective_risk":{ + "mean":0.015698, + "std":1.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.019847, + "std":1.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.952061, + "std":0.004777 + }, + "selective_risk":{ + "mean":0.058112, + "std":0.004089 + }, + "accuracy_8":{ + "mean":0.920545, + "std":0.003333 + }, + "oos_recall":{ + "mean":0.613333, + "std":0.015948 + }, + "oos_recall_wilson95":[ + { + "mean":0.582782, + "std":0.016106 + }, + { + "mean":0.643017, + "std":0.015667 + } + ], + "oos_precision":{ + "mean":0.972596, + "std":0.004088 + }, + "oos_f1":{ + "mean":0.752164, + "std":0.010911 + }, + "oos_misroute_rate":{ + "mean":0.386667, + "std":0.015948 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.265333, + "std":0.020793 + }, + "llm_call_rate":{ + "mean":0.047939, + "std":0.004777 + }, + "llm_cost_usd_per_1k":{ + "mean":0.017695, + "std":0.001752 + }, + "error_recovery_rate":{ + "mean":0.154798, + "std":0.019379 + }, + "recoverable_caught":{ + "mean":0.40008, + "std":0.048312 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":75.0, + "std":11.135529 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "margin", + "entropy" + ], + "hybrid":{ + "tau":{ + "mean":-2.778161, + "std":2.424787 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001478 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001617 + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.00042 + }, + "selective_risk":{ + "mean":0.087613, + "std":0.003119 + }, + "accuracy_8":{ + "mean":0.912182, + "std":0.003318 + }, + "oos_recall":{ + "mean":0.57, + "std":0.012767 + }, + "oos_recall_wilson95":[ + { + "mean":0.539112, + "std":0.012833 + }, + { + "mean":0.600353, + "std":0.012604 + } + ], + "oos_precision":{ + "mean":0.978323, + "std":0.003788 + }, + "oos_f1":{ + "mean":0.720245, + "std":0.009213 + }, + "oos_misroute_rate":{ + "mean":0.43, + "std":0.012767 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.428667, + "std":0.011372 + }, + "llm_call_rate":{ + "mean":0.000424, + "std":0.00042 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000156, + "std":0.000154 + }, + "error_recovery_rate":{ + "mean":0.002064, + "std":0.002018 + }, + "recoverable_caught":{ + "mean":0.005288, + "std":0.005046 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":1.0, + "std":1.0 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.102443, + "std":0.027654 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001478 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001617 + }, + "test":{ + "coverage":{ + "mean":0.999273, + "std":0.000656 + }, + "selective_risk":{ + "mean":0.087337, + "std":0.003167 + }, + "accuracy_8":{ + "mean":0.912485, + "std":0.003203 + }, + "oos_recall":{ + "mean":0.571333, + "std":0.012342 + }, + "oos_recall_wilson95":[ + { + "mean":0.540451, + "std":0.012408 + }, + { + "mean":0.60167, + "std":0.012182 + } + ], + "oos_precision":{ + "mean":0.978371, + "std":0.003799 + }, + "oos_f1":{ + "mean":0.721326, + "std":0.008864 + }, + "oos_misroute_rate":{ + "mean":0.428667, + "std":0.012342 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.427333, + "std":0.011719 + }, + "llm_call_rate":{ + "mean":0.000727, + "std":0.000656 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000267, + "std":0.00024 + }, + "error_recovery_rate":{ + "mean":0.004806, + "std":0.003045 + }, + "recoverable_caught":{ + "mean":0.012321, + "std":0.007457 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":2.333333, + "std":1.527525 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.073649, + "std":0.186905 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001478 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001617 + }, + "test":{ + "coverage":{ + "mean":0.999273, + "std":0.000945 + }, + "selective_risk":{ + "mean":0.087398, + "std":0.002936 + }, + "accuracy_8":{ + "mean":0.912242, + "std":0.003259 + }, + "oos_recall":{ + "mean":0.570333, + "std":0.012342 + }, + "oos_recall_wilson95":[ + { + "mean":0.539446, + "std":0.012407 + }, + { + "mean":0.600682, + "std":0.012183 + } + ], + "oos_precision":{ + "mean":0.978334, + "std":0.003804 + }, + "oos_f1":{ + "mean":0.720518, + "std":0.008877 + }, + "oos_misroute_rate":{ + "mean":0.429667, + "std":0.012342 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.427333, + "std":0.009866 + }, + "llm_call_rate":{ + "mean":0.000727, + "std":0.000945 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000268, + "std":0.000348 + }, + "error_recovery_rate":{ + "mean":0.002736, + "std":0.003065 + }, + "recoverable_caught":{ + "mean":0.006963, + "std":0.007603 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":1.333333, + "std":1.527525 + } + } + }, + "margin":{ + "tau":{ + "mean":0.00821, + "std":0.011253 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001478 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001617 + }, + "test":{ + "coverage":{ + "mean":0.999455, + "std":0.000481 + }, + "selective_risk":{ + "mean":0.087623, + "std":0.003243 + }, + "accuracy_8":{ + "mean":0.912182, + "std":0.003318 + }, + "oos_recall":{ + "mean":0.57, + "std":0.012767 + }, + "oos_recall_wilson95":[ + { + "mean":0.539112, + "std":0.012833 + }, + { + "mean":0.600353, + "std":0.012604 + } + ], + "oos_precision":{ + "mean":0.978323, + "std":0.003788 + }, + "oos_f1":{ + "mean":0.720245, + "std":0.009213 + }, + "oos_misroute_rate":{ + "mean":0.43, + "std":0.012767 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.428667, + "std":0.012097 + }, + "llm_call_rate":{ + "mean":0.000545, + "std":0.000481 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000202, + "std":0.000178 + }, + "error_recovery_rate":{ + "mean":0.002064, + "std":0.002018 + }, + "recoverable_caught":{ + "mean":0.005288, + "std":0.005046 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":1.0, + "std":1.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.101626, + "std":0.02803 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001478 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001617 + }, + "test":{ + "coverage":{ + "mean":0.999273, + "std":0.000656 + }, + "selective_risk":{ + "mean":0.087337, + "std":0.003167 + }, + "accuracy_8":{ + "mean":0.912485, + "std":0.003203 + }, + "oos_recall":{ + "mean":0.571333, + "std":0.012342 + }, + "oos_recall_wilson95":[ + { + "mean":0.540451, + "std":0.012408 + }, + { + "mean":0.60167, + "std":0.012182 + } + ], + "oos_precision":{ + "mean":0.978371, + "std":0.003799 + }, + "oos_f1":{ + "mean":0.721326, + "std":0.008864 + }, + "oos_misroute_rate":{ + "mean":0.428667, + "std":0.012342 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.427333, + "std":0.011719 + }, + "llm_call_rate":{ + "mean":0.000727, + "std":0.000656 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000267, + "std":0.00024 + }, + "error_recovery_rate":{ + "mean":0.004806, + "std":0.003045 + }, + "recoverable_caught":{ + "mean":0.012321, + "std":0.007457 + }, + "small_errors":{ + "mean":483.666667, + "std":18.009257 + }, + "recoverable_errors":{ + "mean":187.333333, + "std":13.868429 + }, + "recovered_errors":{ + "mean":2.333333, + "std":1.527525 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.977097, + "std":0.001117 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.091879, + "std":0.003354 + }, + "accuracy_8":{ + "mean":0.908121, + "std":0.003354 + }, + "oos_recall":{ + "mean":0.547333, + "std":0.014572 + }, + "oos_recall_wilson95":[ + { + "mean":0.516368, + "std":0.014599 + }, + { + "mean":0.577936, + "std":0.014433 + } + ], + "oos_precision":{ + "mean":0.979209, + "std":0.003891 + }, + "oos_f1":{ + "mean":0.702079, + "std":0.011019 + }, + "oos_misroute_rate":{ + "mean":0.452667, + "std":0.014572 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.452667, + "std":0.014572 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.908121, + "std":0.003354 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.944, + "std":0.000656 + }, + "oos_recall":{ + "mean":0.713667, + "std":0.002309 + }, + "oos_recall_wilson95":[ + { + "mean":0.684873, + "std":0.002368 + }, + { + "mean":0.740825, + "std":0.002234 + } + ], + "oos_precision":{ + "mean":0.987091, + "std":0.001598 + }, + "oos_f1":{ + "mean":0.828399, + "std":0.001911 + }, + "oos_misroute_rate":{ + "mean":0.286333, + "std":0.002309 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.091879, + "std":0.003354 + }, + "llm_cost_usd_per_1k":{ + "mean":0.033901, + "std":0.001232 + }, + "error_recovery_rate":{ + "mean":0.390128, + "std":0.015332 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":197.333333, + "std":14.843629 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.002117, + "std":0.000587 + }, + "oos_auroc":{ + "mean":0.767116, + "std":0.028865 + }, + "oos_auprc":{ + "mean":0.224162, + "std":0.033342 + } + }, + "test":{ + "aurc":{ + "mean":0.011877, + "std":0.000773 + }, + "oos_auroc":{ + "mean":0.847075, + "std":0.007393 + }, + "oos_auprc":{ + "mean":0.703962, + "std":0.005498 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.002074, + "std":0.000573 + }, + "oos_auroc":{ + "mean":0.772813, + "std":0.028835 + }, + "oos_auprc":{ + "mean":0.229172, + "std":0.028623 + } + }, + "test":{ + "aurc":{ + "mean":0.011824, + "std":0.000777 + }, + "oos_auroc":{ + "mean":0.851204, + "std":0.006348 + }, + "oos_auprc":{ + "mean":0.714613, + "std":0.007515 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.002649, + "std":0.000539 + }, + "oos_auroc":{ + "mean":0.746887, + "std":0.043098 + }, + "oos_auprc":{ + "mean":0.207294, + "std":0.027757 + } + }, + "test":{ + "aurc":{ + "mean":0.013526, + "std":0.000366 + }, + "oos_auroc":{ + "mean":0.825131, + "std":0.025384 + }, + "oos_auprc":{ + "mean":0.667517, + "std":0.010965 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.002115, + "std":0.000589 + }, + "oos_auroc":{ + "mean":0.767342, + "std":0.029094 + }, + "oos_auprc":{ + "mean":0.224249, + "std":0.033269 + } + }, + "test":{ + "aurc":{ + "mean":0.011884, + "std":0.000768 + }, + "oos_auroc":{ + "mean":0.84718, + "std":0.007375 + }, + "oos_auprc":{ + "mean":0.704118, + "std":0.005664 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "entropy", + "msp", + "entropy" + ], + "hybrid":{ + "tau":{ + "mean":-0.502294, + "std":1.03685 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.988602, + "std":0.00149 + }, + "validation_selective_risk":{ + "mean":0.015662, + "std":2.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.019802, + "std":3e-05 + }, + "test":{ + "coverage":{ + "mean":0.946485, + "std":0.004034 + }, + "selective_risk":{ + "mean":0.058207, + "std":0.003196 + }, + "accuracy_8":{ + "mean":0.919333, + "std":0.003918 + }, + "oos_recall":{ + "mean":0.605333, + "std":0.016773 + }, + "oos_recall_wilson95":[ + { + "mean":0.574703, + "std":0.016921 + }, + { + "mean":0.635157, + "std":0.016496 + } + ], + "oos_precision":{ + "mean":0.976867, + "std":0.002371 + }, + "oos_f1":{ + "mean":0.747389, + "std":0.012976 + }, + "oos_misroute_rate":{ + "mean":0.394667, + "std":0.016773 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.265, + "std":0.01179 + }, + "llm_call_rate":{ + "mean":0.053515, + "std":0.004034 + }, + "llm_cost_usd_per_1k":{ + "mean":0.019729, + "std":0.001499 + }, + "error_recovery_rate":{ + "mean":0.190521, + "std":0.00728 + }, + "recoverable_caught":{ + "mean":0.488543, + "std":0.01577 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":96.333333, + "std":6.429101 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.656132, + "std":0.038524 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.98828, + "std":0.000931 + }, + "validation_selective_risk":{ + "mean":0.015668, + "std":1.5e-05 + }, + "validation_risk_upper95":{ + "mean":0.019808, + "std":1.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.947394, + "std":0.005575 + }, + "selective_risk":{ + "mean":0.058913, + "std":0.003888 + }, + "accuracy_8":{ + "mean":0.919152, + "std":0.004099 + }, + "oos_recall":{ + "mean":0.604, + "std":0.016703 + }, + "oos_recall_wilson95":[ + { + "mean":0.573357, + "std":0.016854 + }, + { + "mean":0.633847, + "std":0.016425 + } + ], + "oos_precision":{ + "mean":0.977379, + "std":0.004508 + }, + "oos_f1":{ + "mean":0.746511, + "std":0.012551 + }, + "oos_misroute_rate":{ + "mean":0.396, + "std":0.016703 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.271, + "std":0.014422 + }, + "llm_call_rate":{ + "mean":0.052606, + "std":0.005575 + }, + "llm_cost_usd_per_1k":{ + "mean":0.019394, + "std":0.002064 + }, + "error_recovery_rate":{ + "mean":0.186047, + "std":0.006597 + }, + "recoverable_caught":{ + "mean":0.477484, + "std":0.028266 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":94.0, + "std":4.358899 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.065452, + "std":0.065831 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.988495, + "std":0.001591 + }, + "validation_selective_risk":{ + "mean":0.015664, + "std":2.5e-05 + }, + "validation_risk_upper95":{ + "mean":0.019804, + "std":3.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.945394, + "std":0.005825 + }, + "selective_risk":{ + "mean":0.057567, + "std":0.002757 + }, + "accuracy_8":{ + "mean":0.919818, + "std":0.003102 + }, + "oos_recall":{ + "mean":0.606667, + "std":0.014468 + }, + "oos_recall_wilson95":[ + { + "mean":0.576046, + "std":0.0146 + }, + { + "mean":0.636471, + "std":0.014226 + } + ], + "oos_precision":{ + "mean":0.977459, + "std":0.002513 + }, + "oos_f1":{ + "mean":0.748595, + "std":0.010902 + }, + "oos_misroute_rate":{ + "mean":0.393333, + "std":0.014468 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.261, + "std":0.008185 + }, + "llm_call_rate":{ + "mean":0.054606, + "std":0.005825 + }, + "llm_cost_usd_per_1k":{ + "mean":0.020134, + "std":0.002164 + }, + "error_recovery_rate":{ + "mean":0.19564, + "std":0.01113 + }, + "recoverable_caught":{ + "mean":0.501241, + "std":0.008971 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":99.0, + "std":9.165151 + } + } + }, + "margin":{ + "tau":{ + "mean":1.30315, + "std":0.111191 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.987634, + "std":0.001221 + }, + "validation_selective_risk":{ + "mean":0.015678, + "std":1.9e-05 + }, + "validation_risk_upper95":{ + "mean":0.019821, + "std":2.4e-05 + }, + "test":{ + "coverage":{ + "mean":0.951576, + "std":0.003527 + }, + "selective_risk":{ + "mean":0.061901, + "std":0.004344 + }, + "accuracy_8":{ + "mean":0.918303, + "std":0.003638 + }, + "oos_recall":{ + "mean":0.598, + "std":0.016 + }, + "oos_recall_wilson95":[ + { + "mean":0.567303, + "std":0.016136 + }, + { + "mean":0.627947, + "std":0.015742 + } + ], + "oos_precision":{ + "mean":0.978265, + "std":0.004634 + }, + "oos_f1":{ + "mean":0.742157, + "std":0.011382 + }, + "oos_misroute_rate":{ + "mean":0.402, + "std":0.016 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.290333, + "std":0.017616 + }, + "llm_call_rate":{ + "mean":0.048424, + "std":0.003527 + }, + "llm_cost_usd_per_1k":{ + "mean":0.017852, + "std":0.001306 + }, + "error_recovery_rate":{ + "mean":0.175579, + "std":0.00508 + }, + "recoverable_caught":{ + "mean":0.450857, + "std":0.030944 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":88.666667, + "std":1.154701 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.652866, + "std":0.039846 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.98828, + "std":0.000931 + }, + "validation_selective_risk":{ + "mean":0.015668, + "std":1.5e-05 + }, + "validation_risk_upper95":{ + "mean":0.019808, + "std":1.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.947394, + "std":0.005575 + }, + "selective_risk":{ + "mean":0.058977, + "std":0.00378 + }, + "accuracy_8":{ + "mean":0.919091, + "std":0.004012 + }, + "oos_recall":{ + "mean":0.603667, + "std":0.016258 + }, + "oos_recall_wilson95":[ + { + "mean":0.57302, + "std":0.016403 + }, + { + "mean":0.63352, + "std":0.015989 + } + ], + "oos_precision":{ + "mean":0.977365, + "std":0.004527 + }, + "oos_f1":{ + "mean":0.746257, + "std":0.012245 + }, + "oos_misroute_rate":{ + "mean":0.396333, + "std":0.016258 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.271333, + "std":0.013868 + }, + "llm_call_rate":{ + "mean":0.052606, + "std":0.005575 + }, + "llm_cost_usd_per_1k":{ + "mean":0.019394, + "std":0.002064 + }, + "error_recovery_rate":{ + "mean":0.186047, + "std":0.006597 + }, + "recoverable_caught":{ + "mean":0.477484, + "std":0.028266 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":94.0, + "std":4.358899 + } + } + } + } + }, + "0.05":{ + "selected_signal":"entropy", + "hybrid":{ + "tau":{ + "mean":-1.938575, + "std":0.040906 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022903, + "std":0.001117 + }, + "validation_risk_upper95":{ + "mean":0.027755, + "std":0.001222 + }, + "test":{ + "coverage":{ + "mean":0.999455, + "std":0.000364 + }, + "selective_risk":{ + "mean":0.091444, + "std":0.003307 + }, + "accuracy_8":{ + "mean":0.908364, + "std":0.003196 + }, + "oos_recall":{ + "mean":0.548667, + "std":0.013013 + }, + "oos_recall_wilson95":[ + { + "mean":0.517703, + "std":0.013042 + }, + { + "mean":0.579258, + "std":0.012884 + } + ], + "oos_precision":{ + "mean":0.979251, + "std":0.00394 + }, + "oos_f1":{ + "mean":0.703204, + "std":0.009689 + }, + "oos_misroute_rate":{ + "mean":0.451333, + "std":0.013013 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.45, + "std":0.013528 + }, + "llm_call_rate":{ + "mean":0.000545, + "std":0.000364 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000202, + "std":0.000133 + }, + "error_recovery_rate":{ + "mean":0.003246, + "std":0.003973 + }, + "recoverable_caught":{ + "mean":0.008191, + "std":0.0098 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":1.666667, + "std":2.081666 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.216662, + "std":0.002193 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022903, + "std":0.001117 + }, + "validation_risk_upper95":{ + "mean":0.027755, + "std":0.001222 + }, + "test":{ + "coverage":{ + "mean":0.999697, + "std":0.00021 + }, + "selective_risk":{ + "mean":0.091664, + "std":0.003564 + }, + "accuracy_8":{ + "mean":0.908182, + "std":0.003406 + }, + "oos_recall":{ + "mean":0.547667, + "std":0.014503 + }, + "oos_recall_wilson95":[ + { + "mean":0.516702, + "std":0.014533 + }, + { + "mean":0.578266, + "std":0.014362 + } + ], + "oos_precision":{ + "mean":0.97922, + "std":0.003891 + }, + "oos_f1":{ + "mean":0.702357, + "std":0.010945 + }, + "oos_misroute_rate":{ + "mean":0.452333, + "std":0.014503 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.451333, + "std":0.015503 + }, + "llm_call_rate":{ + "mean":0.000303, + "std":0.00021 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000113, + "std":7.8e-05 + }, + "error_recovery_rate":{ + "mean":0.001327, + "std":0.001151 + }, + "recoverable_caught":{ + "mean":0.003429, + "std":0.002994 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":0.666667, + "std":0.57735 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.938575, + "std":0.040906 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022903, + "std":0.001117 + }, + "validation_risk_upper95":{ + "mean":0.027755, + "std":0.001222 + }, + "test":{ + "coverage":{ + "mean":0.999455, + "std":0.000364 + }, + "selective_risk":{ + "mean":0.091444, + "std":0.003307 + }, + "accuracy_8":{ + "mean":0.908364, + "std":0.003196 + }, + "oos_recall":{ + "mean":0.548667, + "std":0.013013 + }, + "oos_recall_wilson95":[ + { + "mean":0.517703, + "std":0.013042 + }, + { + "mean":0.579258, + "std":0.012884 + } + ], + "oos_precision":{ + "mean":0.979251, + "std":0.00394 + }, + "oos_f1":{ + "mean":0.703204, + "std":0.009689 + }, + "oos_misroute_rate":{ + "mean":0.451333, + "std":0.013013 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.45, + "std":0.013528 + }, + "llm_call_rate":{ + "mean":0.000545, + "std":0.000364 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000202, + "std":0.000133 + }, + "error_recovery_rate":{ + "mean":0.003246, + "std":0.003973 + }, + "recoverable_caught":{ + "mean":0.008191, + "std":0.0098 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":1.666667, + "std":2.081666 + } + } + }, + "margin":{ + "tau":{ + "mean":0.027924, + "std":0.023718 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022903, + "std":0.001117 + }, + "validation_risk_upper95":{ + "mean":0.027755, + "std":0.001222 + }, + "test":{ + "coverage":{ + "mean":0.998667, + "std":0.001001 + }, + "selective_risk":{ + "mean":0.090972, + "std":0.00269 + }, + "accuracy_8":{ + "mean":0.908364, + "std":0.003217 + }, + "oos_recall":{ + "mean":0.548667, + "std":0.014295 + }, + "oos_recall_wilson95":[ + { + "mean":0.517704, + "std":0.014322 + }, + { + "mean":0.579257, + "std":0.014158 + } + ], + "oos_precision":{ + "mean":0.979257, + "std":0.003899 + }, + "oos_f1":{ + "mean":0.703191, + "std":0.01079 + }, + "oos_misroute_rate":{ + "mean":0.451333, + "std":0.014295 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.447, + "std":0.010817 + }, + "llm_call_rate":{ + "mean":0.001333, + "std":0.001001 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000492, + "std":0.00037 + }, + "error_recovery_rate":{ + "mean":0.003254, + "std":0.004088 + }, + "recoverable_caught":{ + "mean":0.008221, + "std":0.010391 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":1.666667, + "std":2.081666 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.215666, + "std":0.003216 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022903, + "std":0.001117 + }, + "validation_risk_upper95":{ + "mean":0.027755, + "std":0.001222 + }, + "test":{ + "coverage":{ + "mean":0.999697, + "std":0.00021 + }, + "selective_risk":{ + "mean":0.091664, + "std":0.003564 + }, + "accuracy_8":{ + "mean":0.908182, + "std":0.003406 + }, + "oos_recall":{ + "mean":0.547667, + "std":0.014503 + }, + "oos_recall_wilson95":[ + { + "mean":0.516702, + "std":0.014533 + }, + { + "mean":0.578266, + "std":0.014362 + } + ], + "oos_precision":{ + "mean":0.97922, + "std":0.003891 + }, + "oos_f1":{ + "mean":0.702357, + "std":0.010945 + }, + "oos_misroute_rate":{ + "mean":0.452333, + "std":0.014503 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.451333, + "std":0.015503 + }, + "llm_call_rate":{ + "mean":0.000303, + "std":0.00021 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000113, + "std":7.8e-05 + }, + "error_recovery_rate":{ + "mean":0.001327, + "std":0.001151 + }, + "recoverable_caught":{ + "mean":0.003429, + "std":0.002994 + }, + "small_errors":{ + "mean":505.333333, + "std":18.448125 + }, + "recoverable_errors":{ + "mean":197.333333, + "std":14.843629 + }, + "recovered_errors":{ + "mean":0.666667, + "std":0.57735 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.089455, + "std":0.005273, + "values":[ + 0.084182, + 0.094727, + 0.089455 + ] + }, + "accuracy_8":{ + "mean":0.910545, + "std":0.005273, + "values":[ + 0.915818, + 0.905273, + 0.910545 + ] + }, + "oos_recall":{ + "mean":0.560667, + "std":0.026083, + "values":[ + 0.583, + 0.532, + 0.567 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.52976, + "std":0.026165, + "values":[ + 0.552179, + 0.50101, + 0.536091 + ] + }, + { + "mean":0.591109, + "std":0.025801, + "values":[ + 0.613186, + 0.562745, + 0.597397 + ] + } + ], + "oos_precision":{ + "mean":0.978622, + "std":0.004318, + "values":[ + 0.974916, + 0.983364, + 0.977586 + ] + }, + "oos_f1":{ + "mean":0.712615, + "std":0.020093, + "values":[ + 0.729662, + 0.690461, + 0.717722 + ] + }, + "oos_misroute_rate":{ + "mean":0.439333, + "std":0.026083, + "values":[ + 0.417, + 0.468, + 0.433 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.439333, + "std":0.026083, + "values":[ + 0.417, + 0.468, + 0.433 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":492.0, + "std":29.0, + "values":[ + 463, + 521, + 492 + ] + }, + "recoverable_errors":{ + "mean":191.0, + "std":19.0, + "values":[ + 172, + 210, + 191 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.910545, + "std":0.005273, + "values":[ + 0.915818, + 0.905273, + 0.910545 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.945273, + "std":0.001818, + "values":[ + 0.947091, + 0.943455, + 0.945273 + ] + }, + "oos_recall":{ + "mean":0.720667, + "std":0.008737, + "values":[ + 0.728, + 0.711, + 0.723 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.692058, + "std":0.008966, + "values":[ + 0.699586, + 0.682139, + 0.694449 + ] + }, + { + "mean":0.747586, + "std":0.008441, + "values":[ + 0.754669, + 0.738246, + 0.749844 + ] + } + ], + "oos_precision":{ + "mean":0.986759, + "std":0.000899, + "values":[ + 0.987788, + 0.98613, + 0.986357 + ] + }, + "oos_f1":{ + "mean":0.832961, + "std":0.006108, + "values":[ + 0.838227, + 0.826264, + 0.834391 + ] + }, + "oos_misroute_rate":{ + "mean":0.279333, + "std":0.008737, + "values":[ + 0.272, + 0.289, + 0.277 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.089455, + "std":0.005273, + "values":[ + 0.084182, + 0.094727, + 0.089455 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.033006, + "std":0.001941, + "values":[ + 0.031065, + 0.034948, + 0.033006 + ] + }, + "error_recovery_rate":{ + "mean":0.387591, + "std":0.0158, + "values":[ + 0.37149, + 0.403071, + 0.388211 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":492.0, + "std":29.0, + "values":[ + 463, + 521, + 492 + ] + }, + "recoverable_errors":{ + "mean":191.0, + "std":19.0, + "values":[ + 172, + 210, + 191 + ] + }, + "recovered_errors":{ + "mean":191.0, + "std":19.0, + "values":[ + 172, + 210, + 191 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.002484, + "std":0.000325, + "values":[ + 0.002529, + 0.002783, + 0.002139 + ] + }, + "oos_auroc":{ + "mean":0.684601, + "std":0.051137, + "values":[ + 0.683083, + 0.73648, + 0.63424 + ] + }, + "oos_auprc":{ + "mean":0.176517, + "std":0.03039, + "values":[ + 0.184956, + 0.201796, + 0.1428 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.011342, + "std":0.001024, + "values":[ + 0.010771, + 0.012525, + 0.010731 + ] + }, + "oos_auroc":{ + "mean":0.792217, + "std":0.04251, + "values":[ + 0.764607, + 0.84117, + 0.770874 + ] + }, + "oos_auprc":{ + "mean":0.639503, + "std":0.050471, + "values":[ + 0.610295, + 0.697782, + 0.610431 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.002388, + "std":0.000313, + "values":[ + 0.00235, + 0.002718, + 0.002095 + ] + }, + "oos_auroc":{ + "mean":0.689006, + "std":0.053218, + "values":[ + 0.687493, + 0.742963, + 0.63656 + ] + }, + "oos_auprc":{ + "mean":0.183783, + "std":0.027595, + "values":[ + 0.192678, + 0.205833, + 0.152837 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.011148, + "std":0.001134, + "values":[ + 0.01056, + 0.012455, + 0.01043 + ] + }, + "oos_auroc":{ + "mean":0.795579, + "std":0.043316, + "values":[ + 0.767268, + 0.845444, + 0.774026 + ] + }, + "oos_auprc":{ + "mean":0.657601, + "std":0.042317, + "values":[ + 0.633362, + 0.706464, + 0.632977 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.002971, + "std":0.000527, + "values":[ + 0.003422, + 0.003099, + 0.002392 + ] + }, + "oos_auroc":{ + "mean":0.664871, + "std":0.03585, + "values":[ + 0.662803, + 0.70171, + 0.6301 + ] + }, + "oos_auprc":{ + "mean":0.150779, + "std":0.036636, + "values":[ + 0.146356, + 0.189426, + 0.116555 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.013496, + "std":0.001065, + "values":[ + 0.012281, + 0.013936, + 0.01427 + ] + }, + "oos_auroc":{ + "mean":0.775297, + "std":0.036971, + "values":[ + 0.759728, + 0.817507, + 0.748658 + ] + }, + "oos_auprc":{ + "mean":0.573539, + "std":0.076675, + "values":[ + 0.535587, + 0.661789, + 0.523242 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.002479, + "std":0.000324, + "values":[ + 0.002513, + 0.002784, + 0.002139 + ] + }, + "oos_auroc":{ + "mean":0.684706, + "std":0.051132, + "values":[ + 0.683347, + 0.736503, + 0.634267 + ] + }, + "oos_auprc":{ + "mean":0.176447, + "std":0.03041, + "values":[ + 0.184772, + 0.201827, + 0.142741 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.011336, + "std":0.00103, + "values":[ + 0.010758, + 0.012525, + 0.010725 + ] + }, + "oos_auroc":{ + "mean":0.792265, + "std":0.042492, + "values":[ + 0.76468, + 0.841199, + 0.770918 + ] + }, + "oos_auprc":{ + "mean":0.639798, + "std":0.050246, + "values":[ + 0.610976, + 0.697817, + 0.610601 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "entropy", + "msp", + "entropy" + ], + "hybrid":{ + "tau":{ + "mean":-1.339347, + "std":1.764824, + "values":[ + -2.465052, + 0.694625, + -2.247615 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.988387, + "std":0.000645, + "values":[ + 0.988387, + 0.987742, + 0.989032 + ] + }, + "validation_selective_risk":{ + "mean":0.015666, + "std":1e-05, + "values":[ + 0.015666, + 0.015676, + 0.015656 + ] + }, + "validation_risk_upper95":{ + "mean":0.019806, + "std":1.3e-05, + "values":[ + 0.019806, + 0.019819, + 0.019793 + ] + }, + "test":{ + "coverage":{ + "mean":0.951939, + "std":0.009553, + "values":[ + 0.961273, + 0.942182, + 0.952364 + ] + }, + "selective_risk":{ + "mean":0.059781, + "std":0.001027, + "values":[ + 0.060526, + 0.060208, + 0.05861 + ] + }, + "accuracy_8":{ + "mean":0.919455, + "std":0.003938, + "values":[ + 0.921636, + 0.914909, + 0.921818 + ] + }, + "oos_recall":{ + "mean":0.607333, + "std":0.01914, + "values":[ + 0.613, + 0.586, + 0.623 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.576726, + "std":0.019316, + "values":[ + 0.582434, + 0.555199, + 0.592545 + ] + }, + { + "mean":0.637119, + "std":0.018817, + "values":[ + 0.642701, + 0.616143, + 0.652513 + ] + } + ], + "oos_precision":{ + "mean":0.974364, + "std":0.003835, + "values":[ + 0.969937, + 0.976667, + 0.976489 + ] + }, + "oos_f1":{ + "mean":0.748136, + "std":0.014344, + "values":[ + 0.751225, + 0.7325, + 0.760684 + ] + }, + "oos_misroute_rate":{ + "mean":0.392667, + "std":0.01914, + "values":[ + 0.387, + 0.414, + 0.377 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.272667, + "std":0.010693, + "values":[ + 0.282, + 0.275, + 0.261 + ] + }, + "llm_call_rate":{ + "mean":0.048061, + "std":0.009553, + "values":[ + 0.038727, + 0.057818, + 0.047636 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.017731, + "std":0.003517, + "values":[ + 0.014298, + 0.021326, + 0.017568 + ] + }, + "error_recovery_rate":{ + "mean":0.161117, + "std":0.04078, + "values":[ + 0.114471, + 0.190019, + 0.178862 + ] + }, + "recoverable_caught":{ + "mean":0.413434, + "std":0.091344, + "values":[ + 0.30814, + 0.471429, + 0.460733 + ] + }, + "small_errors":{ + "mean":492.0, + "std":29.0, + "values":[ + 463, + 521, + 492 + ] + }, + "recoverable_errors":{ + "mean":191.0, + "std":19.0, + "values":[ + 172, + 210, + 191 + ] + }, + "recovered_errors":{ + "mean":80.0, + "std":24.020824, + "values":[ + 53, + 99, + 88 + ] + } + } + } + }, + "0.05":{ + "selected_signal":"entropy", + "hybrid":{ + "tau":{ + "mean":-3.416748, + "std":1.319133, + "values":[ + -4.129125, + -1.894583, + -4.226536 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "validation_selective_risk":{ + "mean":0.022151, + "std":0.001304, + "values":[ + 0.021935, + 0.023548, + 0.020968 + ] + }, + "validation_risk_upper95":{ + "mean":0.026932, + "std":0.001426, + "values":[ + 0.026698, + 0.028461, + 0.025637 + ] + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.00042, + "values":[ + 0.999818, + 0.999091, + 0.999818 + ] + }, + "selective_risk":{ + "mean":0.089069, + "std":0.004948, + "values":[ + 0.084015, + 0.093904, + 0.089289 + ] + }, + "accuracy_8":{ + "mean":0.910848, + "std":0.005007, + "values":[ + 0.916, + 0.906, + 0.910545 + ] + }, + "oos_recall":{ + "mean":0.562333, + "std":0.024338, + "values":[ + 0.584, + 0.536, + 0.567 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.531429, + "std":0.024423, + "values":[ + 0.553186, + 0.505011, + 0.536091 + ] + }, + { + "mean":0.59276, + "std":0.024066, + "values":[ + 0.614172, + 0.566713, + 0.597397 + ] + } + ], + "oos_precision":{ + "mean":0.978677, + "std":0.004367, + "values":[ + 0.974958, + 0.983486, + 0.977586 + ] + }, + "oos_f1":{ + "mean":0.71401, + "std":0.018583, + "values":[ + 0.730457, + 0.693851, + 0.717722 + ] + }, + "oos_misroute_rate":{ + "mean":0.437667, + "std":0.024338, + "values":[ + 0.416, + 0.464, + 0.433 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.437, + "std":0.023896, + "values":[ + 0.416, + 0.463, + 0.432 + ] + }, + "llm_call_rate":{ + "mean":0.000424, + "std":0.00042, + "values":[ + 0.000182, + 0.000909, + 0.000182 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.000156, + "std":0.000154, + "values":[ + 6.7e-05, + 0.000335, + 6.7e-05 + ] + }, + "error_recovery_rate":{ + "mean":0.003279, + "std":0.003959, + "values":[ + 0.00216, + 0.007678, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.008287, + "std":0.009762, + "values":[ + 0.005814, + 0.019048, + 0.0 + ] + }, + "small_errors":{ + "mean":492.0, + "std":29.0, + "values":[ + 463, + 521, + 492 + ] + }, + "recoverable_errors":{ + "mean":191.0, + "std":19.0, + "values":[ + 172, + 210, + 191 + ] + }, + "recovered_errors":{ + "mean":1.666667, + "std":2.081666, + "values":[ + 1, + 4, + 0 + ] + } + } + } + } + } + } + } + }, + "bert/k25":{ + "model":"bert", + "k":25, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "bert-base-uncased-k25-seed42", + "bert-base-uncased-k25-seed43", + "bert-base-uncased-k25-seed44" + ], + "logits_sha256":[ + "5397a2237935d24210a1b8e47ed837f1807db8beb0a4c18f31d24d0681bef1cc", + "cf40024e86a42cfe34703b4b04e58696d93a03f3408a44f820960b11d554d626", + "c4de18e8e6d5e96011ab20178ec669bd2462c6aae44cd2fd40ae8ba3ce077f30" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":0.369512, + "std":0.010566, + "values":[ + 0.373258, + 0.377694, + 0.357582 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.564944, + "std":0.007385, + "values":[ + 0.56297, + 0.558747, + 0.573116 + ] + }, + "ece_8_summed":{ + "mean":0.356595, + "std":0.004553, + "values":[ + 0.355778, + 0.352506, + 0.361502 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.027257, + "std":0.003283, + "values":[ + 0.023481, + 0.029438, + 0.028852 + ] + }, + "ece_8_summed":{ + "mean":0.064251, + "std":0.004428, + "values":[ + 0.059225, + 0.067579, + 0.065948 + ] + } + } + } + }, + "selected_aggregation":"argmax", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.960215, + "std":0.004938, + "values":[ + 0.958387, + 0.956452, + 0.965806 + ] + }, + "summed":{ + "mean":0.958602, + "std":0.005011, + "values":[ + 0.957097, + 0.954516, + 0.964194 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.960215, + "std":0.004938 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.134848, + "std":0.00646 + }, + "accuracy_8":{ + "mean":0.865152, + "std":0.00646 + }, + "oos_recall":{ + "mean":0.38, + "std":0.031765 + }, + "oos_recall_wilson95":[ + { + "mean":0.350474, + "std":0.031182 + }, + { + "mean":0.410445, + "std":0.032104 + } + ], + "oos_precision":{ + "mean":0.946847, + "std":0.0166 + }, + "oos_f1":{ + "mean":0.541852, + "std":0.032906 + }, + "oos_misroute_rate":{ + "mean":0.62, + "std":0.031765 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.62, + "std":0.031765 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.865152, + "std":0.00646 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.926848, + "std":0.00488 + }, + "oos_recall":{ + "mean":0.644333, + "std":0.022679 + }, + "oos_recall_wilson95":[ + { + "mean":0.614186, + "std":0.023031 + }, + { + "mean":0.673376, + "std":0.022153 + } + ], + "oos_precision":{ + "mean":0.973275, + "std":0.001725 + }, + "oos_f1":{ + "mean":0.775221, + "std":0.01687 + }, + "oos_misroute_rate":{ + "mean":0.355667, + "std":0.022679 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.134848, + "std":0.00646 + }, + "llm_cost_usd_per_1k":{ + "mean":0.049708, + "std":0.002394 + }, + "error_recovery_rate":{ + "mean":0.457792, + "std":0.01535 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":339.333333, + "std":14.0119 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.005429, + "std":0.000698 + }, + "oos_auroc":{ + "mean":0.840822, + "std":0.04945 + }, + "oos_auprc":{ + "mean":0.228994, + "std":0.061069 + } + }, + "test":{ + "aurc":{ + "mean":0.026062, + "std":0.0027 + }, + "oos_auroc":{ + "mean":0.867145, + "std":0.039979 + }, + "oos_auprc":{ + "mean":0.670917, + "std":0.043944 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.007651, + "std":0.001916 + }, + "oos_auroc":{ + "mean":0.865678, + "std":0.03287 + }, + "oos_auprc":{ + "mean":0.262996, + "std":0.037097 + } + }, + "test":{ + "aurc":{ + "mean":0.026704, + "std":0.004117 + }, + "oos_auroc":{ + "mean":0.887984, + "std":0.034346 + }, + "oos_auprc":{ + "mean":0.700662, + "std":0.033693 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.007781, + "std":0.000958 + }, + "oos_auroc":{ + "mean":0.72182, + "std":0.054946 + }, + "oos_auprc":{ + "mean":0.079083, + "std":0.023201 + } + }, + "test":{ + "aurc":{ + "mean":0.039503, + "std":0.002811 + }, + "oos_auroc":{ + "mean":0.748336, + "std":0.036923 + }, + "oos_auprc":{ + "mean":0.376571, + "std":0.04421 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.00605, + "std":0.000737 + }, + "oos_auroc":{ + "mean":0.772183, + "std":0.057661 + }, + "oos_auprc":{ + "mean":0.153787, + "std":0.048567 + } + }, + "test":{ + "aurc":{ + "mean":0.031943, + "std":0.00242 + }, + "oos_auroc":{ + "mean":0.806089, + "std":0.040585 + }, + "oos_auprc":{ + "mean":0.55661, + "std":0.046223 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.113052, + "std":0.020931 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.920753, + "std":0.02376 + }, + "validation_selective_risk":{ + "mean":0.01553, + "std":0.000135 + }, + "validation_risk_upper95":{ + "mean":0.019822, + "std":9.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.833091, + "std":0.038152 + }, + "selective_risk":{ + "mean":0.047798, + "std":0.010293 + }, + "accuracy_8":{ + "mean":0.897273, + "std":0.002857 + }, + "oos_recall":{ + "mean":0.548333, + "std":0.033501 + }, + "oos_recall_wilson95":[ + { + "mean":0.517408, + "std":0.033554 + }, + { + "mean":0.578888, + "std":0.033192 + } + ], + "oos_precision":{ + "mean":0.941416, + "std":0.010232 + }, + "oos_f1":{ + "mean":0.692507, + "std":0.025348 + }, + "oos_misroute_rate":{ + "mean":0.451667, + "std":0.033501 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.168, + "std":0.064969 + }, + "llm_call_rate":{ + "mean":0.166909, + "std":0.038152 + }, + "llm_cost_usd_per_1k":{ + "mean":0.061509, + "std":0.014067 + }, + "error_recovery_rate":{ + "mean":0.341498, + "std":0.037118 + }, + "recoverable_caught":{ + "mean":0.745048, + "std":0.063887 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":253.333333, + "std":31.085902 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.113052, + "std":0.020931 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.920753, + "std":0.02376 + }, + "validation_selective_risk":{ + "mean":0.01553, + "std":0.000135 + }, + "validation_risk_upper95":{ + "mean":0.019822, + "std":9.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.833091, + "std":0.038152 + }, + "selective_risk":{ + "mean":0.047798, + "std":0.010293 + }, + "accuracy_8":{ + "mean":0.897273, + "std":0.002857 + }, + "oos_recall":{ + "mean":0.548333, + "std":0.033501 + }, + "oos_recall_wilson95":[ + { + "mean":0.517408, + "std":0.033554 + }, + { + "mean":0.578888, + "std":0.033192 + } + ], + "oos_precision":{ + "mean":0.941416, + "std":0.010232 + }, + "oos_f1":{ + "mean":0.692507, + "std":0.025348 + }, + "oos_misroute_rate":{ + "mean":0.451667, + "std":0.033501 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.168, + "std":0.064969 + }, + "llm_call_rate":{ + "mean":0.166909, + "std":0.038152 + }, + "llm_cost_usd_per_1k":{ + "mean":0.061509, + "std":0.014067 + }, + "error_recovery_rate":{ + "mean":0.341498, + "std":0.037118 + }, + "recoverable_caught":{ + "mean":0.745048, + "std":0.063887 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":253.333333, + "std":31.085902 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.30588, + "std":0.096815 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.893226, + "std":0.050807 + }, + "validation_selective_risk":{ + "mean":0.015521, + "std":0.000201 + }, + "validation_risk_upper95":{ + "mean":0.019889, + "std":8.5e-05 + }, + "test":{ + "coverage":{ + "mean":0.795212, + "std":0.056142 + }, + "selective_risk":{ + "mean":0.039576, + "std":0.008344 + }, + "accuracy_8":{ + "mean":0.896061, + "std":0.005095 + }, + "oos_recall":{ + "mean":0.555333, + "std":0.021127 + }, + "oos_recall_wilson95":[ + { + "mean":0.524399, + "std":0.021186 + }, + { + "mean":0.585845, + "std":0.020906 + } + ], + "oos_precision":{ + "mean":0.931043, + "std":0.01072 + }, + "oos_f1":{ + "mean":0.69545, + "std":0.014147 + }, + "oos_misroute_rate":{ + "mean":0.444667, + "std":0.021127 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.128333, + "std":0.051782 + }, + "llm_call_rate":{ + "mean":0.204788, + "std":0.056142 + }, + "llm_cost_usd_per_1k":{ + "mean":0.075447, + "std":0.02069 + }, + "error_recovery_rate":{ + "mean":0.368294, + "std":0.030581 + }, + "recoverable_caught":{ + "mean":0.804068, + "std":0.05317 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":273.333333, + "std":29.365513 + } + } + }, + "margin":{ + "tau":{ + "mean":1.288321, + "std":0.239123 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.838495, + "std":0.051524 + }, + "validation_selective_risk":{ + "mean":0.015385, + "std":7.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.019895, + "std":7.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.791455, + "std":0.055252 + }, + "selective_risk":{ + "mean":0.068209, + "std":0.007607 + }, + "accuracy_8":{ + "mean":0.888727, + "std":0.00466 + }, + "oos_recall":{ + "mean":0.531, + "std":0.026889 + }, + "oos_recall_wilson95":[ + { + "mean":0.50004, + "std":0.026863 + }, + { + "mean":0.561723, + "std":0.026708 + } + ], + "oos_precision":{ + "mean":0.919417, + "std":0.022499 + }, + "oos_f1":{ + "mean":0.672746, + "std":0.019914 + }, + "oos_misroute_rate":{ + "mean":0.469, + "std":0.026889 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.251333, + "std":0.052013 + }, + "llm_call_rate":{ + "mean":0.208545, + "std":0.055252 + }, + "llm_cost_usd_per_1k":{ + "mean":0.076942, + "std":0.020377 + }, + "error_recovery_rate":{ + "mean":0.293291, + "std":0.038791 + }, + "recoverable_caught":{ + "mean":0.640276, + "std":0.078608 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":218.0, + "std":36.055513 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.590213, + "std":0.071483 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.892473, + "std":0.044153 + }, + "validation_selective_risk":{ + "mean":0.015419, + "std":9.2e-05 + }, + "validation_risk_upper95":{ + "mean":0.019776, + "std":7.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.832485, + "std":0.050364 + }, + "selective_risk":{ + "mean":0.059958, + "std":0.006174 + }, + "accuracy_8":{ + "mean":0.894, + "std":0.005197 + }, + "oos_recall":{ + "mean":0.538, + "std":0.024269 + }, + "oos_recall_wilson95":[ + { + "mean":0.507037, + "std":0.02427 + }, + { + "mean":0.568672, + "std":0.024083 + } + ], + "oos_precision":{ + "mean":0.933579, + "std":0.021082 + }, + "oos_f1":{ + "mean":0.682246, + "std":0.01784 + }, + "oos_misroute_rate":{ + "mean":0.462, + "std":0.024269 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.223333, + "std":0.046112 + }, + "llm_call_rate":{ + "mean":0.167515, + "std":0.050364 + }, + "llm_cost_usd_per_1k":{ + "mean":0.06177, + "std":0.018579 + }, + "error_recovery_rate":{ + "mean":0.3029, + "std":0.032665 + }, + "recoverable_caught":{ + "mean":0.662241, + "std":0.075676 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":225.333333, + "std":34.645827 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.023213, + "std":0.001684 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039785, + "std":0.004938 + }, + "validation_risk_upper95":{ + "mean":0.045964, + "std":0.005282 + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.000278 + }, + "selective_risk":{ + "mean":0.13448, + "std":0.006697 + }, + "accuracy_8":{ + "mean":0.865333, + "std":0.006546 + }, + "oos_recall":{ + "mean":0.381, + "std":0.032741 + }, + "oos_recall_wilson95":[ + { + "mean":0.351458, + "std":0.032145 + }, + { + "mean":0.411453, + "std":0.033087 + } + ], + "oos_precision":{ + "mean":0.946974, + "std":0.016553 + }, + "oos_f1":{ + "mean":0.542861, + "std":0.033846 + }, + "oos_misroute_rate":{ + "mean":0.619, + "std":0.032741 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.617667, + "std":0.033005 + }, + "llm_call_rate":{ + "mean":0.000424, + "std":0.000278 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000156, + "std":0.000103 + }, + "error_recovery_rate":{ + "mean":0.001371, + "std":0.001416 + }, + "recoverable_caught":{ + "mean":0.002929, + "std":0.002987 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":1.0, + "std":1.0 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.023213, + "std":0.001684 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039785, + "std":0.004938 + }, + "validation_risk_upper95":{ + "mean":0.045964, + "std":0.005282 + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.000278 + }, + "selective_risk":{ + "mean":0.13448, + "std":0.006697 + }, + "accuracy_8":{ + "mean":0.865333, + "std":0.006546 + }, + "oos_recall":{ + "mean":0.381, + "std":0.032741 + }, + "oos_recall_wilson95":[ + { + "mean":0.351458, + "std":0.032145 + }, + { + "mean":0.411453, + "std":0.033087 + } + ], + "oos_precision":{ + "mean":0.946974, + "std":0.016553 + }, + "oos_f1":{ + "mean":0.542861, + "std":0.033846 + }, + "oos_misroute_rate":{ + "mean":0.619, + "std":0.032741 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.617667, + "std":0.033005 + }, + "llm_call_rate":{ + "mean":0.000424, + "std":0.000278 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000156, + "std":0.000103 + }, + "error_recovery_rate":{ + "mean":0.001371, + "std":0.001416 + }, + "recoverable_caught":{ + "mean":0.002929, + "std":0.002987 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":1.0, + "std":1.0 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.860973, + "std":0.045221 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039785, + "std":0.004938 + }, + "validation_risk_upper95":{ + "mean":0.045964, + "std":0.005282 + }, + "test":{ + "coverage":{ + "mean":0.999273, + "std":0.00126 + }, + "selective_risk":{ + "mean":0.134335, + "std":0.007227 + }, + "accuracy_8":{ + "mean":0.865455, + "std":0.006909 + }, + "oos_recall":{ + "mean":0.382, + "std":0.035086 + }, + "oos_recall_wilson95":[ + { + "mean":0.352446, + "std":0.034457 + }, + { + "mean":0.412457, + "std":0.035445 + } + ], + "oos_precision":{ + "mean":0.947046, + "std":0.016771 + }, + "oos_f1":{ + "mean":0.543825, + "std":0.036239 + }, + "oos_misroute_rate":{ + "mean":0.618, + "std":0.035086 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.616667, + "std":0.037314 + }, + "llm_call_rate":{ + "mean":0.000727, + "std":0.00126 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000268, + "std":0.000464 + }, + "error_recovery_rate":{ + "mean":0.002829, + "std":0.0049 + }, + "recoverable_caught":{ + "mean":0.00597, + "std":0.010341 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":2.0, + "std":3.464102 + } + } + }, + "margin":{ + "tau":{ + "mean":0.001853, + "std":0.001543 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039785, + "std":0.004938 + }, + "validation_risk_upper95":{ + "mean":0.045964, + "std":0.005282 + }, + "test":{ + "coverage":{ + "mean":0.999758, + "std":0.000278 + }, + "selective_risk":{ + "mean":0.134821, + "std":0.006395 + }, + "accuracy_8":{ + "mean":0.865152, + "std":0.00646 + }, + "oos_recall":{ + "mean":0.38, + "std":0.031765 + }, + "oos_recall_wilson95":[ + { + "mean":0.350474, + "std":0.031182 + }, + { + "mean":0.410445, + "std":0.032104 + } + ], + "oos_precision":{ + "mean":0.946078, + "std":0.017932 + }, + "oos_f1":{ + "mean":0.541725, + "std":0.032969 + }, + "oos_misroute_rate":{ + "mean":0.62, + "std":0.031765 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.62, + "std":0.031765 + }, + "llm_call_rate":{ + "mean":0.000242, + "std":0.000278 + }, + "llm_cost_usd_per_1k":{ + "mean":8.9e-05, + "std":0.000102 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.0858, + "std":0.007969 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039785, + "std":0.004938 + }, + "validation_risk_upper95":{ + "mean":0.045964, + "std":0.005282 + }, + "test":{ + "coverage":{ + "mean":0.999515, + "std":0.000458 + }, + "selective_risk":{ + "mean":0.134428, + "std":0.006618 + }, + "accuracy_8":{ + "mean":0.865455, + "std":0.006364 + }, + "oos_recall":{ + "mean":0.381667, + "std":0.032517 + }, + "oos_recall_wilson95":[ + { + "mean":0.352111, + "std":0.031924 + }, + { + "mean":0.412128, + "std":0.03286 + } + ], + "oos_precision":{ + "mean":0.947093, + "std":0.016347 + }, + "oos_f1":{ + "mean":0.543558, + "std":0.033534 + }, + "oos_misroute_rate":{ + "mean":0.618333, + "std":0.032517 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.617333, + "std":0.034152 + }, + "llm_call_rate":{ + "mean":0.000485, + "std":0.000458 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000179, + "std":0.000169 + }, + "error_recovery_rate":{ + "mean":0.002228, + "std":0.001997 + }, + "recoverable_caught":{ + "mean":0.004807, + "std":0.004344 + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331 + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119 + }, + "recovered_errors":{ + "mean":1.666667, + "std":1.527525 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.958602, + "std":0.005011 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.140545, + "std":0.007252 + }, + "accuracy_8":{ + "mean":0.859455, + "std":0.007252 + }, + "oos_recall":{ + "mean":0.344333, + "std":0.036964 + }, + "oos_recall_wilson95":[ + { + "mean":0.315594, + "std":0.036131 + }, + { + "mean":0.374264, + "std":0.037514 + } + ], + "oos_precision":{ + "mean":0.955496, + "std":0.01022 + }, + "oos_f1":{ + "mean":0.505517, + "std":0.039706 + }, + "oos_misroute_rate":{ + "mean":0.655667, + "std":0.036964 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.655667, + "std":0.036964 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.859455, + "std":0.007252 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.925758, + "std":0.005095 + }, + "oos_recall":{ + "mean":0.634667, + "std":0.02203 + }, + "oos_recall_wilson95":[ + { + "mean":0.604381, + "std":0.022345 + }, + { + "mean":0.663921, + "std":0.021547 + } + ], + "oos_precision":{ + "mean":0.975345, + "std":0.003659 + }, + "oos_f1":{ + "mean":0.768842, + "std":0.017083 + }, + "oos_misroute_rate":{ + "mean":0.365333, + "std":0.02203 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.140545, + "std":0.007252 + }, + "llm_cost_usd_per_1k":{ + "mean":0.051801, + "std":0.00268 + }, + "error_recovery_rate":{ + "mean":0.471871, + "std":0.020688 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":364.666667, + "std":22.501852 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.012002, + "std":0.002175 + }, + "oos_auroc":{ + "mean":0.899637, + "std":0.013226 + }, + "oos_auprc":{ + "mean":0.478868, + "std":0.081363 + } + }, + "test":{ + "aurc":{ + "mean":0.043384, + "std":0.000727 + }, + "oos_auroc":{ + "mean":0.910658, + "std":0.00369 + }, + "oos_auprc":{ + "mean":0.759767, + "std":0.017169 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.012974, + "std":0.00271 + }, + "oos_auroc":{ + "mean":0.917457, + "std":0.016307 + }, + "oos_auprc":{ + "mean":0.60085, + "std":0.068688 + } + }, + "test":{ + "aurc":{ + "mean":0.04342, + "std":0.001047 + }, + "oos_auroc":{ + "mean":0.923902, + "std":0.001817 + }, + "oos_auprc":{ + "mean":0.810736, + "std":0.012913 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.006523, + "std":0.000677 + }, + "oos_auroc":{ + "mean":0.870969, + "std":0.040605 + }, + "oos_auprc":{ + "mean":0.198401, + "std":0.06129 + } + }, + "test":{ + "aurc":{ + "mean":0.029804, + "std":0.003086 + }, + "oos_auroc":{ + "mean":0.883569, + "std":0.022841 + }, + "oos_auprc":{ + "mean":0.625562, + "std":0.046754 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.005887, + "std":0.000506 + }, + "oos_auroc":{ + "mean":0.88134, + "std":0.037203 + }, + "oos_auprc":{ + "mean":0.26864, + "std":0.081473 + } + }, + "test":{ + "aurc":{ + "mean":0.027869, + "std":0.002702 + }, + "oos_auroc":{ + "mean":0.896513, + "std":0.022276 + }, + "oos_auprc":{ + "mean":0.688784, + "std":0.038843 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.794782, + "std":0.02198 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.943548, + "std":0.01346 + }, + "validation_selective_risk":{ + "mean":0.015613, + "std":9e-05 + }, + "validation_risk_upper95":{ + "mean":0.019856, + "std":0.00012 + }, + "test":{ + "coverage":{ + "mean":0.866182, + "std":0.015104 + }, + "selective_risk":{ + "mean":0.061077, + "std":0.00824 + }, + "accuracy_8":{ + "mean":0.896182, + "std":0.004468 + }, + "oos_recall":{ + "mean":0.525333, + "std":0.037072 + }, + "oos_recall_wilson95":[ + { + "mean":0.494402, + "std":0.037037 + }, + { + "mean":0.556071, + "std":0.036823 + } + ], + "oos_precision":{ + "mean":0.951848, + "std":0.009359 + }, + "oos_f1":{ + "mean":0.676457, + "std":0.030553 + }, + "oos_misroute_rate":{ + "mean":0.474667, + "std":0.037072 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.254667, + "std":0.044298 + }, + "llm_call_rate":{ + "mean":0.133818, + "std":0.015104 + }, + "llm_cost_usd_per_1k":{ + "mean":0.049254, + "std":0.005559 + }, + "error_recovery_rate":{ + "mean":0.347162, + "std":0.048784 + }, + "recoverable_caught":{ + "mean":0.733645, + "std":0.072761 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":268.333333, + "std":39.310728 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.384344, + "std":0.017124 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.864839, + "std":0.026505 + }, + "validation_selective_risk":{ + "mean":0.015415, + "std":0.000115 + }, + "validation_risk_upper95":{ + "mean":0.019847, + "std":8.5e-05 + }, + "test":{ + "coverage":{ + "mean":0.75, + "std":0.023455 + }, + "selective_risk":{ + "mean":0.045068, + "std":0.007571 + }, + "accuracy_8":{ + "mean":0.879939, + "std":0.00674 + }, + "oos_recall":{ + "mean":0.524667, + "std":0.016258 + }, + "oos_recall_wilson95":[ + { + "mean":0.49369, + "std":0.016253 + }, + { + "mean":0.555455, + "std":0.016139 + } + ], + "oos_precision":{ + "mean":0.886654, + "std":0.013678 + }, + "oos_f1":{ + "mean":0.659027, + "std":0.009478 + }, + "oos_misroute_rate":{ + "mean":0.475333, + "std":0.016258 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.160333, + "std":0.039463 + }, + "llm_call_rate":{ + "mean":0.25, + "std":0.023455 + }, + "llm_cost_usd_per_1k":{ + "mean":0.092061, + "std":0.00873 + }, + "error_recovery_rate":{ + "mean":0.395495, + "std":0.028014 + }, + "recoverable_caught":{ + "mean":0.837619, + "std":0.027975 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":305.333333, + "std":18.502252 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.674637, + "std":0.069711 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.803118, + "std":0.105839 + }, + "validation_selective_risk":{ + "mean":0.015228, + "std":0.000404 + }, + "validation_risk_upper95":{ + "mean":0.019845, + "std":0.000119 + }, + "test":{ + "coverage":{ + "mean":0.696182, + "std":0.096652 + }, + "selective_risk":{ + "mean":0.041225, + "std":0.010818 + }, + "accuracy_8":{ + "mean":0.876061, + "std":0.012672 + }, + "oos_recall":{ + "mean":0.541, + "std":0.020075 + }, + "oos_recall_wilson95":[ + { + "mean":0.510033, + "std":0.020096 + }, + { + "mean":0.571654, + "std":0.0199 + } + ], + "oos_precision":{ + "mean":0.867921, + "std":0.031026 + }, + "oos_f1":{ + "mean":0.66602, + "std":0.009458 + }, + "oos_misroute_rate":{ + "mean":0.459, + "std":0.020075 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.127667, + "std":0.066643 + }, + "llm_call_rate":{ + "mean":0.303818, + "std":0.096652 + }, + "llm_cost_usd_per_1k":{ + "mean":0.111953, + "std":0.035778 + }, + "error_recovery_rate":{ + "mean":0.407948, + "std":0.038009 + }, + "recoverable_caught":{ + "mean":0.863298, + "std":0.044294 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":315.333333, + "std":33.080709 + } + } + }, + "margin":{ + "tau":{ + "mean":2.111488, + "std":0.231205 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.930215, + "std":0.012281 + }, + "validation_selective_risk":{ + "mean":0.015604, + "std":0.000142 + }, + "validation_risk_upper95":{ + "mean":0.01988, + "std":0.000127 + }, + "test":{ + "coverage":{ + "mean":0.849697, + "std":0.020291 + }, + "selective_risk":{ + "mean":0.05987, + "std":0.008856 + }, + "accuracy_8":{ + "mean":0.894182, + "std":0.004237 + }, + "oos_recall":{ + "mean":0.528667, + "std":0.035247 + }, + "oos_recall_wilson95":[ + { + "mean":0.497728, + "std":0.035211 + }, + { + "mean":0.559386, + "std":0.035013 + } + ], + "oos_precision":{ + "mean":0.947211, + "std":0.014404 + }, + "oos_f1":{ + "mean":0.678035, + "std":0.028504 + }, + "oos_misroute_rate":{ + "mean":0.471333, + "std":0.035247 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.250667, + "std":0.046576 + }, + "llm_call_rate":{ + "mean":0.150303, + "std":0.020291 + }, + "llm_cost_usd_per_1k":{ + "mean":0.055329, + "std":0.007468 + }, + "error_recovery_rate":{ + "mean":0.35756, + "std":0.050932 + }, + "recoverable_caught":{ + "mean":0.755735, + "std":0.079793 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":276.666667, + "std":44.275652 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.794782, + "std":0.02198 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.943548, + "std":0.01346 + }, + "validation_selective_risk":{ + "mean":0.015613, + "std":9e-05 + }, + "validation_risk_upper95":{ + "mean":0.019856, + "std":0.00012 + }, + "test":{ + "coverage":{ + "mean":0.866182, + "std":0.015104 + }, + "selective_risk":{ + "mean":0.061077, + "std":0.00824 + }, + "accuracy_8":{ + "mean":0.896182, + "std":0.004468 + }, + "oos_recall":{ + "mean":0.525333, + "std":0.037072 + }, + "oos_recall_wilson95":[ + { + "mean":0.494402, + "std":0.037037 + }, + { + "mean":0.556071, + "std":0.036823 + } + ], + "oos_precision":{ + "mean":0.951848, + "std":0.009359 + }, + "oos_f1":{ + "mean":0.676457, + "std":0.030553 + }, + "oos_misroute_rate":{ + "mean":0.474667, + "std":0.037072 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.254667, + "std":0.044298 + }, + "llm_call_rate":{ + "mean":0.133818, + "std":0.015104 + }, + "llm_cost_usd_per_1k":{ + "mean":0.049254, + "std":0.005559 + }, + "error_recovery_rate":{ + "mean":0.347162, + "std":0.048784 + }, + "recoverable_caught":{ + "mean":0.733645, + "std":0.072761 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":268.333333, + "std":39.310728 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.26382, + "std":0.070489 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.99914, + "std":0.00149 + }, + "validation_selective_risk":{ + "mean":0.040682, + "std":0.004228 + }, + "validation_risk_upper95":{ + "mean":0.046927, + "std":0.004522 + }, + "test":{ + "coverage":{ + "mean":0.995455, + "std":0.006801 + }, + "selective_risk":{ + "mean":0.13779, + "std":0.003337 + }, + "accuracy_8":{ + "mean":0.861333, + "std":0.004672 + }, + "oos_recall":{ + "mean":0.354, + "std":0.033407 + }, + "oos_recall_wilson95":[ + { + "mean":0.325023, + "std":0.032669 + }, + { + "mean":0.384095, + "std":0.033888 + } + ], + "oos_precision":{ + "mean":0.955202, + "std":0.010918 + }, + "oos_f1":{ + "mean":0.515934, + "std":0.035279 + }, + "oos_misroute_rate":{ + "mean":0.646, + "std":0.033407 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.639, + "std":0.034044 + }, + "llm_call_rate":{ + "mean":0.004545, + "std":0.006801 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001671, + "std":0.002501 + }, + "error_recovery_rate":{ + "mean":0.015652, + "std":0.022603 + }, + "recoverable_caught":{ + "mean":0.032938, + "std":0.047874 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":12.666667, + "std":18.583146 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.175805, + "std":0.024723 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.998817, + "std":0.002049 + }, + "validation_selective_risk":{ + "mean":0.040696, + "std":0.004242 + }, + "validation_risk_upper95":{ + "mean":0.046944, + "std":0.004537 + }, + "test":{ + "coverage":{ + "mean":0.994303, + "std":0.008343 + }, + "selective_risk":{ + "mean":0.13937, + "std":0.005486 + }, + "accuracy_8":{ + "mean":0.860303, + "std":0.005954 + }, + "oos_recall":{ + "mean":0.348333, + "std":0.033724 + }, + "oos_recall_wilson95":[ + { + "mean":0.319488, + "std":0.032976 + }, + { + "mean":0.37834, + "std":0.034215 + } + ], + "oos_precision":{ + "mean":0.95701, + "std":0.007876 + }, + "oos_f1":{ + "mean":0.510138, + "std":0.035794 + }, + "oos_misroute_rate":{ + "mean":0.651667, + "std":0.033724 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.646667, + "std":0.030989 + }, + "llm_call_rate":{ + "mean":0.005697, + "std":0.008343 + }, + "llm_cost_usd_per_1k":{ + "mean":0.002088, + "std":0.003057 + }, + "error_recovery_rate":{ + "mean":0.009812, + "std":0.015867 + }, + "recoverable_caught":{ + "mean":0.020785, + "std":0.033501 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":8.0, + "std":13.0 + } + } + }, + "entropy":{ + "tau":{ + "mean":-2.017951, + "std":0.045982 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.997634, + "std":0.004097 + }, + "validation_selective_risk":{ + "mean":0.040748, + "std":0.004291 + }, + "validation_risk_upper95":{ + "mean":0.047003, + "std":0.004594 + }, + "test":{ + "coverage":{ + "mean":0.98903, + "std":0.015955 + }, + "selective_risk":{ + "mean":0.140585, + "std":0.006954 + }, + "accuracy_8":{ + "mean":0.858909, + "std":0.008041 + }, + "oos_recall":{ + "mean":0.340333, + "std":0.040857 + }, + "oos_recall_wilson95":[ + { + "mean":0.311706, + "std":0.039913 + }, + { + "mean":0.370183, + "std":0.04149 + } + ], + "oos_precision":{ + "mean":0.956757, + "std":0.008067 + }, + "oos_f1":{ + "mean":0.501184, + "std":0.044075 + }, + "oos_misroute_rate":{ + "mean":0.659667, + "std":0.040857 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.649333, + "std":0.032593 + }, + "llm_call_rate":{ + "mean":0.01097, + "std":0.015955 + }, + "llm_cost_usd_per_1k":{ + "mean":0.004022, + "std":0.005849 + }, + "error_recovery_rate":{ + "mean":0.007742, + "std":0.01341 + }, + "recoverable_caught":{ + "mean":0.016365, + "std":0.028345 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":6.333333, + "std":10.969655 + } + } + }, + "margin":{ + "tau":{ + "mean":0.027875, + "std":0.024652 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.99914, + "std":0.00149 + }, + "validation_selective_risk":{ + "mean":0.040682, + "std":0.004228 + }, + "validation_risk_upper95":{ + "mean":0.046927, + "std":0.004522 + }, + "test":{ + "coverage":{ + "mean":0.997636, + "std":0.00246 + }, + "selective_risk":{ + "mean":0.139002, + "std":0.006002 + }, + "accuracy_8":{ + "mean":0.860606, + "std":0.006388 + }, + "oos_recall":{ + "mean":0.349667, + "std":0.036143 + }, + "oos_recall_wilson95":[ + { + "mean":0.320798, + "std":0.03535 + }, + { + "mean":0.379686, + "std":0.03666 + } + ], + "oos_precision":{ + "mean":0.956265, + "std":0.009231 + }, + "oos_f1":{ + "mean":0.511382, + "std":0.038366 + }, + "oos_misroute_rate":{ + "mean":0.650333, + "std":0.036143 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.648, + "std":0.037041 + }, + "llm_call_rate":{ + "mean":0.002364, + "std":0.00246 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000869, + "std":0.000904 + }, + "error_recovery_rate":{ + "mean":0.009665, + "std":0.009782 + }, + "recoverable_caught":{ + "mean":0.020174, + "std":0.02069 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":7.666667, + "std":8.020806 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.26382, + "std":0.070489 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.99914, + "std":0.00149 + }, + "validation_selective_risk":{ + "mean":0.040682, + "std":0.004228 + }, + "validation_risk_upper95":{ + "mean":0.046927, + "std":0.004522 + }, + "test":{ + "coverage":{ + "mean":0.995455, + "std":0.006801 + }, + "selective_risk":{ + "mean":0.13779, + "std":0.003337 + }, + "accuracy_8":{ + "mean":0.861333, + "std":0.004672 + }, + "oos_recall":{ + "mean":0.354, + "std":0.033407 + }, + "oos_recall_wilson95":[ + { + "mean":0.325023, + "std":0.032669 + }, + { + "mean":0.384095, + "std":0.033888 + } + ], + "oos_precision":{ + "mean":0.955202, + "std":0.010918 + }, + "oos_f1":{ + "mean":0.515934, + "std":0.035279 + }, + "oos_misroute_rate":{ + "mean":0.646, + "std":0.033407 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.639, + "std":0.034044 + }, + "llm_call_rate":{ + "mean":0.004545, + "std":0.006801 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001671, + "std":0.002501 + }, + "error_recovery_rate":{ + "mean":0.015652, + "std":0.022603 + }, + "recoverable_caught":{ + "mean":0.032938, + "std":0.047874 + }, + "small_errors":{ + "mean":773.0, + "std":39.887341 + }, + "recoverable_errors":{ + "mean":364.666667, + "std":22.501852 + }, + "recovered_errors":{ + "mean":12.666667, + "std":18.583146 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.134848, + "std":0.00646, + "values":[ + 0.128545, + 0.141455, + 0.134545 + ] + }, + "accuracy_8":{ + "mean":0.865152, + "std":0.00646, + "values":[ + 0.871455, + 0.858545, + 0.865455 + ] + }, + "oos_recall":{ + "mean":0.38, + "std":0.031765, + "values":[ + 0.415, + 0.372, + 0.353 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.350474, + "std":0.031182, + "values":[ + 0.384843, + 0.342586, + 0.323993 + ] + }, + { + "mean":0.410445, + "std":0.032104, + "values":[ + 0.445808, + 0.402394, + 0.383132 + ] + } + ], + "oos_precision":{ + "mean":0.946847, + "std":0.0166, + "values":[ + 0.956221, + 0.927681, + 0.95664 + ] + }, + "oos_f1":{ + "mean":0.541852, + "std":0.032906, + "values":[ + 0.578801, + 0.531049, + 0.515705 + ] + }, + "oos_misroute_rate":{ + "mean":0.62, + "std":0.031765, + "values":[ + 0.585, + 0.628, + 0.647 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.62, + "std":0.031765, + "values":[ + 0.585, + 0.628, + 0.647 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331, + "values":[ + 707, + 778, + 740 + ] + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119, + "values":[ + 335, + 355, + 328 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.865152, + "std":0.00646, + "values":[ + 0.871455, + 0.858545, + 0.865455 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.926848, + "std":0.00488, + "values":[ + 0.932364, + 0.923091, + 0.925091 + ] + }, + "oos_recall":{ + "mean":0.644333, + "std":0.022679, + "values":[ + 0.67, + 0.636, + 0.627 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.614186, + "std":0.023031, + "values":[ + 0.640254, + 0.60571, + 0.596593 + ] + }, + { + "mean":0.673376, + "std":0.022153, + "values":[ + 0.698445, + 0.665249, + 0.656434 + ] + } + ], + "oos_precision":{ + "mean":0.973275, + "std":0.001725, + "values":[ + 0.975255, + 0.972477, + 0.972093 + ] + }, + "oos_f1":{ + "mean":0.775221, + "std":0.01687, + "values":[ + 0.794309, + 0.769045, + 0.76231 + ] + }, + "oos_misroute_rate":{ + "mean":0.355667, + "std":0.022679, + "values":[ + 0.33, + 0.364, + 0.373 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.134848, + "std":0.00646, + "values":[ + 0.128545, + 0.141455, + 0.134545 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.049708, + "std":0.002394, + "values":[ + 0.047368, + 0.052152, + 0.049604 + ] + }, + "error_recovery_rate":{ + "mean":0.457792, + "std":0.01535, + "values":[ + 0.473833, + 0.456298, + 0.443243 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331, + "values":[ + 707, + 778, + 740 + ] + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119, + "values":[ + 335, + 355, + 328 + ] + }, + "recovered_errors":{ + "mean":339.333333, + "std":14.0119, + "values":[ + 335, + 355, + 328 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.005429, + "std":0.000698, + "values":[ + 0.00588, + 0.005781, + 0.004625 + ] + }, + "oos_auroc":{ + "mean":0.840822, + "std":0.04945, + "values":[ + 0.786093, + 0.854083, + 0.88229 + ] + }, + "oos_auprc":{ + "mean":0.228994, + "std":0.061069, + "values":[ + 0.16473, + 0.235984, + 0.286268 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.026062, + "std":0.0027, + "values":[ + 0.024424, + 0.029178, + 0.024582 + ] + }, + "oos_auroc":{ + "mean":0.867145, + "std":0.039979, + "values":[ + 0.824148, + 0.874094, + 0.903195 + ] + }, + "oos_auprc":{ + "mean":0.670917, + "std":0.043944, + "values":[ + 0.624054, + 0.677498, + 0.711199 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.007651, + "std":0.001916, + "values":[ + 0.008638, + 0.008872, + 0.005443 + ] + }, + "oos_auroc":{ + "mean":0.865678, + "std":0.03287, + "values":[ + 0.827723, + 0.884547, + 0.884763 + ] + }, + "oos_auprc":{ + "mean":0.262996, + "std":0.037097, + "values":[ + 0.224335, + 0.266352, + 0.2983 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.026704, + "std":0.004117, + "values":[ + 0.025513, + 0.031286, + 0.023315 + ] + }, + "oos_auroc":{ + "mean":0.887984, + "std":0.034346, + "values":[ + 0.848682, + 0.903035, + 0.912234 + ] + }, + "oos_auprc":{ + "mean":0.700662, + "std":0.033693, + "values":[ + 0.662404, + 0.713676, + 0.725907 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.007781, + "std":0.000958, + "values":[ + 0.007897, + 0.008675, + 0.00677 + ] + }, + "oos_auroc":{ + "mean":0.72182, + "std":0.054946, + "values":[ + 0.662437, + 0.732167, + 0.770857 + ] + }, + "oos_auprc":{ + "mean":0.079083, + "std":0.023201, + "values":[ + 0.05362, + 0.099025, + 0.084603 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.039503, + "std":0.002811, + "values":[ + 0.036852, + 0.04245, + 0.039207 + ] + }, + "oos_auroc":{ + "mean":0.748336, + "std":0.036923, + "values":[ + 0.716184, + 0.740163, + 0.788661 + ] + }, + "oos_auprc":{ + "mean":0.376571, + "std":0.04421, + "values":[ + 0.340972, + 0.362683, + 0.426057 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.00605, + "std":0.000737, + "values":[ + 0.006169, + 0.00672, + 0.00526 + ] + }, + "oos_auroc":{ + "mean":0.772183, + "std":0.057661, + "values":[ + 0.711313, + 0.779253, + 0.825983 + ] + }, + "oos_auprc":{ + "mean":0.153787, + "std":0.048567, + "values":[ + 0.099547, + 0.168566, + 0.193247 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.031943, + "std":0.00242, + "values":[ + 0.029807, + 0.034572, + 0.03145 + ] + }, + "oos_auroc":{ + "mean":0.806089, + "std":0.040585, + "values":[ + 0.766774, + 0.80366, + 0.847834 + ] + }, + "oos_auprc":{ + "mean":0.55661, + "std":0.046223, + "values":[ + 0.505435, + 0.56907, + 0.595327 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.113052, + "std":0.020931, + "values":[ + 0.118972, + 0.130385, + 0.089798 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.920753, + "std":0.02376, + "values":[ + 0.917097, + 0.899032, + 0.946129 + ] + }, + "validation_selective_risk":{ + "mean":0.01553, + "std":0.000135, + "values":[ + 0.015477, + 0.015429, + 0.015684 + ] + }, + "validation_risk_upper95":{ + "mean":0.019822, + "std":9.2e-05, + "values":[ + 0.019771, + 0.019766, + 0.019928 + ] + }, + "test":{ + "coverage":{ + "mean":0.833091, + "std":0.038152, + "values":[ + 0.824182, + 0.800182, + 0.874909 + ] + }, + "selective_risk":{ + "mean":0.047798, + "std":0.010293, + "values":[ + 0.041032, + 0.042718, + 0.059643 + ] + }, + "accuracy_8":{ + "mean":0.897273, + "std":0.002857, + "values":[ + 0.899818, + 0.894182, + 0.897818 + ] + }, + "oos_recall":{ + "mean":0.548333, + "std":0.033501, + "values":[ + 0.578, + 0.555, + 0.512 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.517408, + "std":0.033554, + "values":[ + 0.547148, + 0.524046, + 0.481032 + ] + }, + { + "mean":0.578888, + "std":0.033192, + "values":[ + 0.608255, + 0.585534, + 0.542876 + ] + } + ], + "oos_precision":{ + "mean":0.941416, + "std":0.010232, + "values":[ + 0.941368, + 0.931208, + 0.951673 + ] + }, + "oos_f1":{ + "mean":0.692507, + "std":0.025348, + "values":[ + 0.716233, + 0.695489, + 0.6658 + ] + }, + "oos_misroute_rate":{ + "mean":0.451667, + "std":0.033501, + "values":[ + 0.422, + 0.445, + 0.488 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.168, + "std":0.064969, + "values":[ + 0.132, + 0.129, + 0.243 + ] + }, + "llm_call_rate":{ + "mean":0.166909, + "std":0.038152, + "values":[ + 0.175818, + 0.199818, + 0.125091 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.061509, + "std":0.014067, + "values":[ + 0.064815, + 0.073628, + 0.046084 + ] + }, + "error_recovery_rate":{ + "mean":0.341498, + "std":0.037118, + "values":[ + 0.362093, + 0.363753, + 0.298649 + ] + }, + "recoverable_caught":{ + "mean":0.745048, + "std":0.063887, + "values":[ + 0.764179, + 0.797183, + 0.67378 + ] + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331, + "values":[ + 707, + 778, + 740 + ] + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119, + "values":[ + 335, + 355, + 328 + ] + }, + "recovered_errors":{ + "mean":253.333333, + "std":31.085902, + "values":[ + 256, + 283, + 221 + ] + } + } + } + }, + "0.05":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.023213, + "std":0.001684, + "values":[ + 0.025006, + 0.021665, + 0.022969 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "validation_selective_risk":{ + "mean":0.039785, + "std":0.004938, + "values":[ + 0.041613, + 0.043548, + 0.034194 + ] + }, + "validation_risk_upper95":{ + "mean":0.045964, + "std":0.005282, + "values":[ + 0.047923, + 0.049986, + 0.039981 + ] + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.000278, + "values":[ + 0.999273, + 0.999818, + 0.999636 + ] + }, + "selective_risk":{ + "mean":0.13448, + "std":0.006697, + "values":[ + 0.127911, + 0.141298, + 0.134231 + ] + }, + "accuracy_8":{ + "mean":0.865333, + "std":0.006546, + "values":[ + 0.871818, + 0.858727, + 0.865455 + ] + }, + "oos_recall":{ + "mean":0.381, + "std":0.032741, + "values":[ + 0.417, + 0.373, + 0.353 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.351458, + "std":0.032145, + "values":[ + 0.386814, + 0.343566, + 0.323993 + ] + }, + { + "mean":0.411453, + "std":0.033087, + "values":[ + 0.447821, + 0.403407, + 0.383132 + ] + } + ], + "oos_precision":{ + "mean":0.946974, + "std":0.016553, + "values":[ + 0.956422, + 0.927861, + 0.95664 + ] + }, + "oos_f1":{ + "mean":0.542861, + "std":0.033846, + "values":[ + 0.58078, + 0.532097, + 0.515705 + ] + }, + "oos_misroute_rate":{ + "mean":0.619, + "std":0.032741, + "values":[ + 0.583, + 0.627, + 0.647 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.617667, + "std":0.033005, + "values":[ + 0.581, + 0.627, + 0.645 + ] + }, + "llm_call_rate":{ + "mean":0.000424, + "std":0.000278, + "values":[ + 0.000727, + 0.000182, + 0.000364 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.000156, + "std":0.000103, + "values":[ + 0.000268, + 6.6e-05, + 0.000134 + ] + }, + "error_recovery_rate":{ + "mean":0.001371, + "std":0.001416, + "values":[ + 0.002829, + 0.001285, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.002929, + "std":0.002987, + "values":[ + 0.00597, + 0.002817, + 0.0 + ] + }, + "small_errors":{ + "mean":741.666667, + "std":35.529331, + "values":[ + 707, + 778, + 740 + ] + }, + "recoverable_errors":{ + "mean":339.333333, + "std":14.0119, + "values":[ + 335, + 355, + 328 + ] + }, + "recovered_errors":{ + "mean":1.0, + "std":1.0, + "values":[ + 2, + 1, + 0 + ] + } + } + } + } + } + } + } + }, + "bert/k5":{ + "model":"bert", + "k":5, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "bert-base-uncased-k5-seed42", + "bert-base-uncased-k5-seed43", + "bert-base-uncased-k5-seed44" + ], + "logits_sha256":[ + "12e3d723ecbdeda888a687bd3b539ef5b8bc07515bd72f28293b0dfc4314f94d", + "78dd86d8b8a700aa8986c7bf23fef4a99720a8f226df3e6d0d21d896cae3aa8a", + "175a274248600f6f85681b2e8a824dd9bf17f281782044110b1c3facb10f1da5" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":0.290057, + "std":0.013649, + "values":[ + 0.305797, + 0.28288, + 0.281494 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.567615, + "std":0.023812, + "values":[ + 0.572152, + 0.588831, + 0.541861 + ] + }, + "ece_8_summed":{ + "mean":0.323516, + "std":0.007745, + "values":[ + 0.330895, + 0.315452, + 0.324201 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.03174, + "std":0.011219, + "values":[ + 0.019495, + 0.034202, + 0.041523 + ] + }, + "ece_8_summed":{ + "mean":0.04096, + "std":0.003972, + "values":[ + 0.039296, + 0.038091, + 0.045494 + ] + } + } + } + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.901935, + "std":0.003489, + "values":[ + 0.904839, + 0.902903, + 0.898065 + ] + }, + "summed":{ + "mean":0.907634, + "std":0.005411, + "values":[ + 0.913871, + 0.904194, + 0.904839 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.901935, + "std":0.003489 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.211879, + "std":0.019711 + }, + "accuracy_8":{ + "mean":0.788121, + "std":0.019711 + }, + "oos_recall":{ + "mean":0.227333, + "std":0.071808 + }, + "oos_recall_wilson95":[ + { + "mean":0.20282, + "std":0.06829 + }, + { + "mean":0.253934, + "std":0.074776 + } + ], + "oos_precision":{ + "mean":0.802204, + "std":0.108608 + }, + "oos_f1":{ + "mean":0.352773, + "std":0.099274 + }, + "oos_misroute_rate":{ + "mean":0.772667, + "std":0.071808 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.772667, + "std":0.071808 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.788121, + "std":0.019711 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.913818, + "std":0.00317 + }, + "oos_recall":{ + "mean":0.614667, + "std":0.007572 + }, + "oos_recall_wilson95":[ + { + "mean":0.584121, + "std":0.007651 + }, + { + "mean":0.644335, + "std":0.007435 + } + ], + "oos_precision":{ + "mean":0.943203, + "std":0.008192 + }, + "oos_f1":{ + "mean":0.744291, + "std":0.008089 + }, + "oos_misroute_rate":{ + "mean":0.385333, + "std":0.007572 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.211879, + "std":0.019711 + }, + "llm_cost_usd_per_1k":{ + "mean":0.078092, + "std":0.007231 + }, + "error_recovery_rate":{ + "mean":0.590659, + "std":0.043699 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":691.333333, + "std":116.328558 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.023211, + "std":0.004275 + }, + "oos_auroc":{ + "mean":0.838501, + "std":0.03907 + }, + "oos_auprc":{ + "mean":0.236372, + "std":0.032566 + } + }, + "test":{ + "aurc":{ + "mean":0.05592, + "std":0.009065 + }, + "oos_auroc":{ + "mean":0.844578, + "std":0.027881 + }, + "oos_auprc":{ + "mean":0.582521, + "std":0.025837 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.031752, + "std":0.00664 + }, + "oos_auroc":{ + "mean":0.843903, + "std":0.021151 + }, + "oos_auprc":{ + "mean":0.213426, + "std":0.007246 + } + }, + "test":{ + "aurc":{ + "mean":0.070119, + "std":0.01054 + }, + "oos_auroc":{ + "mean":0.839518, + "std":0.018718 + }, + "oos_auprc":{ + "mean":0.568223, + "std":0.031436 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.029349, + "std":0.004732 + }, + "oos_auroc":{ + "mean":0.732013, + "std":0.027721 + }, + "oos_auprc":{ + "mean":0.080924, + "std":0.006724 + } + }, + "test":{ + "aurc":{ + "mean":0.074286, + "std":0.013113 + }, + "oos_auroc":{ + "mean":0.728994, + "std":0.02712 + }, + "oos_auprc":{ + "mean":0.336696, + "std":0.018082 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.024278, + "std":0.003827 + }, + "oos_auroc":{ + "mean":0.799874, + "std":0.038036 + }, + "oos_auprc":{ + "mean":0.202168, + "std":0.0381 + } + }, + "test":{ + "aurc":{ + "mean":0.060555, + "std":0.011034 + }, + "oos_auroc":{ + "mean":0.803014, + "std":0.029566 + }, + "oos_auprc":{ + "mean":0.515436, + "std":0.023025 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp", + "msp", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":0.340009, + "std":0.451489 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.501935, + "std":0.08175 + }, + "validation_selective_risk":{ + "mean":0.013876, + "std":0.000447 + }, + "validation_risk_upper95":{ + "mean":0.019715, + "std":7.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.440303, + "std":0.070045 + }, + "selective_risk":{ + "mean":0.024806, + "std":0.001565 + }, + "accuracy_8":{ + "mean":0.859939, + "std":0.005923 + }, + "oos_recall":{ + "mean":0.568, + "std":0.003 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.003014 + }, + { + "mean":0.598384, + "std":0.002963 + } + ], + "oos_precision":{ + "mean":0.808042, + "std":0.01179 + }, + "oos_f1":{ + "mean":0.667074, + "std":0.006016 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.003 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.029, + "std":0.004359 + }, + "llm_call_rate":{ + "mean":0.559697, + "std":0.070045 + }, + "llm_cost_usd_per_1k":{ + "mean":0.206577, + "std":0.025852 + }, + "error_recovery_rate":{ + "mean":0.567041, + "std":0.049499 + }, + "recoverable_caught":{ + "mean":0.959461, + "std":0.017616 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":664.333333, + "std":121.582619 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.084164, + "std":0.008489 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.475699, + "std":0.12643 + }, + "validation_selective_risk":{ + "mean":0.013637, + "std":0.000855 + }, + "validation_risk_upper95":{ + "mean":0.019718, + "std":8.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.416909, + "std":0.10951 + }, + "selective_risk":{ + "mean":0.021136, + "std":0.007145 + }, + "accuracy_8":{ + "mean":0.858182, + "std":0.007643 + }, + "oos_recall":{ + "mean":0.569333, + "std":0.001528 + }, + "oos_recall_wilson95":[ + { + "mean":0.538435, + "std":0.001535 + }, + { + "mean":0.599701, + "std":0.001508 + } + ], + "oos_precision":{ + "mean":0.803688, + "std":0.019254 + }, + "oos_f1":{ + "mean":0.666451, + "std":0.007054 + }, + "oos_misroute_rate":{ + "mean":0.430667, + "std":0.001528 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.023333, + "std":0.01222 + }, + "llm_call_rate":{ + "mean":0.583091, + "std":0.10951 + }, + "llm_cost_usd_per_1k":{ + "mean":0.21523, + "std":0.040452 + }, + "error_recovery_rate":{ + "mean":0.571185, + "std":0.056648 + }, + "recoverable_caught":{ + "mean":0.965926, + "std":0.026547 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":669.666667, + "std":130.714702 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.544834, + "std":0.053081 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.315376, + "std":0.162087 + }, + "validation_selective_risk":{ + "mean":0.011097, + "std":0.003449 + }, + "validation_risk_upper95":{ + "mean":0.018936, + "std":0.001019 + }, + "test":{ + "coverage":{ + "mean":0.272424, + "std":0.135479 + }, + "selective_risk":{ + "mean":0.02134, + "std":0.010894 + }, + "accuracy_8":{ + "mean":0.840303, + "std":0.007287 + }, + "oos_recall":{ + "mean":0.569667, + "std":0.002082 + }, + "oos_recall_wilson95":[ + { + "mean":0.53877, + "std":0.002092 + }, + { + "mean":0.60003, + "std":0.002056 + } + ], + "oos_precision":{ + "mean":0.77917, + "std":0.017662 + }, + "oos_f1":{ + "mean":0.658107, + "std":0.007527 + }, + "oos_misroute_rate":{ + "mean":0.430333, + "std":0.002082 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.013667, + "std":0.010066 + }, + "llm_call_rate":{ + "mean":0.727576, + "std":0.135479 + }, + "llm_cost_usd_per_1k":{ + "mean":0.268692, + "std":0.05018 + }, + "error_recovery_rate":{ + "mean":0.574467, + "std":0.056432 + }, + "recoverable_caught":{ + "mean":0.971466, + "std":0.022742 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":673.333333, + "std":129.962815 + } + } + }, + "margin":{ + "tau":{ + "mean":3.179675, + "std":0.512565 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.405806, + "std":0.086714 + }, + "validation_selective_risk":{ + "mean":0.013133, + "std":0.000825 + }, + "validation_risk_upper95":{ + "mean":0.019644, + "std":0.000154 + }, + "test":{ + "coverage":{ + "mean":0.368788, + "std":0.078471 + }, + "selective_risk":{ + "mean":0.027794, + "std":0.003578 + }, + "accuracy_8":{ + "mean":0.854727, + "std":0.008117 + }, + "oos_recall":{ + "mean":0.567333, + "std":0.003512 + }, + "oos_recall_wilson95":[ + { + "mean":0.536426, + "std":0.003528 + }, + { + "mean":0.597725, + "std":0.003469 + } + ], + "oos_precision":{ + "mean":0.802974, + "std":0.015643 + }, + "oos_f1":{ + "mean":0.664872, + "std":0.00769 + }, + "oos_misroute_rate":{ + "mean":0.432667, + "std":0.003512 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.033667, + "std":0.010017 + }, + "llm_call_rate":{ + "mean":0.631212, + "std":0.078471 + }, + "llm_cost_usd_per_1k":{ + "mean":0.233065, + "std":0.028983 + }, + "error_recovery_rate":{ + "mean":0.570321, + "std":0.052027 + }, + "recoverable_caught":{ + "mean":0.964829, + "std":0.020321 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":668.333333, + "std":125.005333 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.813084, + "std":0.059395 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.498602, + "std":0.078829 + }, + "validation_selective_risk":{ + "mean":0.013963, + "std":0.000523 + }, + "validation_risk_upper95":{ + "mean":0.019835, + "std":4.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.443273, + "std":0.07249 + }, + "selective_risk":{ + "mean":0.028228, + "std":0.003759 + }, + "accuracy_8":{ + "mean":0.861152, + "std":0.006876 + }, + "oos_recall":{ + "mean":0.568, + "std":0.006083 + }, + "oos_recall_wilson95":[ + { + "mean":0.537097, + "std":0.006112 + }, + { + "mean":0.598383, + "std":0.006007 + } + ], + "oos_precision":{ + "mean":0.813129, + "std":0.016208 + }, + "oos_f1":{ + "mean":0.668782, + "std":0.008629 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.006083 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.039, + "std":0.011533 + }, + "llm_call_rate":{ + "mean":0.556727, + "std":0.07249 + }, + "llm_cost_usd_per_1k":{ + "mean":0.205505, + "std":0.026745 + }, + "error_recovery_rate":{ + "mean":0.565257, + "std":0.050903 + }, + "recoverable_caught":{ + "mean":0.956297, + "std":0.017695 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":662.333333, + "std":123.001355 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.04849, + "std":0.00178 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.808065, + "std":0.043924 + }, + "validation_selective_risk":{ + "mean":0.042711, + "std":0.000149 + }, + "validation_risk_upper95":{ + "mean":0.04987, + "std":0.000145 + }, + "test":{ + "coverage":{ + "mean":0.724303, + "std":0.050486 + }, + "selective_risk":{ + "mean":0.076966, + "std":0.00818 + }, + "accuracy_8":{ + "mean":0.865152, + "std":0.006671 + }, + "oos_recall":{ + "mean":0.543667, + "std":0.009452 + }, + "oos_recall_wilson95":[ + { + "mean":0.51269, + "std":0.009464 + }, + { + "mean":0.574309, + "std":0.009367 + } + ], + "oos_precision":{ + "mean":0.860509, + "std":0.015596 + }, + "oos_f1":{ + "mean":0.666242, + "std":0.006425 + }, + "oos_misroute_rate":{ + "mean":0.456333, + "std":0.009452 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.151, + "std":0.03079 + }, + "llm_call_rate":{ + "mean":0.275697, + "std":0.050486 + }, + "llm_cost_usd_per_1k":{ + "mean":0.101646, + "std":0.018617 + }, + "error_recovery_rate":{ + "mean":0.446969, + "std":0.053754 + }, + "recoverable_caught":{ + "mean":0.755123, + "std":0.036481 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":524.333333, + "std":111.468979 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.04849, + "std":0.00178 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.808065, + "std":0.043924 + }, + "validation_selective_risk":{ + "mean":0.042711, + "std":0.000149 + }, + "validation_risk_upper95":{ + "mean":0.04987, + "std":0.000145 + }, + "test":{ + "coverage":{ + "mean":0.724303, + "std":0.050486 + }, + "selective_risk":{ + "mean":0.076966, + "std":0.00818 + }, + "accuracy_8":{ + "mean":0.865152, + "std":0.006671 + }, + "oos_recall":{ + "mean":0.543667, + "std":0.009452 + }, + "oos_recall_wilson95":[ + { + "mean":0.51269, + "std":0.009464 + }, + { + "mean":0.574309, + "std":0.009367 + } + ], + "oos_precision":{ + "mean":0.860509, + "std":0.015596 + }, + "oos_f1":{ + "mean":0.666242, + "std":0.006425 + }, + "oos_misroute_rate":{ + "mean":0.456333, + "std":0.009452 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.151, + "std":0.03079 + }, + "llm_call_rate":{ + "mean":0.275697, + "std":0.050486 + }, + "llm_cost_usd_per_1k":{ + "mean":0.101646, + "std":0.018617 + }, + "error_recovery_rate":{ + "mean":0.446969, + "std":0.053754 + }, + "recoverable_caught":{ + "mean":0.755123, + "std":0.036481 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":524.333333, + "std":111.468979 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.682017, + "std":0.038308 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.671935, + "std":0.107246 + }, + "validation_selective_risk":{ + "mean":0.042008, + "std":0.000742 + }, + "validation_risk_upper95":{ + "mean":0.04991, + "std":7e-05 + }, + "test":{ + "coverage":{ + "mean":0.586909, + "std":0.091577 + }, + "selective_risk":{ + "mean":0.067915, + "std":0.008509 + }, + "accuracy_8":{ + "mean":0.856242, + "std":0.007404 + }, + "oos_recall":{ + "mean":0.557, + "std":0.017349 + }, + "oos_recall_wilson95":[ + { + "mean":0.526064, + "std":0.0174 + }, + { + "mean":0.587499, + "std":0.017166 + } + ], + "oos_precision":{ + "mean":0.821919, + "std":0.03144 + }, + "oos_f1":{ + "mean":0.663607, + "std":0.010939 + }, + "oos_misroute_rate":{ + "mean":0.443, + "std":0.017349 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.099, + "std":0.031193 + }, + "llm_call_rate":{ + "mean":0.413091, + "std":0.091577 + }, + "llm_cost_usd_per_1k":{ + "mean":0.152398, + "std":0.033867 + }, + "error_recovery_rate":{ + "mean":0.487194, + "std":0.071088 + }, + "recoverable_caught":{ + "mean":0.82225, + "std":0.064185 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":572.0, + "std":134.324235 + } + } + }, + "margin":{ + "tau":{ + "mean":1.183455, + "std":0.15186 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.717527, + "std":0.059842 + }, + "validation_selective_risk":{ + "mean":0.042237, + "std":0.000422 + }, + "validation_risk_upper95":{ + "mean":0.04984, + "std":0.00012 + }, + "test":{ + "coverage":{ + "mean":0.667697, + "std":0.053601 + }, + "selective_risk":{ + "mean":0.096704, + "std":0.007771 + }, + "accuracy_8":{ + "mean":0.855212, + "std":0.008468 + }, + "oos_recall":{ + "mean":0.499333, + "std":0.017388 + }, + "oos_recall_wilson95":[ + { + "mean":0.468417, + "std":0.017329 + }, + { + "mean":0.530254, + "std":0.017314 + } + ], + "oos_precision":{ + "mean":0.840535, + "std":0.028871 + }, + "oos_f1":{ + "mean":0.626356, + "std":0.018457 + }, + "oos_misroute_rate":{ + "mean":0.500667, + "std":0.017388 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.227, + "std":0.020075 + }, + "llm_call_rate":{ + "mean":0.332303, + "std":0.053601 + }, + "llm_cost_usd_per_1k":{ + "mean":0.122639, + "std":0.019761 + }, + "error_recovery_rate":{ + "mean":0.428013, + "std":0.055642 + }, + "recoverable_caught":{ + "mean":0.72278, + "std":0.042016 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":502.333333, + "std":111.540725 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.480786, + "std":0.054952 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.779677, + "std":0.054231 + }, + "validation_selective_risk":{ + "mean":0.042602, + "std":0.000291 + }, + "validation_risk_upper95":{ + "mean":0.049896, + "std":9.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.712, + "std":0.06101 + }, + "selective_risk":{ + "mean":0.084281, + "std":0.00591 + }, + "accuracy_8":{ + "mean":0.863879, + "std":0.005948 + }, + "oos_recall":{ + "mean":0.527667, + "std":0.009713 + }, + "oos_recall_wilson95":[ + { + "mean":0.496681, + "std":0.009707 + }, + { + "mean":0.558441, + "std":0.009644 + } + ], + "oos_precision":{ + "mean":0.862075, + "std":0.017037 + }, + "oos_f1":{ + "mean":0.654519, + "std":0.006246 + }, + "oos_misroute_rate":{ + "mean":0.472333, + "std":0.009713 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.186667, + "std":0.033081 + }, + "llm_call_rate":{ + "mean":0.288, + "std":0.06101 + }, + "llm_cost_usd_per_1k":{ + "mean":0.106241, + "std":0.022509 + }, + "error_recovery_rate":{ + "mean":0.439208, + "std":0.061416 + }, + "recoverable_caught":{ + "mean":0.741267, + "std":0.047406 + }, + "small_errors":{ + "mean":1165.333333, + "std":108.408179 + }, + "recoverable_errors":{ + "mean":691.333333, + "std":116.328558 + }, + "recovered_errors":{ + "mean":516.0, + "std":121.280666 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.907634, + "std":0.005411 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.210061, + "std":0.016089 + }, + "accuracy_8":{ + "mean":0.789939, + "std":0.016089 + }, + "oos_recall":{ + "mean":0.191, + "std":0.069549 + }, + "oos_recall_wilson95":[ + { + "mean":0.168289, + "std":0.065507 + }, + { + "mean":0.216076, + "std":0.073058 + } + ], + "oos_precision":{ + "mean":0.825071, + "std":0.112138 + }, + "oos_f1":{ + "mean":0.308311, + "std":0.101202 + }, + "oos_misroute_rate":{ + "mean":0.809, + "std":0.069549 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.809, + "std":0.069549 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.789939, + "std":0.016089 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.912242, + "std":0.0029 + }, + "oos_recall":{ + "mean":0.602333, + "std":0.006429 + }, + "oos_recall_wilson95":[ + { + "mean":0.571665, + "std":0.006486 + }, + { + "mean":0.632219, + "std":0.006323 + } + ], + "oos_precision":{ + "mean":0.943589, + "std":0.007007 + }, + "oos_f1":{ + "mean":0.735294, + "std":0.006872 + }, + "oos_misroute_rate":{ + "mean":0.397667, + "std":0.006429 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.210061, + "std":0.016089 + }, + "llm_cost_usd_per_1k":{ + "mean":0.077404, + "std":0.005888 + }, + "error_recovery_rate":{ + "mean":0.580417, + "std":0.03749 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":672.666667, + "std":95.212044 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.03942, + "std":0.001862 + }, + "oos_auroc":{ + "mean":0.748268, + "std":0.014481 + }, + "oos_auprc":{ + "mean":0.097011, + "std":0.027442 + } + }, + "test":{ + "aurc":{ + "mean":0.1111, + "std":0.00785 + }, + "oos_auroc":{ + "mean":0.733603, + "std":0.020615 + }, + "oos_auprc":{ + "mean":0.391062, + "std":0.042318 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.038192, + "std":0.003548 + }, + "oos_auroc":{ + "mean":0.828339, + "std":0.001873 + }, + "oos_auprc":{ + "mean":0.260224, + "std":0.058936 + } + }, + "test":{ + "aurc":{ + "mean":0.104604, + "std":0.007014 + }, + "oos_auroc":{ + "mean":0.791795, + "std":0.015693 + }, + "oos_auprc":{ + "mean":0.543004, + "std":0.060838 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.016893, + "std":0.000998 + }, + "oos_auroc":{ + "mean":0.809093, + "std":0.038964 + }, + "oos_auprc":{ + "mean":0.133418, + "std":0.024219 + } + }, + "test":{ + "aurc":{ + "mean":0.054195, + "std":0.009106 + }, + "oos_auroc":{ + "mean":0.823607, + "std":0.024182 + }, + "oos_auprc":{ + "mean":0.481954, + "std":0.031101 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.016131, + "std":0.001355 + }, + "oos_auroc":{ + "mean":0.835518, + "std":0.040439 + }, + "oos_auprc":{ + "mean":0.227421, + "std":0.041989 + } + }, + "test":{ + "aurc":{ + "mean":0.050917, + "std":0.008118 + }, + "oos_auroc":{ + "mean":0.848808, + "std":0.028149 + }, + "oos_auprc":{ + "mean":0.583944, + "std":0.051692 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "msp_t", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.460889, + "std":0.979203 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.668065, + "std":0.014061 + }, + "validation_selective_risk":{ + "mean":0.014808, + "std":4.7e-05 + }, + "validation_risk_upper95":{ + "mean":0.01985, + "std":0.000107 + }, + "test":{ + "coverage":{ + "mean":0.581636, + "std":0.008638 + }, + "selective_risk":{ + "mean":0.034648, + "std":0.009893 + }, + "accuracy_8":{ + "mean":0.864485, + "std":0.006748 + }, + "oos_recall":{ + "mean":0.555333, + "std":0.015822 + }, + "oos_recall_wilson95":[ + { + "mean":0.52439, + "std":0.015865 + }, + { + "mean":0.585853, + "std":0.015658 + } + ], + "oos_precision":{ + "mean":0.83186, + "std":0.006152 + }, + "oos_f1":{ + "mean":0.665912, + "std":0.010258 + }, + "oos_misroute_rate":{ + "mean":0.444667, + "std":0.015822 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.08, + "std":0.027055 + }, + "llm_call_rate":{ + "mean":0.418364, + "std":0.008638 + }, + "llm_cost_usd_per_1k":{ + "mean":0.15431, + "std":0.003211 + }, + "error_recovery_rate":{ + "mean":0.549507, + "std":0.028341 + }, + "recoverable_caught":{ + "mean":0.947283, + "std":0.014974 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":636.333333, + "std":80.407297 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.461168, + "std":0.034227, + "n":2 + }, + "feasible":[ + false, + true, + true + ], + "validation_coverage":{ + "mean":0.063118, + "std":0.073109 + }, + "validation_selective_risk":{ + "mean":0.004505, + "std":0.00637, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.019242, + "std":0.000952, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.052364, + "std":0.062306 + }, + "selective_risk":{ + "mean":0.057272, + "std":0.019507, + "n":2 + }, + "accuracy_8":{ + "mean":0.820848, + "std":0.00021 + }, + "oos_recall":{ + "mean":0.567, + "std":0.001732 + }, + "oos_recall_wilson95":[ + { + "mean":0.536091, + "std":0.00174 + }, + { + "mean":0.597397, + "std":0.001711 + } + ], + "oos_precision":{ + "mean":0.74802, + "std":0.000577 + }, + "oos_f1":{ + "mean":0.64505, + "std":0.001336 + }, + "oos_misroute_rate":{ + "mean":0.433, + "std":0.001732 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.013333, + "std":0.013013 + }, + "llm_call_rate":{ + "mean":0.947636, + "std":0.062306 + }, + "llm_cost_usd_per_1k":{ + "mean":0.350075, + "std":0.023099 + }, + "error_recovery_rate":{ + "mean":0.578646, + "std":0.039286 + }, + "recoverable_caught":{ + "mean":0.996815, + "std":0.005516 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":670.666667, + "std":96.671264 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.656994, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.041613, + "std":0.072076 + }, + "validation_selective_risk":{ + "mean":0.007752, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.019237, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.034667, + "std":0.060044 + }, + "selective_risk":{ + "mean":0.043706, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.821091, + "std":0.00063 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.007333, + "std":0.012702 + }, + "llm_call_rate":{ + "mean":0.965333, + "std":0.060044 + }, + "llm_cost_usd_per_1k":{ + "mean":0.356635, + "std":0.022268 + }, + "error_recovery_rate":{ + "mean":0.579532, + "std":0.038368 + }, + "recoverable_caught":{ + "mean":0.998408, + "std":0.002758 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":671.666667, + "std":95.928793 + } + } + }, + "margin":{ + "tau":{ + "mean":2.853687, + "std":0.280311 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.656667, + "std":0.019253 + }, + "validation_selective_risk":{ + "mean":0.014736, + "std":0.000111 + }, + "validation_risk_upper95":{ + "mean":0.019818, + "std":0.000116 + }, + "test":{ + "coverage":{ + "mean":0.570848, + "std":0.010571 + }, + "selective_risk":{ + "mean":0.034876, + "std":0.010185 + }, + "accuracy_8":{ + "mean":0.861636, + "std":0.007613 + }, + "oos_recall":{ + "mean":0.553667, + "std":0.015503 + }, + "oos_recall_wilson95":[ + { + "mean":0.522718, + "std":0.015546 + }, + { + "mean":0.584204, + "std":0.015341 + } + ], + "oos_precision":{ + "mean":0.83265, + "std":0.008225 + }, + "oos_f1":{ + "mean":0.664988, + "std":0.011287 + }, + "oos_misroute_rate":{ + "mean":0.446333, + "std":0.015503 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.081667, + "std":0.02616 + }, + "llm_call_rate":{ + "mean":0.429152, + "std":0.010571 + }, + "llm_cost_usd_per_1k":{ + "mean":0.158325, + "std":0.003976 + }, + "error_recovery_rate":{ + "mean":0.55043, + "std":0.027927 + }, + "recoverable_caught":{ + "mean":0.948928, + "std":0.017487 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":637.333333, + "std":79.46278 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.896451, + "std":0.016544 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.648817, + "std":0.027392 + }, + "validation_selective_risk":{ + "mean":0.014746, + "std":0.000154 + }, + "validation_risk_upper95":{ + "mean":0.019866, + "std":8.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.566061, + "std":0.028177 + }, + "selective_risk":{ + "mean":0.030381, + "std":0.005655 + }, + "accuracy_8":{ + "mean":0.864848, + "std":0.006826 + }, + "oos_recall":{ + "mean":0.562333, + "std":0.005774 + }, + "oos_recall_wilson95":[ + { + "mean":0.531406, + "std":0.005797 + }, + { + "mean":0.592784, + "std":0.005705 + } + ], + "oos_precision":{ + "mean":0.829806, + "std":0.003555 + }, + "oos_f1":{ + "mean":0.670366, + "std":0.004786 + }, + "oos_misroute_rate":{ + "mean":0.437667, + "std":0.005774 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.065333, + "std":0.015275 + }, + "llm_call_rate":{ + "mean":0.433939, + "std":0.028177 + }, + "llm_cost_usd_per_1k":{ + "mean":0.160058, + "std":0.010357 + }, + "error_recovery_rate":{ + "mean":0.555887, + "std":0.039174 + }, + "recoverable_caught":{ + "mean":0.957513, + "std":0.008721 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":644.333333, + "std":94.240826 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.600529, + "std":0.015314 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.891075, + "std":0.004701 + }, + "validation_selective_risk":{ + "mean":0.042959, + "std":9.3e-05 + }, + "validation_risk_upper95":{ + "mean":0.049765, + "std":0.000108 + }, + "test":{ + "coverage":{ + "mean":0.814606, + "std":0.010941 + }, + "selective_risk":{ + "mean":0.101138, + "std":0.012195 + }, + "accuracy_8":{ + "mean":0.861758, + "std":0.009044 + }, + "oos_recall":{ + "mean":0.480667, + "std":0.031501 + }, + "oos_recall_wilson95":[ + { + "mean":0.449874, + "std":0.031305 + }, + { + "mean":0.511608, + "std":0.031457 + } + ], + "oos_precision":{ + "mean":0.890669, + "std":0.004224 + }, + "oos_f1":{ + "mean":0.623973, + "std":0.02676 + }, + "oos_misroute_rate":{ + "mean":0.519333, + "std":0.031501 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.300667, + "std":0.039552 + }, + "llm_call_rate":{ + "mean":0.185394, + "std":0.010941 + }, + "llm_cost_usd_per_1k":{ + "mean":0.068302, + "std":0.004034 + }, + "error_recovery_rate":{ + "mean":0.391329, + "std":0.026666 + }, + "recoverable_caught":{ + "mean":0.674491, + "std":0.029173 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":453.0, + "std":58.966092 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.284372, + "std":0.01513 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.615699, + "std":0.086639 + }, + "validation_selective_risk":{ + "mean":0.041529, + "std":0.000379 + }, + "validation_risk_upper95":{ + "mean":0.049755, + "std":0.000198 + }, + "test":{ + "coverage":{ + "mean":0.566182, + "std":0.090749 + }, + "selective_risk":{ + "mean":0.115505, + "std":0.004158 + }, + "accuracy_8":{ + "mean":0.835212, + "std":0.002729 + }, + "oos_recall":{ + "mean":0.494667, + "std":0.01124 + }, + "oos_recall_wilson95":[ + { + "mean":0.463763, + "std":0.011192 + }, + { + "mean":0.525611, + "std":0.011202 + } + ], + "oos_precision":{ + "mean":0.820269, + "std":0.014672 + }, + "oos_f1":{ + "mean":0.617002, + "std":0.005067 + }, + "oos_misroute_rate":{ + "mean":0.505333, + "std":0.01124 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.256667, + "std":0.036529 + }, + "llm_call_rate":{ + "mean":0.433818, + "std":0.090749 + }, + "llm_cost_usd_per_1k":{ + "mean":0.160125, + "std":0.033556 + }, + "error_recovery_rate":{ + "mean":0.43837, + "std":0.061163 + }, + "recoverable_caught":{ + "mean":0.75353, + "std":0.068226 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":510.0, + "std":110.367568 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.833917, + "std":0.027058 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.623011, + "std":0.094148 + }, + "validation_selective_risk":{ + "mean":0.04152, + "std":0.000737 + }, + "validation_risk_upper95":{ + "mean":0.049705, + "std":8.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.558424, + "std":0.093474 + }, + "selective_risk":{ + "mean":0.107958, + "std":0.008747 + }, + "accuracy_8":{ + "mean":0.843091, + "std":0.003288 + }, + "oos_recall":{ + "mean":0.513333, + "std":0.01365 + }, + "oos_recall_wilson95":[ + { + "mean":0.48237, + "std":0.013627 + }, + { + "mean":0.544195, + "std":0.013569 + } + ], + "oos_precision":{ + "mean":0.830819, + "std":0.027357 + }, + "oos_f1":{ + "mean":0.634233, + "std":0.003182 + }, + "oos_misroute_rate":{ + "mean":0.486667, + "std":0.01365 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.225, + "std":0.053731 + }, + "llm_call_rate":{ + "mean":0.441576, + "std":0.093474 + }, + "llm_cost_usd_per_1k":{ + "mean":0.162922, + "std":0.034576 + }, + "error_recovery_rate":{ + "mean":0.458977, + "std":0.064624 + }, + "recoverable_caught":{ + "mean":0.788299, + "std":0.058989 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":534.0, + "std":117.690271 + } + } + }, + "margin":{ + "tau":{ + "mean":1.061499, + "std":0.07672 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.881613, + "std":0.005597 + }, + "validation_selective_risk":{ + "mean":0.042932, + "std":9.9e-05 + }, + "validation_risk_upper95":{ + "mean":0.049775, + "std":0.000124 + }, + "test":{ + "coverage":{ + "mean":0.817758, + "std":0.008769 + }, + "selective_risk":{ + "mean":0.109081, + "std":0.016303 + }, + "accuracy_8":{ + "mean":0.855636, + "std":0.010045 + }, + "oos_recall":{ + "mean":0.445667, + "std":0.051287 + }, + "oos_recall_wilson95":[ + { + "mean":0.415237, + "std":0.050655 + }, + { + "mean":0.476512, + "std":0.051526 + } + ], + "oos_precision":{ + "mean":0.881874, + "std":0.006921 + }, + "oos_f1":{ + "mean":0.59076, + "std":0.044945 + }, + "oos_misroute_rate":{ + "mean":0.554333, + "std":0.051287 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.351667, + "std":0.055139 + }, + "llm_call_rate":{ + "mean":0.182242, + "std":0.008769 + }, + "llm_cost_usd_per_1k":{ + "mean":0.067165, + "std":0.003224 + }, + "error_recovery_rate":{ + "mean":0.37085, + "std":0.006275 + }, + "recoverable_caught":{ + "mean":0.640289, + "std":0.032065 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":428.666667, + "std":37.740341 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.600529, + "std":0.015314 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.891075, + "std":0.004701 + }, + "validation_selective_risk":{ + "mean":0.042959, + "std":9.3e-05 + }, + "validation_risk_upper95":{ + "mean":0.049765, + "std":0.000108 + }, + "test":{ + "coverage":{ + "mean":0.814606, + "std":0.010941 + }, + "selective_risk":{ + "mean":0.101138, + "std":0.012195 + }, + "accuracy_8":{ + "mean":0.861758, + "std":0.009044 + }, + "oos_recall":{ + "mean":0.480667, + "std":0.031501 + }, + "oos_recall_wilson95":[ + { + "mean":0.449874, + "std":0.031305 + }, + { + "mean":0.511608, + "std":0.031457 + } + ], + "oos_precision":{ + "mean":0.890669, + "std":0.004224 + }, + "oos_f1":{ + "mean":0.623973, + "std":0.02676 + }, + "oos_misroute_rate":{ + "mean":0.519333, + "std":0.031501 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.300667, + "std":0.039552 + }, + "llm_call_rate":{ + "mean":0.185394, + "std":0.010941 + }, + "llm_cost_usd_per_1k":{ + "mean":0.068302, + "std":0.004034 + }, + "error_recovery_rate":{ + "mean":0.391329, + "std":0.026666 + }, + "recoverable_caught":{ + "mean":0.674491, + "std":0.029173 + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171 + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044 + }, + "recovered_errors":{ + "mean":453.0, + "std":58.966092 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.210061, + "std":0.016089, + "values":[ + 0.196909, + 0.205273, + 0.228 + ] + }, + "accuracy_8":{ + "mean":0.789939, + "std":0.016089, + "values":[ + 0.803091, + 0.794727, + 0.772 + ] + }, + "oos_recall":{ + "mean":0.191, + "std":0.069549, + "values":[ + 0.243, + 0.218, + 0.112 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.168289, + "std":0.065507, + "values":[ + 0.217433, + 0.193514, + 0.093919 + ] + }, + { + "mean":0.216076, + "std":0.073058, + "values":[ + 0.270534, + 0.244644, + 0.13305 + ] + } + ], + "oos_precision":{ + "mean":0.825071, + "std":0.112138, + "values":[ + 0.893382, + 0.886179, + 0.695652 + ] + }, + "oos_f1":{ + "mean":0.308311, + "std":0.101202, + "values":[ + 0.382075, + 0.34992, + 0.192937 + ] + }, + "oos_misroute_rate":{ + "mean":0.809, + "std":0.069549, + "values":[ + 0.757, + 0.782, + 0.888 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.809, + "std":0.069549, + "values":[ + 0.757, + 0.782, + 0.888 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171, + "values":[ + 1083, + 1129, + 1254 + ] + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044, + "values":[ + 608, + 628, + 782 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.789939, + "std":0.016089, + "values":[ + 0.803091, + 0.794727, + 0.772 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.912242, + "std":0.0029, + "values":[ + 0.913636, + 0.908909, + 0.914182 + ] + }, + "oos_recall":{ + "mean":0.602333, + "std":0.006429, + "values":[ + 0.607, + 0.595, + 0.605 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.571665, + "std":0.006486, + "values":[ + 0.576373, + 0.564267, + 0.574354 + ] + }, + { + "mean":0.632219, + "std":0.006323, + "values":[ + 0.636808, + 0.625006, + 0.634842 + ] + } + ], + "oos_precision":{ + "mean":0.943589, + "std":0.007007, + "values":[ + 0.946958, + 0.935535, + 0.948276 + ] + }, + "oos_f1":{ + "mean":0.735294, + "std":0.006872, + "values":[ + 0.739793, + 0.727384, + 0.738706 + ] + }, + "oos_misroute_rate":{ + "mean":0.397667, + "std":0.006429, + "values":[ + 0.393, + 0.405, + 0.395 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.210061, + "std":0.016089, + "values":[ + 0.196909, + 0.205273, + 0.228 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.077404, + "std":0.005888, + "values":[ + 0.072583, + 0.075663, + 0.083967 + ] + }, + "error_recovery_rate":{ + "mean":0.580417, + "std":0.03749, + "values":[ + 0.561404, + 0.556244, + 0.623604 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171, + "values":[ + 1083, + 1129, + 1254 + ] + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044, + "values":[ + 608, + 628, + 782 + ] + }, + "recovered_errors":{ + "mean":672.666667, + "std":95.212044, + "values":[ + 608, + 628, + 782 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.03942, + "std":0.001862, + "values":[ + 0.037281, + 0.040293, + 0.040685 + ] + }, + "oos_auroc":{ + "mean":0.748268, + "std":0.014481, + "values":[ + 0.764927, + 0.738687, + 0.74119 + ] + }, + "oos_auprc":{ + "mean":0.097011, + "std":0.027442, + "values":[ + 0.128587, + 0.07892, + 0.083527 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.1111, + "std":0.00785, + "values":[ + 0.102652, + 0.11248, + 0.11817 + ] + }, + "oos_auroc":{ + "mean":0.733603, + "std":0.020615, + "values":[ + 0.755715, + 0.73018, + 0.714914 + ] + }, + "oos_auprc":{ + "mean":0.391062, + "std":0.042318, + "values":[ + 0.439747, + 0.370339, + 0.3631 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.038192, + "std":0.003548, + "values":[ + 0.036377, + 0.035919, + 0.04228 + ] + }, + "oos_auroc":{ + "mean":0.828339, + "std":0.001873, + "values":[ + 0.83012, + 0.82851, + 0.826387 + ] + }, + "oos_auprc":{ + "mean":0.260224, + "std":0.058936, + "values":[ + 0.289177, + 0.299084, + 0.192412 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.104604, + "std":0.007014, + "values":[ + 0.098396, + 0.103204, + 0.112213 + ] + }, + "oos_auroc":{ + "mean":0.791795, + "std":0.015693, + "values":[ + 0.803719, + 0.797651, + 0.774016 + ] + }, + "oos_auprc":{ + "mean":0.543004, + "std":0.060838, + "values":[ + 0.5751, + 0.581072, + 0.472839 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.016893, + "std":0.000998, + "values":[ + 0.015759, + 0.017283, + 0.017637 + ] + }, + "oos_auroc":{ + "mean":0.809093, + "std":0.038964, + "values":[ + 0.77174, + 0.80605, + 0.84949 + ] + }, + "oos_auprc":{ + "mean":0.133418, + "std":0.024219, + "values":[ + 0.114826, + 0.124623, + 0.160806 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.054195, + "std":0.009106, + "values":[ + 0.044402, + 0.055776, + 0.062408 + ] + }, + "oos_auroc":{ + "mean":0.823607, + "std":0.024182, + "values":[ + 0.804519, + 0.815503, + 0.850801 + ] + }, + "oos_auprc":{ + "mean":0.481954, + "std":0.031101, + "values":[ + 0.487702, + 0.448379, + 0.509779 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.016131, + "std":0.001355, + "values":[ + 0.01459, + 0.016669, + 0.017134 + ] + }, + "oos_auroc":{ + "mean":0.835518, + "std":0.040439, + "values":[ + 0.797747, + 0.830627, + 0.87818 + ] + }, + "oos_auprc":{ + "mean":0.227421, + "std":0.041989, + "values":[ + 0.18852, + 0.221809, + 0.271934 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.050917, + "std":0.008118, + "values":[ + 0.04184, + 0.053426, + 0.057484 + ] + }, + "oos_auroc":{ + "mean":0.848808, + "std":0.028149, + "values":[ + 0.828315, + 0.837205, + 0.880904 + ] + }, + "oos_auprc":{ + "mean":0.583944, + "std":0.051692, + "values":[ + 0.601413, + 0.525781, + 0.624639 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "msp_t", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.460889, + "std":0.979203, + "values":[ + 0.879145, + 0.912108, + 2.591415 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.668065, + "std":0.014061, + "values":[ + 0.674516, + 0.651935, + 0.677742 + ] + }, + "validation_selective_risk":{ + "mean":0.014808, + "std":4.7e-05, + "values":[ + 0.014825, + 0.014844, + 0.014755 + ] + }, + "validation_risk_upper95":{ + "mean":0.01985, + "std":0.000107, + "values":[ + 0.019842, + 0.019962, + 0.019748 + ] + }, + "test":{ + "coverage":{ + "mean":0.581636, + "std":0.008638, + "values":[ + 0.590182, + 0.572909, + 0.581818 + ] + }, + "selective_risk":{ + "mean":0.034648, + "std":0.009893, + "values":[ + 0.024338, + 0.035544, + 0.044062 + ] + }, + "accuracy_8":{ + "mean":0.864485, + "std":0.006748, + "values":[ + 0.870909, + 0.857455, + 0.865091 + ] + }, + "oos_recall":{ + "mean":0.555333, + "std":0.015822, + "values":[ + 0.569, + 0.559, + 0.538 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.52439, + "std":0.015865, + "values":[ + 0.5381, + 0.528059, + 0.507013 + ] + }, + { + "mean":0.585853, + "std":0.015658, + "values":[ + 0.599372, + 0.58949, + 0.568696 + ] + } + ], + "oos_precision":{ + "mean":0.83186, + "std":0.006152, + "values":[ + 0.831871, + 0.825702, + 0.838006 + ] + }, + "oos_f1":{ + "mean":0.665912, + "std":0.010258, + "values":[ + 0.675772, + 0.666667, + 0.655298 + ] + }, + "oos_misroute_rate":{ + "mean":0.444667, + "std":0.015822, + "values":[ + 0.431, + 0.441, + 0.462 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.08, + "std":0.027055, + "values":[ + 0.052, + 0.082, + 0.106 + ] + }, + "llm_call_rate":{ + "mean":0.418364, + "std":0.008638, + "values":[ + 0.409818, + 0.427091, + 0.418182 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.15431, + "std":0.003211, + "values":[ + 0.151163, + 0.157582, + 0.154185 + ] + }, + "error_recovery_rate":{ + "mean":0.549507, + "std":0.028341, + "values":[ + 0.540166, + 0.527015, + 0.58134 + ] + }, + "recoverable_caught":{ + "mean":0.947283, + "std":0.014974, + "values":[ + 0.962171, + 0.947452, + 0.932225 + ] + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171, + "values":[ + 1083, + 1129, + 1254 + ] + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044, + "values":[ + 608, + 628, + 782 + ] + }, + "recovered_errors":{ + "mean":636.333333, + "std":80.407297, + "values":[ + 585, + 595, + 729 + ] + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.600529, + "std":0.015314, + "values":[ + 0.585596, + 0.616197, + 0.599794 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.891075, + "std":0.004701, + "values":[ + 0.895484, + 0.886129, + 0.891613 + ] + }, + "validation_selective_risk":{ + "mean":0.042959, + "std":9.3e-05, + "values":[ + 0.042867, + 0.042956, + 0.043054 + ] + }, + "validation_risk_upper95":{ + "mean":0.049765, + "std":0.000108, + "values":[ + 0.049649, + 0.049782, + 0.049863 + ] + }, + "test":{ + "coverage":{ + "mean":0.814606, + "std":0.010941, + "values":[ + 0.826, + 0.813636, + 0.804182 + ] + }, + "selective_risk":{ + "mean":0.101138, + "std":0.012195, + "values":[ + 0.087167, + 0.106592, + 0.109654 + ] + }, + "accuracy_8":{ + "mean":0.861758, + "std":0.009044, + "values":[ + 0.872182, + 0.856, + 0.857091 + ] + }, + "oos_recall":{ + "mean":0.480667, + "std":0.031501, + "values":[ + 0.512, + 0.481, + 0.449 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.449874, + "std":0.031305, + "values":[ + 0.481032, + 0.450164, + 0.418425 + ] + }, + { + "mean":0.511608, + "std":0.031457, + "values":[ + 0.542876, + 0.511981, + 0.479965 + ] + } + ], + "oos_precision":{ + "mean":0.890669, + "std":0.004224, + "values":[ + 0.893543, + 0.88582, + 0.892644 + ] + }, + "oos_f1":{ + "mean":0.623973, + "std":0.02676, + "values":[ + 0.650985, + 0.623461, + 0.597472 + ] + }, + "oos_misroute_rate":{ + "mean":0.519333, + "std":0.031501, + "values":[ + 0.488, + 0.519, + 0.551 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.300667, + "std":0.039552, + "values":[ + 0.255, + 0.324, + 0.323 + ] + }, + "llm_call_rate":{ + "mean":0.185394, + "std":0.010941, + "values":[ + 0.174, + 0.186364, + 0.195818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.068302, + "std":0.004034, + "values":[ + 0.064101, + 0.068662, + 0.072144 + ] + }, + "error_recovery_rate":{ + "mean":0.391329, + "std":0.026666, + "values":[ + 0.397045, + 0.362267, + 0.414673 + ] + }, + "recoverable_caught":{ + "mean":0.674491, + "std":0.029173, + "values":[ + 0.707237, + 0.651274, + 0.664962 + ] + }, + "small_errors":{ + "mean":1155.333333, + "std":88.489171, + "values":[ + 1083, + 1129, + 1254 + ] + }, + "recoverable_errors":{ + "mean":672.666667, + "std":95.212044, + "values":[ + 608, + 628, + 782 + ] + }, + "recovered_errors":{ + "mean":453.0, + "std":58.966092, + "values":[ + 430, + 409, + 520 + ] + } + } + } + } + } + } + } + }, + "bert/k50":{ + "model":"bert", + "k":50, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "bert-base-uncased-k50-seed42", + "bert-base-uncased-k50-seed43", + "bert-base-uncased-k50-seed44" + ], + "logits_sha256":[ + "42bd57f63c3f37264962334df25861a73c5279f7559302c9a467d5fadd246b7d", + "21bb8489d1c181fa6982ff89fb8a587505a5fb00ad53031246c0bdecb16bd949", + "e335e894b2eb54e9f4074b861f202c478078700f308503c82ecc34ec7fc1c6fa" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":0.749156, + "std":0.005865, + "values":[ + 0.755768, + 0.744581, + 0.747119 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.071882, + "std":0.003724, + "values":[ + 0.071064, + 0.075946, + 0.068635 + ] + }, + "ece_8_summed":{ + "mean":0.057501, + "std":0.00151, + "values":[ + 0.059035, + 0.056016, + 0.057453 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.040737, + "std":0.000307, + "values":[ + 0.040799, + 0.040404, + 0.041008 + ] + }, + "ece_8_summed":{ + "mean":0.054827, + "std":0.002161, + "values":[ + 0.056158, + 0.052333, + 0.05599 + ] + } + } + } + }, + "selected_aggregation":[ + "summed", + "summed", + "argmax" + ], + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.970645, + "std":0.00129, + "values":[ + 0.970645, + 0.971935, + 0.969355 + ] + }, + "summed":{ + "mean":0.97086, + "std":0.001777, + "values":[ + 0.970968, + 0.972581, + 0.969032 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.970645, + "std":0.00129 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.109212, + "std":0.004034 + }, + "accuracy_8":{ + "mean":0.890788, + "std":0.004034 + }, + "oos_recall":{ + "mean":0.463, + "std":0.021794 + }, + "oos_recall_wilson95":[ + { + "mean":0.432315, + "std":0.021627 + }, + { + "mean":0.493968, + "std":0.021795 + } + ], + "oos_precision":{ + "mean":0.982855, + "std":0.006139 + }, + "oos_f1":{ + "mean":0.629318, + "std":0.02124 + }, + "oos_misroute_rate":{ + "mean":0.537, + "std":0.021794 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.537, + "std":0.021794 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.890788, + "std":0.004034 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.935394, + "std":0.002259 + }, + "oos_recall":{ + "mean":0.672667, + "std":0.01159 + }, + "oos_recall_wilson95":[ + { + "mean":0.642977, + "std":0.011814 + }, + { + "mean":0.701035, + "std":0.011277 + } + ], + "oos_precision":{ + "mean":0.983884, + "std":0.002742 + }, + "oos_f1":{ + "mean":0.799014, + "std":0.009057 + }, + "oos_misroute_rate":{ + "mean":0.327333, + "std":0.01159 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.109212, + "std":0.004034 + }, + "llm_cost_usd_per_1k":{ + "mean":0.04028, + "std":0.001475 + }, + "error_recovery_rate":{ + "mean":0.408394, + "std":0.004985 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":245.333333, + "std":10.503968 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.003182, + "std":0.00058 + }, + "oos_auroc":{ + "mean":0.75909, + "std":0.065239 + }, + "oos_auprc":{ + "mean":0.231192, + "std":0.041803 + } + }, + "test":{ + "aurc":{ + "mean":0.017057, + "std":0.00057 + }, + "oos_auroc":{ + "mean":0.820728, + "std":0.04435 + }, + "oos_auprc":{ + "mean":0.653848, + "std":0.029452 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.003175, + "std":0.000582 + }, + "oos_auroc":{ + "mean":0.762237, + "std":0.062077 + }, + "oos_auprc":{ + "mean":0.253723, + "std":0.026802 + } + }, + "test":{ + "aurc":{ + "mean":0.016492, + "std":0.000608 + }, + "oos_auroc":{ + "mean":0.82563, + "std":0.041939 + }, + "oos_auprc":{ + "mean":0.681673, + "std":0.026399 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.003642, + "std":0.00032 + }, + "oos_auroc":{ + "mean":0.749059, + "std":0.059641 + }, + "oos_auprc":{ + "mean":0.157387, + "std":0.037008 + } + }, + "test":{ + "aurc":{ + "mean":0.022227, + "std":0.001433 + }, + "oos_auroc":{ + "mean":0.797167, + "std":0.040388 + }, + "oos_auprc":{ + "mean":0.527717, + "std":0.034943 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.003204, + "std":0.000544 + }, + "oos_auroc":{ + "mean":0.757556, + "std":0.06915 + }, + "oos_auprc":{ + "mean":0.214348, + "std":0.044987 + } + }, + "test":{ + "aurc":{ + "mean":0.018031, + "std":0.000623 + }, + "oos_auroc":{ + "mean":0.816601, + "std":0.047245 + }, + "oos_auprc":{ + "mean":0.629625, + "std":0.031493 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"entropy", + "hybrid":{ + "tau":{ + "mean":-3.015192, + "std":0.093357 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.973226, + "std":0.002115 + }, + "validation_selective_risk":{ + "mean":0.015578, + "std":3.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.019745, + "std":4.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.912727, + "std":0.009248 + }, + "selective_risk":{ + "mean":0.056075, + "std":0.005148 + }, + "accuracy_8":{ + "mean":0.909091, + "std":0.001924 + }, + "oos_recall":{ + "mean":0.558, + "std":0.006245 + }, + "oos_recall_wilson95":[ + { + "mean":0.527057, + "std":0.006265 + }, + { + "mean":0.588499, + "std":0.006177 + } + ], + "oos_precision":{ + "mean":0.971563, + "std":0.002602 + }, + "oos_f1":{ + "mean":0.708858, + "std":0.005212 + }, + "oos_misroute_rate":{ + "mean":0.442, + "std":0.006245 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.243, + "std":0.025515 + }, + "llm_call_rate":{ + "mean":0.087273, + "std":0.009248 + }, + "llm_cost_usd_per_1k":{ + "mean":0.032166, + "std":0.003404 + }, + "error_recovery_rate":{ + "mean":0.24207, + "std":0.041309 + }, + "recoverable_caught":{ + "mean":0.592698, + "std":0.100574 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":146.0, + "std":29.597297 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.353885, + "std":0.020558 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.97129, + "std":0.002115 + }, + "validation_selective_risk":{ + "mean":0.015609, + "std":3.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.019784, + "std":4.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.916364, + "std":0.009349 + }, + "selective_risk":{ + "mean":0.058109, + "std":0.003879 + }, + "accuracy_8":{ + "mean":0.908909, + "std":0.001734 + }, + "oos_recall":{ + "mean":0.559333, + "std":0.010214 + }, + "oos_recall_wilson95":[ + { + "mean":0.528397, + "std":0.010254 + }, + { + "mean":0.589815, + "std":0.010097 + } + ], + "oos_precision":{ + "mean":0.969405, + "std":0.002138 + }, + "oos_f1":{ + "mean":0.709326, + "std":0.007759 + }, + "oos_misroute_rate":{ + "mean":0.440667, + "std":0.010214 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.252, + "std":0.021656 + }, + "llm_call_rate":{ + "mean":0.083636, + "std":0.009349 + }, + "llm_cost_usd_per_1k":{ + "mean":0.030835, + "std":0.003442 + }, + "error_recovery_rate":{ + "mean":0.23162, + "std":0.036532 + }, + "recoverable_caught":{ + "mean":0.566966, + "std":0.087191 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":139.666667, + "std":26.764404 + } + } + }, + "entropy":{ + "tau":{ + "mean":-3.015192, + "std":0.093357 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.973226, + "std":0.002115 + }, + "validation_selective_risk":{ + "mean":0.015578, + "std":3.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.019745, + "std":4.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.912727, + "std":0.009248 + }, + "selective_risk":{ + "mean":0.056075, + "std":0.005148 + }, + "accuracy_8":{ + "mean":0.909091, + "std":0.001924 + }, + "oos_recall":{ + "mean":0.558, + "std":0.006245 + }, + "oos_recall_wilson95":[ + { + "mean":0.527057, + "std":0.006265 + }, + { + "mean":0.588499, + "std":0.006177 + } + ], + "oos_precision":{ + "mean":0.971563, + "std":0.002602 + }, + "oos_f1":{ + "mean":0.708858, + "std":0.005212 + }, + "oos_misroute_rate":{ + "mean":0.442, + "std":0.006245 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.243, + "std":0.025515 + }, + "llm_call_rate":{ + "mean":0.087273, + "std":0.009248 + }, + "llm_cost_usd_per_1k":{ + "mean":0.032166, + "std":0.003404 + }, + "error_recovery_rate":{ + "mean":0.24207, + "std":0.041309 + }, + "recoverable_caught":{ + "mean":0.592698, + "std":0.100574 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":146.0, + "std":29.597297 + } + } + }, + "margin":{ + "tau":{ + "mean":1.293326, + "std":0.07364 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.95828, + "std":0.003311 + }, + "validation_selective_risk":{ + "mean":0.015597, + "std":0.000143 + }, + "validation_risk_upper95":{ + "mean":0.019801, + "std":0.000153 + }, + "test":{ + "coverage":{ + "mean":0.911212, + "std":0.010289 + }, + "selective_risk":{ + "mean":0.061782, + "std":0.001222 + }, + "accuracy_8":{ + "mean":0.908, + "std":0.000909 + }, + "oos_recall":{ + "mean":0.56, + "std":0.005292 + }, + "oos_recall_wilson95":[ + { + "mean":0.529063, + "std":0.005312 + }, + { + "mean":0.590478, + "std":0.005231 + } + ], + "oos_precision":{ + "mean":0.969995, + "std":0.002285 + }, + "oos_f1":{ + "mean":0.710049, + "std":0.00364 + }, + "oos_misroute_rate":{ + "mean":0.44, + "std":0.005292 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.269667, + "std":0.010263 + }, + "llm_call_rate":{ + "mean":0.088788, + "std":0.010289 + }, + "llm_cost_usd_per_1k":{ + "mean":0.032747, + "std":0.003792 + }, + "error_recovery_rate":{ + "mean":0.215458, + "std":0.01691 + }, + "recoverable_caught":{ + "mean":0.527551, + "std":0.040443 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":129.666667, + "std":14.74223 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.539768, + "std":0.013822 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.970108, + "std":0.000672 + }, + "validation_selective_risk":{ + "mean":0.015628, + "std":1.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.019808, + "std":1.4e-05 + }, + "test":{ + "coverage":{ + "mean":0.918303, + "std":0.003968 + }, + "selective_risk":{ + "mean":0.06012, + "std":0.001417 + }, + "accuracy_8":{ + "mean":0.90897, + "std":0.001211 + }, + "oos_recall":{ + "mean":0.561, + "std":0.009165 + }, + "oos_recall_wilson95":[ + { + "mean":0.530069, + "std":0.009201 + }, + { + "mean":0.591464, + "std":0.009059 + } + ], + "oos_precision":{ + "mean":0.970583, + "std":0.000464 + }, + "oos_f1":{ + "mean":0.710998, + "std":0.007473 + }, + "oos_misroute_rate":{ + "mean":0.439, + "std":0.009165 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.262667, + "std":0.011846 + }, + "llm_call_rate":{ + "mean":0.081697, + "std":0.003968 + }, + "llm_cost_usd_per_1k":{ + "mean":0.030123, + "std":0.001458 + }, + "error_recovery_rate":{ + "mean":0.22634, + "std":0.026198 + }, + "recoverable_caught":{ + "mean":0.554001, + "std":0.060541 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":136.333333, + "std":20.55075 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":-1.420356, + "std":2.635913 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.029355, + "std":0.00129 + }, + "validation_risk_upper95":{ + "mean":0.034766, + "std":0.001395 + }, + "test":{ + "coverage":{ + "mean":0.998606, + "std":0.000378 + }, + "selective_risk":{ + "mean":0.108212, + "std":0.00391 + }, + "accuracy_8":{ + "mean":0.891455, + "std":0.004229 + }, + "oos_recall":{ + "mean":0.466667, + "std":0.022502 + }, + "oos_recall_wilson95":[ + { + "mean":0.435953, + "std":0.022337 + }, + { + "mean":0.497636, + "std":0.022494 + } + ], + "oos_precision":{ + "mean":0.982989, + "std":0.006077 + }, + "oos_f1":{ + "mean":0.632714, + "std":0.021775 + }, + "oos_misroute_rate":{ + "mean":0.533333, + "std":0.022502 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.530667, + "std":0.020793 + }, + "llm_call_rate":{ + "mean":0.001394, + "std":0.000378 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000514, + "std":0.000141 + }, + "error_recovery_rate":{ + "mean":0.006687, + "std":0.003334 + }, + "recoverable_caught":{ + "mean":0.016441, + "std":0.008354 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":4.0, + "std":2.0 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.06626, + "std":0.007311 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.029355, + "std":0.00129 + }, + "validation_risk_upper95":{ + "mean":0.034766, + "std":0.001395 + }, + "test":{ + "coverage":{ + "mean":0.998485, + "std":0.000915 + }, + "selective_risk":{ + "mean":0.108044, + "std":0.003231 + }, + "accuracy_8":{ + "mean":0.891515, + "std":0.003789 + }, + "oos_recall":{ + "mean":0.466667, + "std":0.020599 + }, + "oos_recall_wilson95":[ + { + "mean":0.435949, + "std":0.020448 + }, + { + "mean":0.497639, + "std":0.020593 + } + ], + "oos_precision":{ + "mean":0.983004, + "std":0.006036 + }, + "oos_f1":{ + "mean":0.632747, + "std":0.020038 + }, + "oos_misroute_rate":{ + "mean":0.533333, + "std":0.020599 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.53, + "std":0.017349 + }, + "llm_call_rate":{ + "mean":0.001515, + "std":0.000915 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000558, + "std":0.000338 + }, + "error_recovery_rate":{ + "mean":0.007145, + "std":0.003306 + }, + "recoverable_caught":{ + "mean":0.017511, + "std":0.008179 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":4.333333, + "std":2.081666 + } + } + }, + "entropy":{ + "tau":{ + "mean":-4.471986, + "std":0.007829 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.029355, + "std":0.00129 + }, + "validation_risk_upper95":{ + "mean":0.034766, + "std":0.001395 + }, + "test":{ + "coverage":{ + "mean":0.999152, + "std":0.000378 + }, + "selective_risk":{ + "mean":0.108638, + "std":0.00375 + }, + "accuracy_8":{ + "mean":0.891152, + "std":0.004087 + }, + "oos_recall":{ + "mean":0.465, + "std":0.021932 + }, + "oos_recall_wilson95":[ + { + "mean":0.434298, + "std":0.021768 + }, + { + "mean":0.495969, + "std":0.021927 + } + ], + "oos_precision":{ + "mean":0.98293, + "std":0.006103 + }, + "oos_f1":{ + "mean":0.631177, + "std":0.021306 + }, + "oos_misroute_rate":{ + "mean":0.535, + "std":0.021932 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.533333, + "std":0.020232 + }, + "llm_call_rate":{ + "mean":0.000848, + "std":0.000378 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000313, + "std":0.000141 + }, + "error_recovery_rate":{ + "mean":0.003882, + "std":0.000926 + }, + "recoverable_caught":{ + "mean":0.009523, + "std":0.002383 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":2.333333, + "std":0.57735 + } + } + }, + "margin":{ + "tau":{ + "mean":0.01461, + "std":0.011639 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.029355, + "std":0.00129 + }, + "validation_risk_upper95":{ + "mean":0.034766, + "std":0.001395 + }, + "test":{ + "coverage":{ + "mean":0.998788, + "std":0.001315 + }, + "selective_risk":{ + "mean":0.108313, + "std":0.003832 + }, + "accuracy_8":{ + "mean":0.891212, + "std":0.003888 + }, + "oos_recall":{ + "mean":0.465333, + "std":0.020984 + }, + "oos_recall_wilson95":[ + { + "mean":0.434627, + "std":0.020826 + }, + { + "mean":0.496305, + "std":0.020981 + } + ], + "oos_precision":{ + "mean":0.982258, + "std":0.006868 + }, + "oos_f1":{ + "mean":0.631361, + "std":0.0205 + }, + "oos_misroute_rate":{ + "mean":0.534667, + "std":0.020984 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.531667, + "std":0.020207 + }, + "llm_call_rate":{ + "mean":0.001212, + "std":0.001315 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000447, + "std":0.000484 + }, + "error_recovery_rate":{ + "mean":0.004401, + "std":0.003405 + }, + "recoverable_caught":{ + "mean":0.010825, + "std":0.008487 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":2.666667, + "std":2.081666 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.107157, + "std":0.010495 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.029355, + "std":0.00129 + }, + "validation_risk_upper95":{ + "mean":0.034766, + "std":0.001395 + }, + "test":{ + "coverage":{ + "mean":0.998242, + "std":0.000639 + }, + "selective_risk":{ + "mean":0.107888, + "std":0.003509 + }, + "accuracy_8":{ + "mean":0.891576, + "std":0.004067 + }, + "oos_recall":{ + "mean":0.467333, + "std":0.021733 + }, + "oos_recall_wilson95":[ + { + "mean":0.436613, + "std":0.021577 + }, + { + "mean":0.498304, + "std":0.021723 + } + ], + "oos_precision":{ + "mean":0.982309, + "std":0.006611 + }, + "oos_f1":{ + "mean":0.633205, + "std":0.021216 + }, + "oos_misroute_rate":{ + "mean":0.532667, + "std":0.021733 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.529, + "std":0.019079 + }, + "llm_call_rate":{ + "mean":0.001758, + "std":0.000639 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000647, + "std":0.000236 + }, + "error_recovery_rate":{ + "mean":0.007764, + "std":0.001853 + }, + "recoverable_caught":{ + "mean":0.019045, + "std":0.004766 + }, + "small_errors":{ + "mean":600.666667, + "std":22.188586 + }, + "recoverable_errors":{ + "mean":245.333333, + "std":10.503968 + }, + "recovered_errors":{ + "mean":4.666667, + "std":1.154701 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.97086, + "std":0.001777 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.113152, + "std":0.003364 + }, + "accuracy_8":{ + "mean":0.886848, + "std":0.003364 + }, + "oos_recall":{ + "mean":0.439, + "std":0.019287 + }, + "oos_recall_wilson95":[ + { + "mean":0.408548, + "std":0.019079 + }, + { + "mean":0.469919, + "std":0.019348 + } + ], + "oos_precision":{ + "mean":0.984156, + "std":0.006349 + }, + "oos_f1":{ + "mean":0.607041, + "std":0.019512 + }, + "oos_misroute_rate":{ + "mean":0.561, + "std":0.019287 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.561, + "std":0.019287 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.886848, + "std":0.003364 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.934121, + "std":0.001961 + }, + "oos_recall":{ + "mean":0.665333, + "std":0.010693 + }, + "oos_recall_wilson95":[ + { + "mean":0.635508, + "std":0.010885 + }, + { + "mean":0.693893, + "std":0.010418 + } + ], + "oos_precision":{ + "mean":0.983234, + "std":0.001916 + }, + "oos_f1":{ + "mean":0.793607, + "std":0.008197 + }, + "oos_misroute_rate":{ + "mean":0.334667, + "std":0.010693 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.113152, + "std":0.003364 + }, + "llm_cost_usd_per_1k":{ + "mean":0.041726, + "std":0.001232 + }, + "error_recovery_rate":{ + "mean":0.417782, + "std":0.001455 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":260.0, + "std":7.81025 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.004072, + "std":0.000635 + }, + "oos_auroc":{ + "mean":0.849659, + "std":0.060998 + }, + "oos_auprc":{ + "mean":0.32257, + "std":0.032253 + } + }, + "test":{ + "aurc":{ + "mean":0.017458, + "std":0.001183 + }, + "oos_auroc":{ + "mean":0.887299, + "std":0.042232 + }, + "oos_auprc":{ + "mean":0.751853, + "std":0.040869 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.003962, + "std":0.000694 + }, + "oos_auroc":{ + "mean":0.85614, + "std":0.057685 + }, + "oos_auprc":{ + "mean":0.363486, + "std":0.050539 + } + }, + "test":{ + "aurc":{ + "mean":0.017275, + "std":0.001212 + }, + "oos_auroc":{ + "mean":0.892384, + "std":0.039936 + }, + "oos_auprc":{ + "mean":0.768229, + "std":0.036108 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.004654, + "std":0.000323 + }, + "oos_auroc":{ + "mean":0.83963, + "std":0.074462 + }, + "oos_auprc":{ + "mean":0.248116, + "std":0.033531 + } + }, + "test":{ + "aurc":{ + "mean":0.018948, + "std":0.000685 + }, + "oos_auroc":{ + "mean":0.875058, + "std":0.050686 + }, + "oos_auprc":{ + "mean":0.703706, + "std":0.046713 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.003826, + "std":0.000427 + }, + "oos_auroc":{ + "mean":0.844827, + "std":0.066289 + }, + "oos_auprc":{ + "mean":0.286413, + "std":0.035477 + } + }, + "test":{ + "aurc":{ + "mean":0.017007, + "std":0.001325 + }, + "oos_auroc":{ + "mean":0.88298, + "std":0.044695 + }, + "oos_auprc":{ + "mean":0.733513, + "std":0.042739 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":0.042789, + "std":1.179061 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.975806, + "std":0.001406 + }, + "validation_selective_risk":{ + "mean":0.015537, + "std":2.2e-05 + }, + "validation_risk_upper95":{ + "mean":0.019692, + "std":2.8e-05 + }, + "test":{ + "coverage":{ + "mean":0.917576, + "std":0.013675 + }, + "selective_risk":{ + "mean":0.0607, + "std":0.006652 + }, + "accuracy_8":{ + "mean":0.908121, + "std":0.001857 + }, + "oos_recall":{ + "mean":0.551333, + "std":0.005033 + }, + "oos_recall_wilson95":[ + { + "mean":0.52037, + "std":0.005046 + }, + { + "mean":0.581904, + "std":0.004982 + } + ], + "oos_precision":{ + "mean":0.976391, + "std":0.000834 + }, + "oos_f1":{ + "mean":0.70472, + "std":0.003964 + }, + "oos_misroute_rate":{ + "mean":0.448667, + "std":0.005033 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.270667, + "std":0.037099 + }, + "llm_call_rate":{ + "mean":0.082424, + "std":0.013675 + }, + "llm_cost_usd_per_1k":{ + "mean":0.030355, + "std":0.005025 + }, + "error_recovery_rate":{ + "mean":0.261625, + "std":0.042448 + }, + "recoverable_caught":{ + "mean":0.626181, + "std":0.10115 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":163.333333, + "std":30.746273 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.558169, + "std":0.02405 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.975161, + "std":0.001117 + }, + "validation_selective_risk":{ + "mean":0.015547, + "std":1.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.019705, + "std":2.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.913758, + "std":0.005611 + }, + "selective_risk":{ + "mean":0.059944, + "std":0.003819 + }, + "accuracy_8":{ + "mean":0.908121, + "std":0.002508 + }, + "oos_recall":{ + "mean":0.548333, + "std":0.008505 + }, + "oos_recall_wilson95":[ + { + "mean":0.517365, + "std":0.008524 + }, + { + "mean":0.578932, + "std":0.008421 + } + ], + "oos_precision":{ + "mean":0.976857, + "std":0.00139 + }, + "oos_f1":{ + "mean":0.702364, + "std":0.006617 + }, + "oos_misroute_rate":{ + "mean":0.451667, + "std":0.008505 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.269, + "std":0.017349 + }, + "llm_call_rate":{ + "mean":0.086242, + "std":0.005611 + }, + "llm_cost_usd_per_1k":{ + "mean":0.031758, + "std":0.002059 + }, + "error_recovery_rate":{ + "mean":0.267115, + "std":0.035735 + }, + "recoverable_caught":{ + "mean":0.639325, + "std":0.085026 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":166.666667, + "std":26.727015 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.352729, + "std":0.041444 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.975484, + "std":0.001117 + }, + "validation_selective_risk":{ + "mean":0.015542, + "std":1.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.019699, + "std":2.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.910545, + "std":0.005203 + }, + "selective_risk":{ + "mean":0.058628, + "std":0.003193 + }, + "accuracy_8":{ + "mean":0.907818, + "std":0.001192 + }, + "oos_recall":{ + "mean":0.55, + "std":0.005292 + }, + "oos_recall_wilson95":[ + { + "mean":0.519033, + "std":0.005305 + }, + { + "mean":0.580584, + "std":0.005238 + } + ], + "oos_precision":{ + "mean":0.977502, + "std":0.002528 + }, + "oos_f1":{ + "mean":0.703915, + "std":0.003992 + }, + "oos_misroute_rate":{ + "mean":0.45, + "std":0.005292 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.254667, + "std":0.016503 + }, + "llm_call_rate":{ + "mean":0.089455, + "std":0.005203 + }, + "llm_cost_usd_per_1k":{ + "mean":0.032937, + "std":0.001914 + }, + "error_recovery_rate":{ + "mean":0.267755, + "std":0.030733 + }, + "recoverable_caught":{ + "mean":0.640854, + "std":0.072938 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":167.0, + "std":23.643181 + } + } + }, + "margin":{ + "tau":{ + "mean":1.89276, + "std":0.302717 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.972796, + "std":0.002818 + }, + "validation_selective_risk":{ + "mean":0.015585, + "std":4.5e-05 + }, + "validation_risk_upper95":{ + "mean":0.019753, + "std":5.7e-05 + }, + "test":{ + "coverage":{ + "mean":0.916242, + "std":0.012683 + }, + "selective_risk":{ + "mean":0.06086, + "std":0.006635 + }, + "accuracy_8":{ + "mean":0.907636, + "std":0.001924 + }, + "oos_recall":{ + "mean":0.55, + "std":0.00781 + }, + "oos_recall_wilson95":[ + { + "mean":0.519035, + "std":0.007831 + }, + { + "mean":0.580583, + "std":0.00773 + } + ], + "oos_precision":{ + "mean":0.972944, + "std":0.005108 + }, + "oos_f1":{ + "mean":0.702702, + "std":0.005018 + }, + "oos_misroute_rate":{ + "mean":0.45, + "std":0.00781 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.276667, + "std":0.033531 + }, + "llm_call_rate":{ + "mean":0.083758, + "std":0.012683 + }, + "llm_cost_usd_per_1k":{ + "mean":0.030845, + "std":0.004667 + }, + "error_recovery_rate":{ + "mean":0.261659, + "std":0.042105 + }, + "recoverable_caught":{ + "mean":0.626232, + "std":0.099993 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":163.333333, + "std":30.435725 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.744825, + "std":0.048932 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.974946, + "std":0.002587 + }, + "validation_selective_risk":{ + "mean":0.015551, + "std":4.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.01971, + "std":5.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.918, + "std":0.013208 + }, + "selective_risk":{ + "mean":0.060341, + "std":0.007088 + }, + "accuracy_8":{ + "mean":0.908364, + "std":0.002049 + }, + "oos_recall":{ + "mean":0.552667, + "std":0.007024 + }, + "oos_recall_wilson95":[ + { + "mean":0.521708, + "std":0.007043 + }, + { + "mean":0.583223, + "std":0.006951 + } + ], + "oos_precision":{ + "mean":0.974758, + "std":0.003657 + }, + "oos_f1":{ + "mean":0.705363, + "std":0.00488 + }, + "oos_misroute_rate":{ + "mean":0.447333, + "std":0.007024 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.271667, + "std":0.035796 + }, + "llm_call_rate":{ + "mean":0.082, + "std":0.013208 + }, + "llm_cost_usd_per_1k":{ + "mean":0.030202, + "std":0.004856 + }, + "error_recovery_rate":{ + "mean":0.262152, + "std":0.043004 + }, + "recoverable_caught":{ + "mean":0.627439, + "std":0.102437 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":163.666667, + "std":31.085902 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":-0.478771, + "std":1.269717 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.02914, + "std":0.001777 + }, + "validation_risk_upper95":{ + "mean":0.034533, + "std":0.001921 + }, + "test":{ + "coverage":{ + "mean":0.996909, + "std":0.002212 + }, + "selective_risk":{ + "mean":0.111074, + "std":0.002037 + }, + "accuracy_8":{ + "mean":0.888061, + "std":0.002338 + }, + "oos_recall":{ + "mean":0.445333, + "std":0.013577 + }, + "oos_recall_wilson95":[ + { + "mean":0.414804, + "std":0.01344 + }, + { + "mean":0.476281, + "std":0.01361 + } + ], + "oos_precision":{ + "mean":0.98372, + "std":0.004927 + }, + "oos_f1":{ + "mean":0.613052, + "std":0.013746 + }, + "oos_misroute_rate":{ + "mean":0.554667, + "std":0.013577 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.548667, + "std":0.010116 + }, + "llm_call_rate":{ + "mean":0.003091, + "std":0.002212 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001139, + "std":0.000814 + }, + "error_recovery_rate":{ + "mean":0.012635, + "std":0.011381 + }, + "recoverable_caught":{ + "mean":0.030255, + "std":0.027295 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":8.0, + "std":7.211103 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.196451, + "std":0.026103 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.02914, + "std":0.001777 + }, + "validation_risk_upper95":{ + "mean":0.034533, + "std":0.001921 + }, + "test":{ + "coverage":{ + "mean":0.998242, + "std":0.001927 + }, + "selective_risk":{ + "mean":0.1122, + "std":0.002539 + }, + "accuracy_8":{ + "mean":0.887455, + "std":0.002834 + }, + "oos_recall":{ + "mean":0.442, + "std":0.016523 + }, + "oos_recall_wilson95":[ + { + "mean":0.41151, + "std":0.01635 + }, + { + "mean":0.472933, + "std":0.016569 + } + ], + "oos_precision":{ + "mean":0.984286, + "std":0.006247 + }, + "oos_f1":{ + "mean":0.609968, + "std":0.016833 + }, + "oos_misroute_rate":{ + "mean":0.558, + "std":0.016523 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.555, + "std":0.013856 + }, + "llm_call_rate":{ + "mean":0.001758, + "std":0.001927 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000646, + "std":0.000708 + }, + "error_recovery_rate":{ + "mean":0.006842, + "std":0.007112 + }, + "recoverable_caught":{ + "mean":0.016395, + "std":0.017083 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":4.333333, + "std":4.50925 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.984239, + "std":0.063475 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.02914, + "std":0.001777 + }, + "validation_risk_upper95":{ + "mean":0.034533, + "std":0.001921 + }, + "test":{ + "coverage":{ + "mean":0.996667, + "std":0.002922 + }, + "selective_risk":{ + "mean":0.111527, + "std":0.001953 + }, + "accuracy_8":{ + "mean":0.887576, + "std":0.002753 + }, + "oos_recall":{ + "mean":0.443, + "std":0.016093 + }, + "oos_recall_wilson95":[ + { + "mean":0.412499, + "std":0.015925 + }, + { + "mean":0.473937, + "std":0.016139 + } + ], + "oos_precision":{ + "mean":0.984333, + "std":0.006189 + }, + "oos_f1":{ + "mean":0.610932, + "std":0.016369 + }, + "oos_misroute_rate":{ + "mean":0.557, + "std":0.016093 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.55, + "std":0.009644 + }, + "llm_call_rate":{ + "mean":0.003333, + "std":0.002922 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001226, + "std":0.001075 + }, + "error_recovery_rate":{ + "mean":0.010532, + "std":0.009121 + }, + "recoverable_caught":{ + "mean":0.025205, + "std":0.021828 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":6.666667, + "std":5.773503 + } + } + }, + "margin":{ + "tau":{ + "mean":0.003754, + "std":0.003737 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.02914, + "std":0.001777 + }, + "validation_risk_upper95":{ + "mean":0.034533, + "std":0.001921 + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.000278 + }, + "selective_risk":{ + "mean":0.112957, + "std":0.003192 + }, + "accuracy_8":{ + "mean":0.88703, + "std":0.003213 + }, + "oos_recall":{ + "mean":0.44, + "std":0.01852 + }, + "oos_recall_wilson95":[ + { + "mean":0.409536, + "std":0.018322 + }, + { + "mean":0.470924, + "std":0.018577 + } + ], + "oos_precision":{ + "mean":0.984201, + "std":0.006307 + }, + "oos_f1":{ + "mean":0.608016, + "std":0.018753 + }, + "oos_misroute_rate":{ + "mean":0.56, + "std":0.01852 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.559667, + "std":0.018339 + }, + "llm_call_rate":{ + "mean":0.000424, + "std":0.000278 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000155, + "std":0.000101 + }, + "error_recovery_rate":{ + "mean":0.001581, + "std":0.001582 + }, + "recoverable_caught":{ + "mean":0.003778, + "std":0.003774 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":1.0, + "std":1.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.245849, + "std":0.031899 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.02914, + "std":0.001777 + }, + "validation_risk_upper95":{ + "mean":0.034533, + "std":0.001921 + }, + "test":{ + "coverage":{ + "mean":0.998061, + "std":0.001961 + }, + "selective_risk":{ + "mean":0.11149, + "std":0.002587 + }, + "accuracy_8":{ + "mean":0.887879, + "std":0.002548 + }, + "oos_recall":{ + "mean":0.444, + "std":0.014799 + }, + "oos_recall_wilson95":[ + { + "mean":0.413486, + "std":0.014647 + }, + { + "mean":0.474942, + "std":0.014837 + } + ], + "oos_precision":{ + "mean":0.983659, + "std":0.004998 + }, + "oos_f1":{ + "mean":0.611764, + "std":0.014952 + }, + "oos_misroute_rate":{ + "mean":0.556, + "std":0.014799 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.552, + "std":0.013748 + }, + "llm_call_rate":{ + "mean":0.001939, + "std":0.001961 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000715, + "std":0.000722 + }, + "error_recovery_rate":{ + "mean":0.00947, + "std":0.011371 + }, + "recoverable_caught":{ + "mean":0.022708, + "std":0.027323 + }, + "small_errors":{ + "mean":622.333333, + "std":18.502252 + }, + "recoverable_errors":{ + "mean":260.0, + "std":7.81025 + }, + "recovered_errors":{ + "mean":6.0, + "std":7.211103 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.111515, + "std":0.002989, + "values":[ + 0.114909, + 0.109273, + 0.110364 + ] + }, + "accuracy_8":{ + "mean":0.888485, + "std":0.002989, + "values":[ + 0.885091, + 0.890727, + 0.889636 + ] + }, + "oos_recall":{ + "mean":0.448333, + "std":0.015535, + "values":[ + 0.431, + 0.461, + 0.453 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.417775, + "std":0.01537, + "values":[ + 0.400628, + 0.430312, + 0.422385 + ] + }, + { + "mean":0.479287, + "std":0.015581, + "values":[ + 0.4619, + 0.491986, + 0.483974 + ] + } + ], + "oos_precision":{ + "mean":0.983115, + "std":0.007196, + "values":[ + 0.979545, + 0.991398, + 0.978402 + ] + }, + "oos_f1":{ + "mean":0.615746, + "std":0.015671, + "values":[ + 0.598611, + 0.629352, + 0.619275 + ] + }, + "oos_misroute_rate":{ + "mean":0.551667, + "std":0.015535, + "values":[ + 0.569, + 0.539, + 0.547 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.551667, + "std":0.015535, + "values":[ + 0.569, + 0.539, + 0.547 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":613.333333, + "std":16.441817, + "values":[ + 632, + 601, + 607 + ] + }, + "recoverable_errors":{ + "mean":253.666667, + "std":10.263203, + "values":[ + 265, + 251, + 245 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.888485, + "std":0.002989, + "values":[ + 0.885091, + 0.890727, + 0.889636 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.934606, + "std":0.001589, + "values":[ + 0.933273, + 0.936364, + 0.934182 + ] + }, + "oos_recall":{ + "mean":0.668333, + "std":0.007572, + "values":[ + 0.663, + 0.677, + 0.665 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.63856, + "std":0.007712, + "values":[ + 0.633128, + 0.647387, + 0.635163 + ] + }, + { + "mean":0.696819, + "std":0.007374, + "values":[ + 0.691624, + 0.705259, + 0.693574 + ] + } + ], + "oos_precision":{ + "mean":0.983314, + "std":0.001845, + "values":[ + 0.982222, + 0.985444, + 0.982275 + ] + }, + "oos_f1":{ + "mean":0.795778, + "std":0.005959, + "values":[ + 0.791642, + 0.802608, + 0.793083 + ] + }, + "oos_misroute_rate":{ + "mean":0.331667, + "std":0.007572, + "values":[ + 0.337, + 0.323, + 0.335 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.111515, + "std":0.002989, + "values":[ + 0.114909, + 0.109273, + 0.110364 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.041126, + "std":0.001088, + "values":[ + 0.04236, + 0.040306, + 0.04071 + ] + }, + "error_recovery_rate":{ + "mean":0.413522, + "std":0.008612, + "values":[ + 0.419304, + 0.417637, + 0.403624 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":613.333333, + "std":16.441817, + "values":[ + 632, + 601, + 607 + ] + }, + "recoverable_errors":{ + "mean":253.666667, + "std":10.263203, + "values":[ + 265, + 251, + 245 + ] + }, + "recovered_errors":{ + "mean":253.666667, + "std":10.263203, + "values":[ + 265, + 251, + 245 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.003465, + "std":0.000893, + "values":[ + 0.00334, + 0.004413, + 0.002641 + ] + }, + "oos_auroc":{ + "mean":0.821469, + "std":0.015648, + "values":[ + 0.82596, + 0.804067, + 0.83438 + ] + }, + "oos_auprc":{ + "mean":0.296342, + "std":0.02377, + "values":[ + 0.290332, + 0.32254, + 0.276154 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.016924, + "std":0.001135, + "values":[ + 0.016095, + 0.018218, + 0.016458 + ] + }, + "oos_auroc":{ + "mean":0.865529, + "std":0.015824, + "values":[ + 0.848922, + 0.880432, + 0.867232 + ] + }, + "oos_auprc":{ + "mean":0.714406, + "std":0.027997, + "values":[ + 0.717843, + 0.740526, + 0.684849 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.00341, + "std":0.000841, + "values":[ + 0.003161, + 0.004347, + 0.002721 + ] + }, + "oos_auroc":{ + "mean":0.826863, + "std":0.011692, + "values":[ + 0.833307, + 0.813367, + 0.833917 + ] + }, + "oos_auprc":{ + "mean":0.320232, + "std":0.040148, + "values":[ + 0.312925, + 0.363531, + 0.284239 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.016584, + "std":0.001271, + "values":[ + 0.015878, + 0.018052, + 0.015824 + ] + }, + "oos_auroc":{ + "mean":0.870068, + "std":0.015696, + "values":[ + 0.855572, + 0.886737, + 0.867894 + ] + }, + "oos_auprc":{ + "mean":0.73506, + "std":0.02599, + "values":[ + 0.736886, + 0.760089, + 0.708205 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.004193, + "std":0.000857, + "values":[ + 0.004329, + 0.004974, + 0.003277 + ] + }, + "oos_auroc":{ + "mean":0.804332, + "std":0.021945, + "values":[ + 0.817157, + 0.778993, + 0.816847 + ] + }, + "oos_auprc":{ + "mean":0.218964, + "std":0.032626, + "values":[ + 0.254566, + 0.211829, + 0.190496 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.019977, + "std":0.002268, + "values":[ + 0.018165, + 0.019245, + 0.02252 + ] + }, + "oos_auroc":{ + "mean":0.845524, + "std":0.010822, + "values":[ + 0.835752, + 0.857156, + 0.843665 + ] + }, + "oos_auprc":{ + "mean":0.64012, + "std":0.064674, + "values":[ + 0.668942, + 0.685372, + 0.566045 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.003443, + "std":0.000824, + "values":[ + 0.003427, + 0.004276, + 0.002627 + ] + }, + "oos_auroc":{ + "mean":0.817294, + "std":0.02169, + "values":[ + 0.820397, + 0.79422, + 0.837267 + ] + }, + "oos_auprc":{ + "mean":0.264514, + "std":0.005719, + "values":[ + 0.271117, + 0.261149, + 0.261276 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.016937, + "std":0.001264, + "values":[ + 0.015478, + 0.017718, + 0.017614 + ] + }, + "oos_auroc":{ + "mean":0.861651, + "std":0.014071, + "values":[ + 0.845472, + 0.871033, + 0.868448 + ] + }, + "oos_auprc":{ + "mean":0.694384, + "std":0.0272, + "values":[ + 0.700915, + 0.717723, + 0.664513 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "entropy", + "msp_t", + "entropy" + ], + "hybrid":{ + "tau":{ + "mean":-1.18306, + "std":1.810917, + "values":[ + -1.318183, + 0.691634, + -2.922631 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.975054, + "std":0.002266, + "values":[ + 0.974839, + 0.977419, + 0.972903 + ] + }, + "validation_selective_risk":{ + "mean":0.015549, + "std":3.6e-05, + "values":[ + 0.015553, + 0.015512, + 0.015584 + ] + }, + "validation_risk_upper95":{ + "mean":0.019708, + "std":4.6e-05, + "values":[ + 0.019712, + 0.01966, + 0.019751 + ] + }, + "test":{ + "coverage":{ + "mean":0.914848, + "std":0.015874, + "values":[ + 0.907273, + 0.933091, + 0.904182 + ] + }, + "selective_risk":{ + "mean":0.058597, + "std":0.008491, + "values":[ + 0.055511, + 0.0682, + 0.052081 + ] + }, + "accuracy_8":{ + "mean":0.908242, + "std":0.001994, + "values":[ + 0.908909, + 0.906, + 0.909818 + ] + }, + "oos_recall":{ + "mean":0.554, + "std":0.007211, + "values":[ + 0.556, + 0.546, + 0.56 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.523045, + "std":0.007231, + "values":[ + 0.525049, + 0.515024, + 0.529062 + ] + }, + { + "mean":0.584542, + "std":0.007136, + "values":[ + 0.586523, + 0.576624, + 0.590479 + ] + } + ], + "oos_precision":{ + "mean":0.97368, + "std":0.004227, + "values":[ + 0.975439, + 0.976744, + 0.968858 + ] + }, + "oos_f1":{ + "mean":0.706163, + "std":0.005003, + "values":[ + 0.70828, + 0.700449, + 0.709759 + ] + }, + "oos_misroute_rate":{ + "mean":0.446, + "std":0.007211, + "values":[ + 0.444, + 0.454, + 0.44 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.257667, + "std":0.046715, + "values":[ + 0.238, + 0.311, + 0.224 + ] + }, + "llm_call_rate":{ + "mean":0.085152, + "std":0.015874, + "values":[ + 0.092727, + 0.066909, + 0.095818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.031369, + "std":0.005846, + "values":[ + 0.034138, + 0.024654, + 0.035316 + ] + }, + "error_recovery_rate":{ + "mean":0.255903, + "std":0.039644, + "values":[ + 0.291139, + 0.212978, + 0.263591 + ] + }, + "recoverable_caught":{ + "mean":0.61912, + "std":0.096762, + "values":[ + 0.69434, + 0.50996, + 0.653061 + ] + }, + "small_errors":{ + "mean":613.333333, + "std":16.441817, + "values":[ + 632, + 601, + 607 + ] + }, + "recoverable_errors":{ + "mean":253.666667, + "std":10.263203, + "values":[ + 265, + 251, + 245 + ] + }, + "recovered_errors":{ + "mean":157.333333, + "std":28.095077, + "values":[ + 184, + 128, + 160 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":-0.537914, + "std":1.219665, + "values":[ + -1.944545, + 0.225611, + 0.105192 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "validation_selective_risk":{ + "mean":0.029032, + "std":0.001613, + "values":[ + 0.029032, + 0.027419, + 0.030645 + ] + }, + "validation_risk_upper95":{ + "mean":0.034417, + "std":0.001745, + "values":[ + 0.034418, + 0.032671, + 0.03616 + ] + }, + "test":{ + "coverage":{ + "mean":0.997697, + "std":0.002044, + "values":[ + 0.995455, + 0.999455, + 0.998182 + ] + }, + "selective_risk":{ + "mean":0.110196, + "std":0.002169, + "values":[ + 0.112694, + 0.108787, + 0.109107 + ] + }, + "accuracy_8":{ + "mean":0.889273, + "std":0.002519, + "values":[ + 0.886364, + 0.890727, + 0.890727 + ] + }, + "oos_recall":{ + "mean":0.452667, + "std":0.012741, + "values":[ + 0.438, + 0.461, + 0.459 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.422062, + "std":0.012612, + "values":[ + 0.407544, + 0.430312, + 0.42833 + ] + }, + { + "mean":0.483634, + "std":0.012773, + "values":[ + 0.46893, + 0.491986, + 0.489984 + ] + } + ], + "oos_precision":{ + "mean":0.982605, + "std":0.005803, + "values":[ + 0.979866, + 0.98927, + 0.978678 + ] + }, + "oos_f1":{ + "mean":0.619743, + "std":0.01259, + "values":[ + 0.60539, + 0.628922, + 0.624915 + ] + }, + "oos_misroute_rate":{ + "mean":0.547333, + "std":0.012741, + "values":[ + 0.562, + 0.539, + 0.541 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.543333, + "std":0.009292, + "values":[ + 0.554, + 0.537, + 0.539 + ] + }, + "llm_call_rate":{ + "mean":0.002303, + "std":0.002044, + "values":[ + 0.004545, + 0.000545, + 0.001818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.000849, + "std":0.000752, + "values":[ + 0.001673, + 0.000201, + 0.000671 + ] + }, + "error_recovery_rate":{ + "mean":0.008569, + "std":0.007993, + "values":[ + 0.015823, + 0.0, + 0.009885 + ] + }, + "recoverable_caught":{ + "mean":0.020742, + "std":0.019145, + "values":[ + 0.037736, + 0.0, + 0.02449 + ] + }, + "small_errors":{ + "mean":613.333333, + "std":16.441817, + "values":[ + 632, + 601, + 607 + ] + }, + "recoverable_errors":{ + "mean":253.666667, + "std":10.263203, + "values":[ + 265, + 251, + 245 + ] + }, + "recovered_errors":{ + "mean":5.333333, + "std":5.033223, + "values":[ + 10, + 0, + 6 + ] + } + } + } + } + } + } + } + }, + "majority/k1":{ + "model":"majority", + "k":1, + "oos_train":null, + "kind":"majority", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "majority-k1-seed42", + "majority-k1-seed43", + "majority-k1-seed44" + ], + "logits_sha256":[ + "ad9e1099ed7d88354021da1cda7ae310d1313af9d7d4654eafd03fabb8318e34", + "1b6de003a49322dea12ae7a224e9e6934b084fbbf6014bf7cb6721950b9d80e6", + "52a40757315fd9be80c2fd0cf5329f0b4c446ff25cf9a23ff714beab956155c9" + ], + "result":{ + "calibration":{ + "temperature":null, + "note":"not fitted: every row has the same scores", + "test_ece":null + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "summed":{ + "mean":0.245161, + "std":0.0, + "values":[ + 0.245161, + 0.245161, + 0.245161 + ] + } + }, + "signals_reported":[], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.032258, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.818182, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.181818, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.181818, + "std":0.0 + }, + "oos_f1":{ + "mean":0.307692, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.181818, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.899273, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.839631, + "std":0.0 + }, + "oos_f1":{ + "mean":0.912825, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.818182, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.302608, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.876889, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3946.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.245161, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0 + }, + "oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0 + }, + { + "mean":0.003827, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.0, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0 + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "oos_recall":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + { + "mean":0.003827, + "std":0.0, + "values":[ + 0.003827, + 0.003827, + 0.003827 + ] + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_f1":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0, + "values":[ + 0.825455, + 0.825455, + 0.825455 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0, + "values":[ + 0.754316, + 0.754316, + 0.754316 + ] + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0, + "values":[ + 0.648032, + 0.648032, + 0.648032 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0, + "values":[ + 0.292636, + 0.292636, + 0.292636 + ] + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0, + "values":[ + 0.779817, + 0.779817, + 0.779817 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null + }, + "0.05":{ + "selected_signal":null, + "hybrid":null + } + } + } + } + }, + "majority/k10":{ + "model":"majority", + "k":10, + "oos_train":null, + "kind":"majority", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "majority-k10-seed42", + "majority-k10-seed43", + "majority-k10-seed44" + ], + "logits_sha256":[ + "73c39d0e3e96319a1a3e339c4f7beeb57fa9c05ffbdda933ad50d8da0b47ce42", + "10fd16db95cd9c12358fdb5a5d5d17c0ca0f6248c6f2c7506e03c8d30da5d650", + "68305d812ce5fa3dc38d311014e4231cb19cb2a5fc25548011c3d448accfd491" + ], + "result":{ + "calibration":{ + "temperature":null, + "note":"not fitted: every row has the same scores", + "test_ece":null + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "summed":{ + "mean":0.245161, + "std":0.0, + "values":[ + 0.245161, + 0.245161, + 0.245161 + ] + } + }, + "signals_reported":[], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.032258, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.818182, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.181818, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.181818, + "std":0.0 + }, + "oos_f1":{ + "mean":0.307692, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.181818, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.899273, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.839631, + "std":0.0 + }, + "oos_f1":{ + "mean":0.912825, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.818182, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.302608, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.876889, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3946.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.245161, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0 + }, + "oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0 + }, + { + "mean":0.003827, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.0, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0 + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "oos_recall":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + { + "mean":0.003827, + "std":0.0, + "values":[ + 0.003827, + 0.003827, + 0.003827 + ] + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_f1":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0, + "values":[ + 0.825455, + 0.825455, + 0.825455 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0, + "values":[ + 0.754316, + 0.754316, + 0.754316 + ] + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0, + "values":[ + 0.648032, + 0.648032, + 0.648032 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0, + "values":[ + 0.292636, + 0.292636, + 0.292636 + ] + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0, + "values":[ + 0.779817, + 0.779817, + 0.779817 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null + }, + "0.05":{ + "selected_signal":null, + "hybrid":null + } + } + } + } + }, + "majority/k100":{ + "model":"majority", + "k":100, + "oos_train":null, + "kind":"majority", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "majority-k100-seed42", + "majority-k100-seed43", + "majority-k100-seed44" + ], + "logits_sha256":[ + "45de4fe9a649263e6470be6fdbe46e8a4fceb5a97a0afe36c21062b7d6447fb4", + "d57231c421661de1827d36c2b11d1a9cba0d5107f57447d639b25a70bcc46bb6", + "7422da060f87bd4325a17ef3f38948acf314349861f27c352038cbad2834c72b" + ], + "result":{ + "calibration":{ + "temperature":null, + "note":"not fitted: every row has the same scores", + "test_ece":null + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "summed":{ + "mean":0.245161, + "std":0.0, + "values":[ + 0.245161, + 0.245161, + 0.245161 + ] + } + }, + "signals_reported":[], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.032258, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.818182, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.181818, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.181818, + "std":0.0 + }, + "oos_f1":{ + "mean":0.307692, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.181818, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.899273, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.839631, + "std":0.0 + }, + "oos_f1":{ + "mean":0.912825, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.818182, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.302608, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.876889, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3946.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.245161, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0 + }, + "oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0 + }, + { + "mean":0.003827, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.0, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0 + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "oos_recall":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + { + "mean":0.003827, + "std":0.0, + "values":[ + 0.003827, + 0.003827, + 0.003827 + ] + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_f1":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0, + "values":[ + 0.825455, + 0.825455, + 0.825455 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0, + "values":[ + 0.754316, + 0.754316, + 0.754316 + ] + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0, + "values":[ + 0.648032, + 0.648032, + 0.648032 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0, + "values":[ + 0.292636, + 0.292636, + 0.292636 + ] + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0, + "values":[ + 0.779817, + 0.779817, + 0.779817 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null + }, + "0.05":{ + "selected_signal":null, + "hybrid":null + } + } + } + } + }, + "majority/k25":{ + "model":"majority", + "k":25, + "oos_train":null, + "kind":"majority", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "majority-k25-seed42", + "majority-k25-seed43", + "majority-k25-seed44" + ], + "logits_sha256":[ + "0847cf359ec51cb09bfec7ae8dfea24c09254963a5ea73a82875b4d1780c593e", + "1d6145a48b53b954eba6a1d453b77f5eafc5f9cada582ff9012f7d39f1201ad5", + "be5db7a3d7fda2f50097788b959e62a40acd27100a64ad4350f1527883183e1f" + ], + "result":{ + "calibration":{ + "temperature":null, + "note":"not fitted: every row has the same scores", + "test_ece":null + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "summed":{ + "mean":0.245161, + "std":0.0, + "values":[ + 0.245161, + 0.245161, + 0.245161 + ] + } + }, + "signals_reported":[], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.032258, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.818182, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.181818, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.181818, + "std":0.0 + }, + "oos_f1":{ + "mean":0.307692, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.181818, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.899273, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.839631, + "std":0.0 + }, + "oos_f1":{ + "mean":0.912825, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.818182, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.302608, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.876889, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3946.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.245161, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0 + }, + "oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0 + }, + { + "mean":0.003827, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.0, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0 + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "oos_recall":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + { + "mean":0.003827, + "std":0.0, + "values":[ + 0.003827, + 0.003827, + 0.003827 + ] + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_f1":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0, + "values":[ + 0.825455, + 0.825455, + 0.825455 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0, + "values":[ + 0.754316, + 0.754316, + 0.754316 + ] + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0, + "values":[ + 0.648032, + 0.648032, + 0.648032 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0, + "values":[ + 0.292636, + 0.292636, + 0.292636 + ] + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0, + "values":[ + 0.779817, + 0.779817, + 0.779817 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null + }, + "0.05":{ + "selected_signal":null, + "hybrid":null + } + } + } + } + }, + "majority/k5":{ + "model":"majority", + "k":5, + "oos_train":null, + "kind":"majority", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "majority-k5-seed42", + "majority-k5-seed43", + "majority-k5-seed44" + ], + "logits_sha256":[ + "9e6bb6f163ed274f4c53aedd58604b2c1259d3c083384ea7fb2d49455f9db466", + "3314ad9a70b585c6f4fccb9eb0bd56a5df209206e20b552c464727e10bbaba98", + "1da11415daed0f054fcb2477402f323bdcc4c0908abfebacfc116bc6f1ec5837" + ], + "result":{ + "calibration":{ + "temperature":null, + "note":"not fitted: every row has the same scores", + "test_ece":null + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "summed":{ + "mean":0.245161, + "std":0.0, + "values":[ + 0.245161, + 0.245161, + 0.245161 + ] + } + }, + "signals_reported":[], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.032258, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.818182, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.181818, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.181818, + "std":0.0 + }, + "oos_f1":{ + "mean":0.307692, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.181818, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.899273, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.839631, + "std":0.0 + }, + "oos_f1":{ + "mean":0.912825, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.818182, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.302608, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.876889, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3946.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.245161, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0 + }, + "oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0 + }, + { + "mean":0.003827, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.0, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0 + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "oos_recall":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + { + "mean":0.003827, + "std":0.0, + "values":[ + 0.003827, + 0.003827, + 0.003827 + ] + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_f1":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0, + "values":[ + 0.825455, + 0.825455, + 0.825455 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0, + "values":[ + 0.754316, + 0.754316, + 0.754316 + ] + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0, + "values":[ + 0.648032, + 0.648032, + 0.648032 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0, + "values":[ + 0.292636, + 0.292636, + 0.292636 + ] + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0, + "values":[ + 0.779817, + 0.779817, + 0.779817 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null + }, + "0.05":{ + "selected_signal":null, + "hybrid":null + } + } + } + } + }, + "majority/k50":{ + "model":"majority", + "k":50, + "oos_train":null, + "kind":"majority", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "majority-k50-seed42", + "majority-k50-seed43", + "majority-k50-seed44" + ], + "logits_sha256":[ + "7c4872ed6ac1c993a1850633e26c44fac500db0efdc4923ea6a1ad1486a5ef97", + "ddd5754bd4acf617e268c6c68c49b3ab257bef78fb1e41e56cdb2d5508c3638e", + "15276ebf3a368cb46ccb4beb03a529067369266cbfcfe1d5e6294e022d374e7c" + ], + "result":{ + "calibration":{ + "temperature":null, + "note":"not fitted: every row has the same scores", + "test_ece":null + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "summed":{ + "mean":0.245161, + "std":0.0, + "values":[ + 0.245161, + 0.245161, + 0.245161 + ] + } + }, + "signals_reported":[], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.032258, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.818182, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.181818, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.181818, + "std":0.0 + }, + "oos_f1":{ + "mean":0.307692, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.181818, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.899273, + "std":0.0 + }, + "oos_recall":{ + "mean":1.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.996173, + "std":0.0 + }, + { + "mean":1.0, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.839631, + "std":0.0 + }, + "oos_f1":{ + "mean":0.912825, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.818182, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.302608, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.876889, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4500.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3946.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3946.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.245161, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0 + }, + "oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0 + }, + { + "mean":0.003827, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.0, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0 + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":4360.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0 + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + }, + "0.05":{ + "selected_signal":null, + "hybrid":null, + "by_signal":{} + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "accuracy_8":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "oos_recall":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + { + "mean":0.003827, + "std":0.0, + "values":[ + 0.003827, + 0.003827, + 0.003827 + ] + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_f1":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.207273, + "std":0.0, + "values":[ + 0.207273, + 0.207273, + 0.207273 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.825455, + "std":0.0, + "values":[ + 0.825455, + 0.825455, + 0.825455 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.754316, + "std":0.0, + "values":[ + 0.754316, + 0.754316, + 0.754316 + ] + }, + "oos_f1":{ + "mean":0.648032, + "std":0.0, + "values":[ + 0.648032, + 0.648032, + 0.648032 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.792727, + "std":0.0, + "values":[ + 0.792727, + 0.792727, + 0.792727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.292636, + "std":0.0, + "values":[ + 0.292636, + 0.292636, + 0.292636 + ] + }, + "error_recovery_rate":{ + "mean":0.779817, + "std":0.0, + "values":[ + 0.779817, + 0.779817, + 0.779817 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":4360.0, + "std":0.0, + "values":[ + 4360, + 4360, + 4360 + ] + }, + "recoverable_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + }, + "recovered_errors":{ + "mean":3400.0, + "std":0.0, + "values":[ + 3400, + 3400, + 3400 + ] + } + }, + "signals":{}, + "fallback":{ + "0.02":{ + "selected_signal":null, + "hybrid":null + }, + "0.05":{ + "selected_signal":null, + "hybrid":null + } + } + } + } + }, + "modernbert-oos0/k100":{ + "model":"modernbert-oos0", + "k":100, + "oos_train":0, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "ModernBERT-base-k100-oos0-seed42", + "ModernBERT-base-k100-oos0-seed43", + "ModernBERT-base-k100-oos0-seed44" + ], + "logits_sha256":[ + "c83e3c569facc09d5eb14cc3f837883fb59b0d5d68583257e9366e3750066be6", + "09029343a7529ac08bbc145fef45edcc1bfb04f6b976dc956473c6c546175343", + "6edfa2106d6d72ae26449b5598cff759ae601c44c42648de129ceab6e5530a61" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":1.505629, + "std":0.019611, + "values":[ + 1.519161, + 1.514588, + 1.483139 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.101611, + "std":0.001637, + "values":[ + 0.103502, + 0.100653, + 0.100679 + ] + }, + "ece_8_summed":{ + "mean":0.116852, + "std":0.001351, + "values":[ + 0.118374, + 0.116385, + 0.115796 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.041727, + "std":0.001651, + "values":[ + 0.040634, + 0.04092, + 0.043626 + ] + }, + "ece_8_summed":{ + "mean":0.071162, + "std":0.000405, + "values":[ + 0.070696, + 0.071351, + 0.071438 + ] + } + } + } + }, + "selected_aggregation":[ + "summed", + "summed", + "argmax" + ], + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.960108, + "std":0.000812, + "values":[ + 0.959355, + 0.960968, + 0.96 + ] + }, + "summed":{ + "mean":0.960323, + "std":0.001163, + "values":[ + 0.96, + 0.961613, + 0.959355 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.960108, + "std":0.000812 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.190909, + "std":0.000364 + }, + "accuracy_8":{ + "mean":0.809091, + "std":0.000364 + }, + "oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0 + }, + { + "mean":0.003827, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.0, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.809091, + "std":0.000364 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.917333, + "std":0.00021 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.986683, + "std":0.00099 + }, + "oos_f1":{ + "mean":0.720965, + "std":0.000264 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.190909, + "std":0.000364 + }, + "llm_cost_usd_per_1k":{ + "mean":0.070239, + "std":0.000134 + }, + "error_recovery_rate":{ + "mean":0.566984, + "std":0.000564 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":595.333333, + "std":1.154701 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.002339, + "std":0.000452 + }, + "oos_auroc":{ + "mean":0.973973, + "std":0.005383 + }, + "oos_auprc":{ + "mean":0.680482, + "std":0.059569 + } + }, + "test":{ + "aurc":{ + "mean":0.024676, + "std":0.000301 + }, + "oos_auroc":{ + "mean":0.973488, + "std":0.000788 + }, + "oos_auprc":{ + "mean":0.895078, + "std":0.00372 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.00222, + "std":0.000418 + }, + "oos_auroc":{ + "mean":0.977523, + "std":0.00442 + }, + "oos_auprc":{ + "mean":0.749335, + "std":0.057612 + } + }, + "test":{ + "aurc":{ + "mean":0.02406, + "std":0.000269 + }, + "oos_auroc":{ + "mean":0.977331, + "std":0.000825 + }, + "oos_auprc":{ + "mean":0.91661, + "std":0.00441 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.002688, + "std":0.000548 + }, + "oos_auroc":{ + "mean":0.965118, + "std":0.007603 + }, + "oos_auprc":{ + "mean":0.469064, + "std":0.062607 + } + }, + "test":{ + "aurc":{ + "mean":0.026647, + "std":0.000587 + }, + "oos_auroc":{ + "mean":0.962406, + "std":0.001189 + }, + "oos_auprc":{ + "mean":0.820282, + "std":0.007475 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.002194, + "std":0.000372 + }, + "oos_auroc":{ + "mean":0.977906, + "std":0.003772 + }, + "oos_auprc":{ + "mean":0.737454, + "std":0.051905 + } + }, + "test":{ + "aurc":{ + "mean":0.023888, + "std":0.000197 + }, + "oos_auroc":{ + "mean":0.977988, + "std":0.00044 + }, + "oos_auprc":{ + "mean":0.915221, + "std":0.002638 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "entropy", + "msp_t", + "entropy" + ], + "hybrid":{ + "tau":{ + "mean":-1.067557, + "std":1.191662 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.970215, + "std":0.000986 + }, + "validation_selective_risk":{ + "mean":0.015627, + "std":1.6e-05 + }, + "validation_risk_upper95":{ + "mean":0.019806, + "std":2e-05 + }, + "test":{ + "coverage":{ + "mean":0.875636, + "std":0.007401 + }, + "selective_risk":{ + "mean":0.081703, + "std":0.006953 + }, + "accuracy_8":{ + "mean":0.882606, + "std":0.003745 + }, + "oos_recall":{ + "mean":0.403333, + "std":0.020306 + }, + "oos_recall_wilson95":[ + { + "mean":0.373372, + "std":0.019967 + }, + { + "mean":0.434035, + "std":0.02049 + } + ], + "oos_precision":{ + "mean":0.979048, + "std":0.002781 + }, + "oos_f1":{ + "mean":0.571087, + "std":0.020161 + }, + "oos_misroute_rate":{ + "mean":0.596667, + "std":0.020306 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.360667, + "std":0.035218 + }, + "llm_call_rate":{ + "mean":0.124364, + "std":0.007401 + }, + "llm_cost_usd_per_1k":{ + "mean":0.045735, + "std":0.002732 + }, + "error_recovery_rate":{ + "mean":0.392088, + "std":0.021098 + }, + "recoverable_caught":{ + "mean":0.691518, + "std":0.03682 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":411.666667, + "std":21.45538 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.568431, + "std":0.034798 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.965054, + "std":0.002909 + }, + "validation_selective_risk":{ + "mean":0.01571, + "std":4.7e-05 + }, + "validation_risk_upper95":{ + "mean":0.019911, + "std":6e-05 + }, + "test":{ + "coverage":{ + "mean":0.874, + "std":0.01044 + }, + "selective_risk":{ + "mean":0.084185, + "std":0.008502 + }, + "accuracy_8":{ + "mean":0.880121, + "std":0.004572 + }, + "oos_recall":{ + "mean":0.391333, + "std":0.024786 + }, + "oos_recall_wilson95":[ + { + "mean":0.361582, + "std":0.024329 + }, + { + "mean":0.421916, + "std":0.025053 + } + ], + "oos_precision":{ + "mean":0.976783, + "std":0.00258 + }, + "oos_f1":{ + "mean":0.558468, + "std":0.025246 + }, + "oos_misroute_rate":{ + "mean":0.608667, + "std":0.024786 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.374, + "std":0.042036 + }, + "llm_call_rate":{ + "mean":0.126, + "std":0.01044 + }, + "llm_cost_usd_per_1k":{ + "mean":0.046347, + "std":0.003856 + }, + "error_recovery_rate":{ + "mean":0.381929, + "std":0.025903 + }, + "recoverable_caught":{ + "mean":0.673591, + "std":0.045148 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":401.0, + "std":26.664583 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.721203, + "std":0.078585 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.970215, + "std":0.000986 + }, + "validation_selective_risk":{ + "mean":0.015627, + "std":1.6e-05 + }, + "validation_risk_upper95":{ + "mean":0.019806, + "std":2e-05 + }, + "test":{ + "coverage":{ + "mean":0.876545, + "std":0.006771 + }, + "selective_risk":{ + "mean":0.082455, + "std":0.00624 + }, + "accuracy_8":{ + "mean":0.882364, + "std":0.003483 + }, + "oos_recall":{ + "mean":0.402667, + "std":0.019655 + }, + "oos_recall_wilson95":[ + { + "mean":0.372715, + "std":0.019325 + }, + { + "mean":0.433364, + "std":0.019835 + } + ], + "oos_precision":{ + "mean":0.979015, + "std":0.002784 + }, + "oos_f1":{ + "mean":0.570425, + "std":0.019505 + }, + "oos_misroute_rate":{ + "mean":0.597333, + "std":0.019655 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.364333, + "std":0.03177 + }, + "llm_call_rate":{ + "mean":0.123455, + "std":0.006771 + }, + "llm_cost_usd_per_1k":{ + "mean":0.045399, + "std":0.0025 + }, + "error_recovery_rate":{ + "mean":0.391134, + "std":0.020165 + }, + "recoverable_caught":{ + "mean":0.689835, + "std":0.035138 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":410.666667, + "std":20.502032 + } + } + }, + "margin":{ + "tau":{ + "mean":0.90933, + "std":0.127623 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.957419, + "std":0.004218 + }, + "validation_selective_risk":{ + "mean":0.015611, + "std":0.000126 + }, + "validation_risk_upper95":{ + "mean":0.019819, + "std":0.000131 + }, + "test":{ + "coverage":{ + "mean":0.873212, + "std":0.015016 + }, + "selective_risk":{ + "mean":0.092981, + "std":0.014071 + }, + "accuracy_8":{ + "mean":0.873091, + "std":0.00772 + }, + "oos_recall":{ + "mean":0.355667, + "std":0.040253 + }, + "oos_recall_wilson95":[ + { + "mean":0.326677, + "std":0.039293 + }, + { + "mean":0.385761, + "std":0.040906 + } + ], + "oos_precision":{ + "mean":0.970137, + "std":0.002372 + }, + "oos_f1":{ + "mean":0.519593, + "std":0.043547 + }, + "oos_misroute_rate":{ + "mean":0.644333, + "std":0.040253 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.421, + "std":0.071267 + }, + "llm_call_rate":{ + "mean":0.126788, + "std":0.015016 + }, + "llm_cost_usd_per_1k":{ + "mean":0.046659, + "std":0.005538 + }, + "error_recovery_rate":{ + "mean":0.350524, + "std":0.041122 + }, + "recoverable_caught":{ + "mean":0.618195, + "std":0.072185 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":368.0, + "std":42.579338 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.300668, + "std":0.015257 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.969462, + "std":0.000931 + }, + "validation_selective_risk":{ + "mean":0.015639, + "std":1.5e-05 + }, + "validation_risk_upper95":{ + "mean":0.019821, + "std":1.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.873636, + "std":0.006117 + }, + "selective_risk":{ + "mean":0.080307, + "std":0.005739 + }, + "accuracy_8":{ + "mean":0.883576, + "std":0.003098 + }, + "oos_recall":{ + "mean":0.407667, + "std":0.016166 + }, + "oos_recall_wilson95":[ + { + "mean":0.37763, + "std":0.015907 + }, + { + "mean":0.43841, + "std":0.016301 + } + ], + "oos_precision":{ + "mean":0.978434, + "std":0.001678 + }, + "oos_f1":{ + "mean":0.575401, + "std":0.016017 + }, + "oos_misroute_rate":{ + "mean":0.592333, + "std":0.016166 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.354333, + "std":0.028361 + }, + "llm_call_rate":{ + "mean":0.126364, + "std":0.006117 + }, + "llm_cost_usd_per_1k":{ + "mean":0.046472, + "std":0.00226 + }, + "error_recovery_rate":{ + "mean":0.397479, + "std":0.01744 + }, + "recoverable_caught":{ + "mean":0.701026, + "std":0.030269 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":417.333333, + "std":17.672955 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":-1.288443, + "std":2.321544 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039892, + "std":0.000812 + }, + "validation_risk_upper95":{ + "mean":0.046087, + "std":0.000867 + }, + "test":{ + "coverage":{ + "mean":0.999212, + "std":0.000735 + }, + "selective_risk":{ + "mean":0.190271, + "std":0.000869 + }, + "accuracy_8":{ + "mean":0.809758, + "std":0.000897 + }, + "oos_recall":{ + "mean":0.003667, + "std":0.003215 + }, + "oos_recall_wilson95":[ + { + "mean":0.00163, + "std":0.001445 + }, + { + "mean":0.009502, + "std":0.004963 + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.007293, + "std":0.006393 + }, + "oos_misroute_rate":{ + "mean":0.996333, + "std":0.003215 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.995667, + "std":0.004041 + }, + "llm_call_rate":{ + "mean":0.000788, + "std":0.000735 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000289, + "std":0.000269 + }, + "error_recovery_rate":{ + "mean":0.003495, + "std":0.003063 + }, + "recoverable_caught":{ + "mean":0.006162, + "std":0.005399 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":3.666667, + "std":3.21455 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.086392, + "std":0.017794 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039892, + "std":0.000812 + }, + "validation_risk_upper95":{ + "mean":0.046087, + "std":0.000867 + }, + "test":{ + "coverage":{ + "mean":0.99903, + "std":0.001001 + }, + "selective_risk":{ + "mean":0.190123, + "std":0.001029 + }, + "accuracy_8":{ + "mean":0.809879, + "std":0.001001 + }, + "oos_recall":{ + "mean":0.004333, + "std":0.004041 + }, + "oos_recall_wilson95":[ + { + "mean":0.002066, + "std":0.002031 + }, + { + "mean":0.010395, + "std":0.006039 + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.008608, + "std":0.008021 + }, + "oos_misroute_rate":{ + "mean":0.995667, + "std":0.004041 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.994667, + "std":0.005508 + }, + "llm_call_rate":{ + "mean":0.00097, + "std":0.001001 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000355, + "std":0.000367 + }, + "error_recovery_rate":{ + "mean":0.00413, + "std":0.00385 + }, + "recoverable_caught":{ + "mean":0.00728, + "std":0.006783 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":4.333333, + "std":4.041452 + } + } + }, + "entropy":{ + "tau":{ + "mean":-3.949794, + "std":0.042275 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039892, + "std":0.000812 + }, + "validation_risk_upper95":{ + "mean":0.046087, + "std":0.000867 + }, + "test":{ + "coverage":{ + "mean":0.999939, + "std":0.000105 + }, + "selective_risk":{ + "mean":0.19086, + "std":0.000439 + }, + "accuracy_8":{ + "mean":0.809152, + "std":0.000458 + }, + "oos_recall":{ + "mean":0.000333, + "std":0.000577 + }, + "oos_recall_wilson95":[ + { + "mean":5.9e-05, + "std":0.000102 + }, + { + "mean":0.004432, + "std":0.001048 + } + ], + "oos_precision":{ + "mean":0.333333, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.000666, + "std":0.001154 + }, + "oos_misroute_rate":{ + "mean":0.999667, + "std":0.000577 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.999667, + "std":0.000577 + }, + "llm_call_rate":{ + "mean":6.1e-05, + "std":0.000105 + }, + "llm_cost_usd_per_1k":{ + "mean":2.2e-05, + "std":3.8e-05 + }, + "error_recovery_rate":{ + "mean":0.000318, + "std":0.000551 + }, + "recoverable_caught":{ + "mean":0.000561, + "std":0.000972 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":0.333333, + "std":0.57735 + } + } + }, + "margin":{ + "tau":{ + "mean":0.009622, + "std":0.006818 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039892, + "std":0.000812 + }, + "validation_risk_upper95":{ + "mean":0.046087, + "std":0.000867 + }, + "test":{ + "coverage":{ + "mean":0.998061, + "std":0.000897 + }, + "selective_risk":{ + "mean":0.189397, + "std":0.000701 + }, + "accuracy_8":{ + "mean":0.810364, + "std":0.000656 + }, + "oos_recall":{ + "mean":0.006667, + "std":0.004163 + }, + "oos_recall_wilson95":[ + { + "mean":0.003349, + "std":0.002522 + }, + { + "mean":0.01376, + "std":0.005775 + } + ], + "oos_precision":{ + "mean":1.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.013222, + "std":0.008232 + }, + "oos_misroute_rate":{ + "mean":0.993333, + "std":0.004163 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.99, + "std":0.005292 + }, + "llm_call_rate":{ + "mean":0.001939, + "std":0.000897 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000714, + "std":0.000329 + }, + "error_recovery_rate":{ + "mean":0.006665, + "std":0.003428 + }, + "recoverable_caught":{ + "mean":0.01176, + "std":0.006056 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":7.0, + "std":3.605551 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.048299, + "std":0.008281 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039892, + "std":0.000812 + }, + "validation_risk_upper95":{ + "mean":0.046087, + "std":0.000867 + }, + "test":{ + "coverage":{ + "mean":0.999212, + "std":0.000735 + }, + "selective_risk":{ + "mean":0.190271, + "std":0.000869 + }, + "accuracy_8":{ + "mean":0.809758, + "std":0.000897 + }, + "oos_recall":{ + "mean":0.003667, + "std":0.003215 + }, + "oos_recall_wilson95":[ + { + "mean":0.00163, + "std":0.001445 + }, + { + "mean":0.009502, + "std":0.004963 + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.007293, + "std":0.006393 + }, + "oos_misroute_rate":{ + "mean":0.996333, + "std":0.003215 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.995667, + "std":0.004041 + }, + "llm_call_rate":{ + "mean":0.000788, + "std":0.000735 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000289, + "std":0.000269 + }, + "error_recovery_rate":{ + "mean":0.003495, + "std":0.003063 + }, + "recoverable_caught":{ + "mean":0.006162, + "std":0.005399 + }, + "small_errors":{ + "mean":1050.0, + "std":2.0 + }, + "recoverable_errors":{ + "mean":595.333333, + "std":1.154701 + }, + "recovered_errors":{ + "mean":3.666667, + "std":3.21455 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.960323, + "std":0.001163 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.190182, + "std":0.000727 + }, + "accuracy_8":{ + "mean":0.809818, + "std":0.000727 + }, + "oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0 + }, + { + "mean":0.003827, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.0, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.809818, + "std":0.000727 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.917333, + "std":0.000278 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.986683, + "std":0.00099 + }, + "oos_f1":{ + "mean":0.720965, + "std":0.000264 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.190182, + "std":0.000727 + }, + "llm_cost_usd_per_1k":{ + "mean":0.069968, + "std":0.000271 + }, + "error_recovery_rate":{ + "mean":0.565328, + "std":0.000357 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":591.333333, + "std":2.516611 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.002094, + "std":0.000335 + }, + "oos_auroc":{ + "mean":0.978887, + "std":0.003538 + }, + "oos_auprc":{ + "mean":0.743408, + "std":0.040177 + } + }, + "test":{ + "aurc":{ + "mean":0.02349, + "std":0.000187 + }, + "oos_auroc":{ + "mean":0.978537, + "std":0.000532 + }, + "oos_auprc":{ + "mean":0.914566, + "std":0.002307 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.002066, + "std":0.000342 + }, + "oos_auroc":{ + "mean":0.980076, + "std":0.003412 + }, + "oos_auprc":{ + "mean":0.769172, + "std":0.045548 + } + }, + "test":{ + "aurc":{ + "mean":0.023318, + "std":0.000207 + }, + "oos_auroc":{ + "mean":0.980064, + "std":0.000706 + }, + "oos_auprc":{ + "mean":0.926301, + "std":0.003795 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.002148, + "std":0.000339 + }, + "oos_auroc":{ + "mean":0.977146, + "std":0.003119 + }, + "oos_auprc":{ + "mean":0.680013, + "std":0.047955 + } + }, + "test":{ + "aurc":{ + "mean":0.02383, + "std":0.000227 + }, + "oos_auroc":{ + "mean":0.975989, + "std":0.000725 + }, + "oos_auprc":{ + "mean":0.889547, + "std":0.005242 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.002062, + "std":0.00032 + }, + "oos_auroc":{ + "mean":0.980386, + "std":0.002221 + }, + "oos_auprc":{ + "mean":0.769788, + "std":0.026091 + } + }, + "test":{ + "aurc":{ + "mean":0.023141, + "std":0.000184 + }, + "oos_auroc":{ + "mean":0.980798, + "std":0.000585 + }, + "oos_auprc":{ + "mean":0.925321, + "std":0.003309 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"entropy", + "hybrid":{ + "tau":{ + "mean":-0.920308, + "std":0.03909 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.972151, + "std":0.001304 + }, + "validation_selective_risk":{ + "mean":0.015596, + "std":2.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.019766, + "std":2.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.880545, + "std":0.005203 + }, + "selective_risk":{ + "mean":0.084843, + "std":0.005459 + }, + "accuracy_8":{ + "mean":0.884121, + "std":0.002917 + }, + "oos_recall":{ + "mean":0.409667, + "std":0.015044 + }, + "oos_recall_wilson95":[ + { + "mean":0.379598, + "std":0.014815 + }, + { + "mean":0.440427, + "std":0.015159 + } + ], + "oos_precision":{ + "mean":0.980136, + "std":0.002871 + }, + "oos_f1":{ + "mean":0.577696, + "std":0.014514 + }, + "oos_misroute_rate":{ + "mean":0.590333, + "std":0.015044 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.379, + "std":0.025239 + }, + "llm_call_rate":{ + "mean":0.119455, + "std":0.005203 + }, + "llm_cost_usd_per_1k":{ + "mean":0.043905, + "std":0.001926 + }, + "error_recovery_rate":{ + "mean":0.397426, + "std":0.015495 + }, + "recoverable_caught":{ + "mean":0.70301, + "std":0.027723 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":415.666667, + "std":14.640128 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.718768, + "std":0.037369 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.970753, + "std":0.001133 + }, + "validation_selective_risk":{ + "mean":0.015618, + "std":1.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.019795, + "std":2.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.879818, + "std":0.009448 + }, + "selective_risk":{ + "mean":0.086181, + "std":0.008806 + }, + "accuracy_8":{ + "mean":0.882909, + "std":0.005068 + }, + "oos_recall":{ + "mean":0.402667, + "std":0.025813 + }, + "oos_recall_wilson95":[ + { + "mean":0.372727, + "std":0.025374 + }, + { + "mean":0.433351, + "std":0.026055 + } + ], + "oos_precision":{ + "mean":0.979885, + "std":0.005145 + }, + "oos_f1":{ + "mean":0.570418, + "std":0.025688 + }, + "oos_misroute_rate":{ + "mean":0.597333, + "std":0.025813 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.391, + "std":0.040853 + }, + "llm_call_rate":{ + "mean":0.120182, + "std":0.009448 + }, + "llm_cost_usd_per_1k":{ + "mean":0.04418, + "std":0.00349 + }, + "error_recovery_rate":{ + "mean":0.393623, + "std":0.027476 + }, + "recoverable_caught":{ + "mean":0.696293, + "std":0.049002 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":411.666667, + "std":27.465129 + } + } + }, + "entropy":{ + "tau":{ + "mean":-0.920308, + "std":0.03909 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.972151, + "std":0.001304 + }, + "validation_selective_risk":{ + "mean":0.015596, + "std":2.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.019766, + "std":2.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.880545, + "std":0.005203 + }, + "selective_risk":{ + "mean":0.084843, + "std":0.005459 + }, + "accuracy_8":{ + "mean":0.884121, + "std":0.002917 + }, + "oos_recall":{ + "mean":0.409667, + "std":0.015044 + }, + "oos_recall_wilson95":[ + { + "mean":0.379598, + "std":0.014815 + }, + { + "mean":0.440427, + "std":0.015159 + } + ], + "oos_precision":{ + "mean":0.980136, + "std":0.002871 + }, + "oos_f1":{ + "mean":0.577696, + "std":0.014514 + }, + "oos_misroute_rate":{ + "mean":0.590333, + "std":0.015044 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.379, + "std":0.025239 + }, + "llm_call_rate":{ + "mean":0.119455, + "std":0.005203 + }, + "llm_cost_usd_per_1k":{ + "mean":0.043905, + "std":0.001926 + }, + "error_recovery_rate":{ + "mean":0.397426, + "std":0.015495 + }, + "recoverable_caught":{ + "mean":0.70301, + "std":0.027723 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":415.666667, + "std":14.640128 + } + } + }, + "margin":{ + "tau":{ + "mean":0.993144, + "std":0.078989 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.969677, + "std":0.000853 + }, + "validation_selective_risk":{ + "mean":0.015635, + "std":1.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.019817, + "std":1.7e-05 + }, + "test":{ + "coverage":{ + "mean":0.875879, + "std":0.005756 + }, + "selective_risk":{ + "mean":0.083492, + "std":0.005878 + }, + "accuracy_8":{ + "mean":0.884, + "std":0.002446 + }, + "oos_recall":{ + "mean":0.408333, + "std":0.012702 + }, + "oos_recall_wilson95":[ + { + "mean":0.378282, + "std":0.012513 + }, + { + "mean":0.439086, + "std":0.012793 + } + ], + "oos_precision":{ + "mean":0.981723, + "std":0.006122 + }, + "oos_f1":{ + "mean":0.576657, + "std":0.011519 + }, + "oos_misroute_rate":{ + "mean":0.591667, + "std":0.012702 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.380667, + "std":0.027025 + }, + "llm_call_rate":{ + "mean":0.124121, + "std":0.005756 + }, + "llm_cost_usd_per_1k":{ + "mean":0.04564, + "std":0.002126 + }, + "error_recovery_rate":{ + "mean":0.401878, + "std":0.012729 + }, + "recoverable_caught":{ + "mean":0.710879, + "std":0.022636 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":420.333333, + "std":11.846237 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.498101, + "std":0.020679 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.971398, + "std":0.000812 + }, + "validation_selective_risk":{ + "mean":0.015608, + "std":1.3e-05 + }, + "validation_risk_upper95":{ + "mean":0.019782, + "std":1.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.874061, + "std":0.006151 + }, + "selective_risk":{ + "mean":0.078741, + "std":0.005834 + }, + "accuracy_8":{ + "mean":0.887515, + "std":0.002244 + }, + "oos_recall":{ + "mean":0.427333, + "std":0.011846 + }, + "oos_recall_wilson95":[ + { + "mean":0.397014, + "std":0.011699 + }, + { + "mean":0.458209, + "std":0.011903 + } + ], + "oos_precision":{ + "mean":0.981022, + "std":0.00661 + }, + "oos_f1":{ + "mean":0.595239, + "std":0.010227 + }, + "oos_misroute_rate":{ + "mean":0.572667, + "std":0.011846 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.351333, + "std":0.024987 + }, + "llm_call_rate":{ + "mean":0.125939, + "std":0.006151 + }, + "llm_cost_usd_per_1k":{ + "mean":0.046298, + "std":0.002271 + }, + "error_recovery_rate":{ + "mean":0.416541, + "std":0.014206 + }, + "recoverable_caught":{ + "mean":0.736816, + "std":0.025264 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":435.666667, + "std":13.316656 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":-0.498523, + "std":1.180254 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039677, + "std":0.001163 + }, + "validation_risk_upper95":{ + "mean":0.045857, + "std":0.001243 + }, + "test":{ + "coverage":{ + "mean":0.999273, + "std":0.000727 + }, + "selective_risk":{ + "mean":0.189592, + "std":0.00067 + }, + "accuracy_8":{ + "mean":0.810424, + "std":0.000735 + }, + "oos_recall":{ + "mean":0.003333, + "std":0.003512 + }, + "oos_recall_wilson95":[ + { + "mean":0.001472, + "std":0.001742 + }, + { + "mean":0.008996, + "std":0.005279 + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.006628, + "std":0.006974 + }, + "oos_misroute_rate":{ + "mean":0.996667, + "std":0.003512 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.996, + "std":0.004 + }, + "llm_call_rate":{ + "mean":0.000727, + "std":0.000727 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000267, + "std":0.000267 + }, + "error_recovery_rate":{ + "mean":0.003183, + "std":0.003358 + }, + "recoverable_caught":{ + "mean":0.005632, + "std":0.005944 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":3.333333, + "std":3.511885 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.226828, + "std":0.013507 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039677, + "std":0.001163 + }, + "validation_risk_upper95":{ + "mean":0.045857, + "std":0.001243 + }, + "test":{ + "coverage":{ + "mean":0.998848, + "std":0.000735 + }, + "selective_risk":{ + "mean":0.189248, + "std":0.001192 + }, + "accuracy_8":{ + "mean":0.810848, + "std":0.001365 + }, + "oos_recall":{ + "mean":0.005667, + "std":0.004509 + }, + "oos_recall_wilson95":[ + { + "mean":0.00279, + "std":0.002632 + }, + { + "mean":0.012327, + "std":0.006363 + } + ], + "oos_precision":{ + "mean":1.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.011243, + "std":0.008922 + }, + "oos_misroute_rate":{ + "mean":0.994333, + "std":0.004509 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.993667, + "std":0.004041 + }, + "llm_call_rate":{ + "mean":0.001152, + "std":0.000735 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000421, + "std":0.000269 + }, + "error_recovery_rate":{ + "mean":0.005424, + "std":0.004314 + }, + "recoverable_caught":{ + "mean":0.009597, + "std":0.007636 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":5.666667, + "std":4.50925 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.846056, + "std":0.022957 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039677, + "std":0.001163 + }, + "validation_risk_upper95":{ + "mean":0.045857, + "std":0.001243 + }, + "test":{ + "coverage":{ + "mean":0.997939, + "std":0.001857 + }, + "selective_risk":{ + "mean":0.188508, + "std":0.001771 + }, + "accuracy_8":{ + "mean":0.811455, + "std":0.001734 + }, + "oos_recall":{ + "mean":0.009, + "std":0.007937 + }, + "oos_recall_wilson95":[ + { + "mean":0.005063, + "std":0.005569 + }, + { + "mean":0.016695, + "std":0.010249 + } + ], + "oos_precision":{ + "mean":1.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.017758, + "std":0.015534 + }, + "oos_misroute_rate":{ + "mean":0.991, + "std":0.007937 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.988667, + "std":0.010214 + }, + "llm_call_rate":{ + "mean":0.002061, + "std":0.001857 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000755, + "std":0.000681 + }, + "error_recovery_rate":{ + "mean":0.008608, + "std":0.007588 + }, + "recoverable_caught":{ + "mean":0.015231, + "std":0.013433 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":9.0, + "std":7.937254 + } + } + }, + "margin":{ + "tau":{ + "mean":0.006722, + "std":0.005623 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039677, + "std":0.001163 + }, + "validation_risk_upper95":{ + "mean":0.045857, + "std":0.001243 + }, + "test":{ + "coverage":{ + "mean":0.999152, + "std":0.001169 + }, + "selective_risk":{ + "mean":0.189554, + "std":0.001014 + }, + "accuracy_8":{ + "mean":0.810424, + "std":0.000915 + }, + "oos_recall":{ + "mean":0.003, + "std":0.003606 + }, + "oos_recall_wilson95":[ + { + "mean":0.001314, + "std":0.001822 + }, + { + "mean":0.008489, + "std":0.005382 + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.005965, + "std":0.007158 + }, + "oos_misroute_rate":{ + "mean":0.997, + "std":0.003606 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.996333, + "std":0.004726 + }, + "llm_call_rate":{ + "mean":0.000848, + "std":0.001169 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000312, + "std":0.00043 + }, + "error_recovery_rate":{ + "mean":0.003184, + "std":0.003981 + }, + "recoverable_caught":{ + "mean":0.005634, + "std":0.007047 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":3.333333, + "std":4.163332 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.18539, + "std":0.019404 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.039677, + "std":0.001163 + }, + "validation_risk_upper95":{ + "mean":0.045857, + "std":0.001243 + }, + "test":{ + "coverage":{ + "mean":0.999333, + "std":0.000735 + }, + "selective_risk":{ + "mean":0.189641, + "std":0.00075 + }, + "accuracy_8":{ + "mean":0.810364, + "std":0.000833 + }, + "oos_recall":{ + "mean":0.003, + "std":0.003606 + }, + "oos_recall_wilson95":[ + { + "mean":0.001314, + "std":0.001822 + }, + { + "mean":0.008489, + "std":0.005382 + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.005965, + "std":0.007158 + }, + "oos_misroute_rate":{ + "mean":0.997, + "std":0.003606 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.996333, + "std":0.004041 + }, + "llm_call_rate":{ + "mean":0.000667, + "std":0.000735 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000244, + "std":0.00027 + }, + "error_recovery_rate":{ + "mean":0.002866, + "std":0.003448 + }, + "recoverable_caught":{ + "mean":0.00507, + "std":0.006103 + }, + "small_errors":{ + "mean":1046.0, + "std":4.0 + }, + "recoverable_errors":{ + "mean":591.333333, + "std":2.516611 + }, + "recovered_errors":{ + "mean":3.0, + "std":3.605551 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.190667, + "std":0.00042, + "values":[ + 0.190909, + 0.190182, + 0.190909 + ] + }, + "accuracy_8":{ + "mean":0.809333, + "std":0.00042, + "values":[ + 0.809091, + 0.809818, + 0.809091 + ] + }, + "oos_recall":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + { + "mean":0.003827, + "std":0.0, + "values":[ + 0.003827, + 0.003827, + 0.003827 + ] + } + ], + "oos_precision":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_f1":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1048.666667, + "std":2.309401, + "values":[ + 1050, + 1046, + 1050 + ] + }, + "recoverable_errors":{ + "mean":593.666667, + "std":2.516611, + "values":[ + 594, + 591, + 596 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.809333, + "std":0.00042, + "values":[ + 0.809091, + 0.809818, + 0.809091 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.917273, + "std":0.000182, + "values":[ + 0.917091, + 0.917273, + 0.917455 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.987254, + "std":0.00099, + "values":[ + 0.986111, + 0.987826, + 0.987826 + ] + }, + "oos_f1":{ + "mean":0.721117, + "std":0.000264, + "values":[ + 0.720812, + 0.72127, + 0.72127 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.190667, + "std":0.00042, + "values":[ + 0.190909, + 0.190182, + 0.190909 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.070148, + "std":0.000156, + "values":[ + 0.070239, + 0.069968, + 0.070238 + ] + }, + "error_recovery_rate":{ + "mean":0.566114, + "std":0.00135, + "values":[ + 0.565714, + 0.56501, + 0.567619 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1048.666667, + "std":2.309401, + "values":[ + 1050, + 1046, + 1050 + ] + }, + "recoverable_errors":{ + "mean":593.666667, + "std":2.516611, + "values":[ + 594, + 591, + 596 + ] + }, + "recovered_errors":{ + "mean":593.666667, + "std":2.516611, + "values":[ + 594, + 591, + 596 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.002237, + "std":0.000494, + "values":[ + 0.001709, + 0.002315, + 0.002688 + ] + }, + "oos_auroc":{ + "mean":0.976897, + "std":0.006299, + "values":[ + 0.982923, + 0.97741, + 0.970357 + ] + }, + "oos_auprc":{ + "mean":0.715388, + "std":0.08199, + "values":[ + 0.78824, + 0.731323, + 0.6266 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.023854, + "std":0.000446, + "values":[ + 0.023622, + 0.023573, + 0.024368 + ] + }, + "oos_auroc":{ + "mean":0.97695, + "std":0.002231, + "values":[ + 0.978348, + 0.978124, + 0.974377 + ] + }, + "oos_auprc":{ + "mean":0.908569, + "std":0.008479, + "values":[ + 0.914588, + 0.912247, + 0.898873 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.00217, + "std":0.00045, + "values":[ + 0.001673, + 0.00229, + 0.002548 + ] + }, + "oos_auroc":{ + "mean":0.978966, + "std":0.004784, + "values":[ + 0.984, + 0.978417, + 0.97448 + ] + }, + "oos_auprc":{ + "mean":0.756189, + "std":0.062772, + "values":[ + 0.820694, + 0.752564, + 0.695308 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.023561, + "std":0.000216, + "values":[ + 0.023452, + 0.023422, + 0.023809 + ] + }, + "oos_auroc":{ + "mean":0.979173, + "std":0.000865, + "values":[ + 0.979815, + 0.979516, + 0.978189 + ] + }, + "oos_auprc":{ + "mean":0.923273, + "std":0.001954, + "values":[ + 0.925307, + 0.923101, + 0.921412 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.002465, + "std":0.000722, + "values":[ + 0.001768, + 0.002419, + 0.00321 + ] + }, + "oos_auroc":{ + "mean":0.972551, + "std":0.010789, + "values":[ + 0.980237, + 0.9772, + 0.960217 + ] + }, + "oos_auprc":{ + "mean":0.623337, + "std":0.144489, + "values":[ + 0.691279, + 0.721332, + 0.457401 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.024665, + "std":0.001236, + "values":[ + 0.023876, + 0.02403, + 0.02609 + ] + }, + "oos_auroc":{ + "mean":0.97166, + "std":0.006864, + "values":[ + 0.975952, + 0.975284, + 0.963743 + ] + }, + "oos_auprc":{ + "mean":0.86389, + "std":0.041999, + "values":[ + 0.892502, + 0.883495, + 0.815673 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.002151, + "std":0.000407, + "values":[ + 0.001695, + 0.00228, + 0.002479 + ] + }, + "oos_auroc":{ + "mean":0.979147, + "std":0.003792, + "values":[ + 0.98295, + 0.979123, + 0.975367 + ] + }, + "oos_auprc":{ + "mean":0.751084, + "std":0.05669, + "values":[ + 0.795516, + 0.7705, + 0.687237 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.023401, + "std":0.00027, + "values":[ + 0.023218, + 0.023274, + 0.023711 + ] + }, + "oos_auroc":{ + "mean":0.979811, + "std":0.001238, + "values":[ + 0.980798, + 0.980212, + 0.978422 + ] + }, + "oos_auprc":{ + "mean":0.922028, + "std":0.004238, + "values":[ + 0.926467, + 0.921591, + 0.918025 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"entropy", + "hybrid":{ + "tau":{ + "mean":-1.1938, + "std":0.440804, + "values":[ + -0.959934, + -0.919213, + -1.702253 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.971613, + "std":0.002115, + "values":[ + 0.973548, + 0.971935, + 0.969355 + ] + }, + "validation_selective_risk":{ + "mean":0.015604, + "std":3.4e-05, + "values":[ + 0.015573, + 0.015599, + 0.015641 + ] + }, + "validation_risk_upper95":{ + "mean":0.019777, + "std":4.3e-05, + "values":[ + 0.019738, + 0.019771, + 0.019823 + ] + }, + "test":{ + "coverage":{ + "mean":0.879273, + "std":0.007264, + "values":[ + 0.885455, + 0.881091, + 0.871273 + ] + }, + "selective_risk":{ + "mean":0.08475, + "std":0.005611, + "values":[ + 0.089733, + 0.085844, + 0.078673 + ] + }, + "accuracy_8":{ + "mean":0.883152, + "std":0.001794, + "values":[ + 0.881091, + 0.884364, + 0.884 + ] + }, + "oos_recall":{ + "mean":0.406, + "std":0.01044, + "values":[ + 0.394, + 0.411, + 0.413 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.375983, + "std":0.010273, + "values":[ + 0.364175, + 0.380902, + 0.382872 + ] + }, + { + "mean":0.436737, + "std":0.010527, + "values":[ + 0.424636, + 0.44178, + 0.443794 + ] + } + ], + "oos_precision":{ + "mean":0.979937, + "std":0.003204, + "values":[ + 0.982544, + 0.980907, + 0.976359 + ] + }, + "oos_f1":{ + "mean":0.574067, + "std":0.010073, + "values":[ + 0.562455, + 0.579281, + 0.580464 + ] + }, + "oos_misroute_rate":{ + "mean":0.594, + "std":0.01044, + "values":[ + 0.606, + 0.589, + 0.587 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.376667, + "std":0.029023, + "values":[ + 0.402, + 0.383, + 0.345 + ] + }, + "llm_call_rate":{ + "mean":0.120727, + "std":0.007264, + "values":[ + 0.114545, + 0.118909, + 0.128727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.04438, + "std":0.002694, + "values":[ + 0.042086, + 0.043708, + 0.047347 + ] + }, + "error_recovery_rate":{ + "mean":0.394158, + "std":0.011494, + "values":[ + 0.380952, + 0.399618, + 0.401905 + ] + }, + "recoverable_caught":{ + "mean":0.696243, + "std":0.019786, + "values":[ + 0.673401, + 0.707276, + 0.708054 + ] + }, + "small_errors":{ + "mean":1048.666667, + "std":2.309401, + "values":[ + 1050, + 1046, + 1050 + ] + }, + "recoverable_errors":{ + "mean":593.666667, + "std":2.516611, + "values":[ + 594, + 591, + 596 + ] + }, + "recovered_errors":{ + "mean":413.333333, + "std":11.718931, + "values":[ + 400, + 418, + 422 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":-0.534049, + "std":1.151601, + "values":[ + -1.861188, + 0.201693, + 0.05735 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "validation_selective_risk":{ + "mean":0.039462, + "std":0.000931, + "values":[ + 0.04, + 0.038387, + 0.04 + ] + }, + "validation_risk_upper95":{ + "mean":0.045627, + "std":0.000995, + "values":[ + 0.046202, + 0.044477, + 0.046202 + ] + }, + "test":{ + "coverage":{ + "mean":0.998788, + "std":0.00042, + "values":[ + 0.999273, + 0.998545, + 0.998545 + ] + }, + "selective_risk":{ + "mean":0.189684, + "std":0.00066, + "values":[ + 0.19032, + 0.189002, + 0.189731 + ] + }, + "accuracy_8":{ + "mean":0.810303, + "std":0.000735, + "values":[ + 0.809636, + 0.811091, + 0.810182 + ] + }, + "oos_recall":{ + "mean":0.005333, + "std":0.002082, + "values":[ + 0.003, + 0.007, + 0.006 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.002389, + "std":0.001228, + "values":[ + 0.001021, + 0.003395, + 0.002753 + ] + }, + { + "mean":0.012063, + "std":0.00292, + "values":[ + 0.008783, + 0.014378, + 0.013028 + ] + } + ], + "oos_precision":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "oos_f1":{ + "mean":0.010604, + "std":0.004123, + "values":[ + 0.005982, + 0.013903, + 0.011928 + ] + }, + "oos_misroute_rate":{ + "mean":0.994667, + "std":0.002082, + "values":[ + 0.997, + 0.993, + 0.994 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.993333, + "std":0.002309, + "values":[ + 0.996, + 0.992, + 0.992 + ] + }, + "llm_call_rate":{ + "mean":0.001212, + "std":0.00042, + "values":[ + 0.000727, + 0.001455, + 0.001455 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.000445, + "std":0.000154, + "values":[ + 0.000267, + 0.000534, + 0.000533 + ] + }, + "error_recovery_rate":{ + "mean":0.005088, + "std":0.001993, + "values":[ + 0.002857, + 0.006692, + 0.005714 + ] + }, + "recoverable_caught":{ + "mean":0.008987, + "std":0.003523, + "values":[ + 0.005051, + 0.011844, + 0.010067 + ] + }, + "small_errors":{ + "mean":1048.666667, + "std":2.309401, + "values":[ + 1050, + 1046, + 1050 + ] + }, + "recoverable_errors":{ + "mean":593.666667, + "std":2.516611, + "values":[ + 594, + 591, + 596 + ] + }, + "recovered_errors":{ + "mean":5.333333, + "std":2.081666, + "values":[ + 3, + 7, + 6 + ] + } + } + } + } + } + } + } + }, + "modernbert/k1":{ + "model":"modernbert", + "k":1, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "ModernBERT-base-k1-seed42", + "ModernBERT-base-k1-seed43", + "ModernBERT-base-k1-seed44" + ], + "logits_sha256":[ + "aaa5c7cc5e068eed5da1dc4a55a91587761246e273c254564fd6bf69a27c2ae2", + "c2eae04634a5224685fed57cf0600993eb975a269a9ef61ec97f33598705786b", + "84067b0f3e6ad7ab14ed6b9b919b1b4b24ed1d0c3399dd81573f325adf2021b4" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":1.42348, + "std":0.082226, + "values":[ + 1.338303, + 1.429738, + 1.502398 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.10486, + "std":0.00877, + "values":[ + 0.09476, + 0.110546, + 0.109275 + ] + }, + "ece_8_summed":{ + "mean":0.039135, + "std":0.011453, + "values":[ + 0.02938, + 0.051745, + 0.036279 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.02294, + "std":0.01031, + "values":[ + 0.014707, + 0.019608, + 0.034503 + ] + }, + "ece_8_summed":{ + "mean":0.103995, + "std":0.018483, + "values":[ + 0.099704, + 0.124246, + 0.088036 + ] + } + } + } + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.449355, + "std":0.020713, + "values":[ + 0.468065, + 0.452903, + 0.427097 + ] + }, + "summed":{ + "mean":0.497742, + "std":0.026041, + "values":[ + 0.518065, + 0.506774, + 0.468387 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.449355, + "std":0.020713 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.616242, + "std":0.013153 + }, + "accuracy_8":{ + "mean":0.383758, + "std":0.013153 + }, + "oos_recall":{ + "mean":0.080667, + "std":0.028148 + }, + "oos_recall_wilson95":[ + { + "mean":0.065561, + "std":0.025277 + }, + { + "mean":0.098982, + "std":0.030805 + } + ], + "oos_precision":{ + "mean":0.388133, + "std":0.044003 + }, + "oos_f1":{ + "mean":0.132829, + "std":0.041499 + }, + "oos_misroute_rate":{ + "mean":0.919333, + "std":0.028148 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.919333, + "std":0.028148 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.383758, + "std":0.013153 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.859152, + "std":0.007287 + }, + "oos_recall":{ + "mean":0.591333, + "std":0.016166 + }, + "oos_recall_wilson95":[ + { + "mean":0.560583, + "std":0.016299 + }, + { + "mean":0.621385, + "std":0.015909 + } + ], + "oos_precision":{ + "mean":0.828957, + "std":0.02133 + }, + "oos_f1":{ + "mean":0.690264, + "std":0.018331 + }, + "oos_misroute_rate":{ + "mean":0.408667, + "std":0.016166 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.616242, + "std":0.013153 + }, + "llm_cost_usd_per_1k":{ + "mean":0.227558, + "std":0.004886 + }, + "error_recovery_rate":{ + "mean":0.771539, + "std":0.007061 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2614.666667, + "std":32.65476 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.404138, + "std":0.027703 + }, + "oos_auroc":{ + "mean":0.559918, + "std":0.033945 + }, + "oos_auprc":{ + "mean":0.039889, + "std":0.006668 + } + }, + "test":{ + "aurc":{ + "mean":0.468591, + "std":0.018202 + }, + "oos_auroc":{ + "mean":0.574522, + "std":0.009485 + }, + "oos_auprc":{ + "mean":0.216415, + "std":0.010021 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.40398, + "std":0.028149 + }, + "oos_auroc":{ + "mean":0.572066, + "std":0.033156 + }, + "oos_auprc":{ + "mean":0.03907, + "std":0.004649 + } + }, + "test":{ + "aurc":{ + "mean":0.466075, + "std":0.017896 + }, + "oos_auroc":{ + "mean":0.587486, + "std":0.012441 + }, + "oos_auprc":{ + "mean":0.226453, + "std":0.008664 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.424007, + "std":0.025072 + }, + "oos_auroc":{ + "mean":0.541648, + "std":0.022525 + }, + "oos_auprc":{ + "mean":0.038543, + "std":0.003849 + } + }, + "test":{ + "aurc":{ + "mean":0.487802, + "std":0.017677 + }, + "oos_auroc":{ + "mean":0.547051, + "std":0.005518 + }, + "oos_auprc":{ + "mean":0.198131, + "std":0.006215 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.402907, + "std":0.027895 + }, + "oos_auroc":{ + "mean":0.563896, + "std":0.033625 + }, + "oos_auprc":{ + "mean":0.041653, + "std":0.009436 + } + }, + "test":{ + "aurc":{ + "mean":0.466832, + "std":0.01771 + }, + "oos_auroc":{ + "mean":0.578278, + "std":0.010299 + }, + "oos_auprc":{ + "mean":0.21896, + "std":0.010093 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.771539, + "std":0.007061 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2614.666667, + "std":32.65476 + } + } + }, + "by_signal":{ + "msp":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.771539, + "std":0.007061 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2614.666667, + "std":32.65476 + } + } + }, + "entropy":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.771539, + "std":0.007061 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2614.666667, + "std":32.65476 + } + } + }, + "margin":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.771539, + "std":0.007061 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2614.666667, + "std":32.65476 + } + } + }, + "msp_t":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.771539, + "std":0.007061 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2614.666667, + "std":32.65476 + } + } + } + } + }, + "0.05":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":3.262226, + "std":0.257402 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.021505, + "std":0.001221 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.039076, + "std":0.002073 + }, + "test":{ + "coverage":{ + "mean":0.014545, + "std":0.002446 + }, + "selective_risk":{ + "mean":0.080282, + "std":0.056007 + }, + "accuracy_8":{ + "mean":0.820909, + "std":0.000182 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.750331, + "std":0.000991 + }, + "oos_f1":{ + "mean":0.646557, + "std":0.000368 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002, + "std":0.002 + }, + "llm_call_rate":{ + "mean":0.985455, + "std":0.002446 + }, + "llm_cost_usd_per_1k":{ + "mean":0.364117, + "std":0.000899 + }, + "error_recovery_rate":{ + "mean":0.770463, + "std":0.007381 + }, + "recoverable_caught":{ + "mean":0.998602, + "std":0.000783 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2611.0, + "std":31.176915 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.956288, + "std":0.001737, + "n":2 + }, + "feasible":[ + false, + true, + true + ], + "validation_coverage":{ + "mean":0.011613, + "std":0.010062 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.047727, + "std":0.00119, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.007576, + "std":0.006637 + }, + "selective_risk":{ + "mean":0.067337, + "std":0.053635, + "n":2 + }, + "accuracy_8":{ + "mean":0.820848, + "std":0.00021 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.749341, + "std":0.000989 + }, + "oos_f1":{ + "mean":0.646189, + "std":0.000368 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.001, + "std":0.001 + }, + "llm_call_rate":{ + "mean":0.992424, + "std":0.006637 + }, + "llm_cost_usd_per_1k":{ + "mean":0.366696, + "std":0.002449 + }, + "error_recovery_rate":{ + "mean":0.77115, + "std":0.007289 + }, + "recoverable_caught":{ + "mean":0.999494, + "std":0.000876 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2613.333333, + "std":31.469562 + } + } + }, + "entropy":{ + "tau":{ + "mean":-0.350296, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.005591, + "std":0.009685 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.049456, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.004121, + "std":0.007138 + }, + "selective_risk":{ + "mean":0.029412, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820848, + "std":0.00021 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.749012, + "std":0.001142 + }, + "oos_f1":{ + "mean":0.646067, + "std":0.000424 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.000333, + "std":0.000577 + }, + "llm_call_rate":{ + "mean":0.995879, + "std":0.007138 + }, + "llm_cost_usd_per_1k":{ + "mean":0.36797, + "std":0.002635 + }, + "error_recovery_rate":{ + "mean":0.771539, + "std":0.007061 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2614.666667, + "std":32.65476 + } + } + }, + "margin":{ + "tau":{ + "mean":3.262226, + "std":0.257402 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.021505, + "std":0.001221 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.039076, + "std":0.002073 + }, + "test":{ + "coverage":{ + "mean":0.014545, + "std":0.002446 + }, + "selective_risk":{ + "mean":0.080282, + "std":0.056007 + }, + "accuracy_8":{ + "mean":0.820909, + "std":0.000182 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.750331, + "std":0.000991 + }, + "oos_f1":{ + "mean":0.646557, + "std":0.000368 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002, + "std":0.002 + }, + "llm_call_rate":{ + "mean":0.985455, + "std":0.002446 + }, + "llm_cost_usd_per_1k":{ + "mean":0.364117, + "std":0.000899 + }, + "error_recovery_rate":{ + "mean":0.770463, + "std":0.007381 + }, + "recoverable_caught":{ + "mean":0.998602, + "std":0.000783 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2611.0, + "std":31.176915 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.714544, + "std":0.0, + "n":1 + }, + "feasible":[ + false, + true, + false + ], + "validation_coverage":{ + "mean":0.005699, + "std":0.009871 + }, + "validation_selective_risk":{ + "mean":0.0, + "std":0.0, + "n":1 + }, + "validation_risk_upper95":{ + "mean":0.048569, + "std":0.0, + "n":1 + }, + "test":{ + "coverage":{ + "mean":0.004242, + "std":0.007348 + }, + "selective_risk":{ + "mean":0.028571, + "std":0.0, + "n":1 + }, + "accuracy_8":{ + "mean":0.820848, + "std":0.00021 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.749012, + "std":0.001142 + }, + "oos_f1":{ + "mean":0.646067, + "std":0.000424 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.000333, + "std":0.000577 + }, + "llm_call_rate":{ + "mean":0.995758, + "std":0.007348 + }, + "llm_cost_usd_per_1k":{ + "mean":0.367925, + "std":0.002713 + }, + "error_recovery_rate":{ + "mean":0.771539, + "std":0.007061 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3389.333333, + "std":72.341781 + }, + "recoverable_errors":{ + "mean":2614.666667, + "std":32.65476 + }, + "recovered_errors":{ + "mean":2614.666667, + "std":32.65476 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.497742, + "std":0.026041 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.582909, + "std":0.024735 + }, + "accuracy_8":{ + "mean":0.417091, + "std":0.024735 + }, + "oos_recall":{ + "mean":0.017333, + "std":0.007095 + }, + "oos_recall_wilson95":[ + { + "mean":0.011005, + "std":0.005507 + }, + { + "mean":0.027356, + "std":0.008629 + } + ], + "oos_precision":{ + "mean":0.536179, + "std":0.093577 + }, + "oos_f1":{ + "mean":0.033456, + "std":0.013233 + }, + "oos_misroute_rate":{ + "mean":0.982667, + "std":0.007095 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.982667, + "std":0.007095 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.417091, + "std":0.024735 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.854909, + "std":0.00411 + }, + "oos_recall":{ + "mean":0.571, + "std":0.002646 + }, + "oos_recall_wilson95":[ + { + "mean":0.54011, + "std":0.002659 + }, + { + "mean":0.601347, + "std":0.002612 + } + ], + "oos_precision":{ + "mean":0.838153, + "std":0.012436 + }, + "oos_f1":{ + "mean":0.679239, + "std":0.005797 + }, + "oos_misroute_rate":{ + "mean":0.429, + "std":0.002646 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.582909, + "std":0.024735 + }, + "llm_cost_usd_per_1k":{ + "mean":0.215199, + "std":0.009164 + }, + "error_recovery_rate":{ + "mean":0.750974, + "std":0.005265 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2408.0, + "std":116.413058 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.309432, + "std":0.027806 + }, + "oos_auroc":{ + "mean":0.626827, + "std":0.028635 + }, + "oos_auprc":{ + "mean":0.049292, + "std":0.008642 + } + }, + "test":{ + "aurc":{ + "mean":0.382036, + "std":0.030285 + }, + "oos_auroc":{ + "mean":0.626271, + "std":0.013557 + }, + "oos_auprc":{ + "mean":0.252584, + "std":0.015859 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.316008, + "std":0.026655 + }, + "oos_auroc":{ + "mean":0.662003, + "std":0.02613 + }, + "oos_auprc":{ + "mean":0.073606, + "std":0.027321 + } + }, + "test":{ + "aurc":{ + "mean":0.385837, + "std":0.029689 + }, + "oos_auroc":{ + "mean":0.653792, + "std":0.019812 + }, + "oos_auprc":{ + "mean":0.289128, + "std":0.038405 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.302951, + "std":0.027287 + }, + "oos_auroc":{ + "mean":0.601588, + "std":0.032908 + }, + "oos_auprc":{ + "mean":0.041971, + "std":0.004423 + } + }, + "test":{ + "aurc":{ + "mean":0.38097, + "std":0.029687 + }, + "oos_auroc":{ + "mean":0.589918, + "std":0.012587 + }, + "oos_auprc":{ + "mean":0.215822, + "std":0.010581 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.291633, + "std":0.022804 + }, + "oos_auroc":{ + "mean":0.650058, + "std":0.015363 + }, + "oos_auprc":{ + "mean":0.056116, + "std":0.008242 + } + }, + "test":{ + "aurc":{ + "mean":0.366584, + "std":0.02742 + }, + "oos_auroc":{ + "mean":0.624161, + "std":0.023137 + }, + "oos_auprc":{ + "mean":0.249447, + "std":0.026098 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.750974, + "std":0.005265 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2408.0, + "std":116.413058 + } + } + }, + "by_signal":{ + "msp":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.750974, + "std":0.005265 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2408.0, + "std":116.413058 + } + } + }, + "entropy":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.750974, + "std":0.005265 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2408.0, + "std":116.413058 + } + } + }, + "margin":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.750974, + "std":0.005265 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2408.0, + "std":116.413058 + } + } + }, + "msp_t":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.750974, + "std":0.005265 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2408.0, + "std":116.413058 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp", + "msp_t", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.080174, + "std":0.555253 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.036022, + "std":0.016707 + }, + "validation_selective_risk":{ + "mean":0.0119, + "std":0.010345 + }, + "validation_risk_upper95":{ + "mean":0.047077, + "std":0.002492 + }, + "test":{ + "coverage":{ + "mean":0.028485, + "std":0.016547 + }, + "selective_risk":{ + "mean":0.077161, + "std":0.040823 + }, + "accuracy_8":{ + "mean":0.820667, + "std":0.000458 + }, + "oos_recall":{ + "mean":0.567667, + "std":0.000577 + }, + "oos_recall_wilson95":[ + { + "mean":0.53676, + "std":0.00058 + }, + { + "mean":0.598055, + "std":0.00057 + } + ], + "oos_precision":{ + "mean":0.75222, + "std":0.003273 + }, + "oos_f1":{ + "mean":0.647038, + "std":0.000833 + }, + "oos_misroute_rate":{ + "mean":0.432333, + "std":0.000577 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.005667, + "std":0.005132 + }, + "llm_call_rate":{ + "mean":0.971515, + "std":0.016547 + }, + "llm_cost_usd_per_1k":{ + "mean":0.358961, + "std":0.006113 + }, + "error_recovery_rate":{ + "mean":0.74923, + "std":0.005299 + }, + "recoverable_caught":{ + "mean":0.997678, + "std":0.001641 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2402.333333, + "std":113.808318 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.950561, + "std":0.016296, + "n":2 + }, + "feasible":[ + true, + false, + true + ], + "validation_coverage":{ + "mean":0.015484, + "std":0.014256 + }, + "validation_selective_risk":{ + "mean":0.005747, + "std":0.008128, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.047601, + "std":0.003233, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.010485, + "std":0.009886 + }, + "selective_risk":{ + "mean":0.07557, + "std":0.063358, + "n":2 + }, + "accuracy_8":{ + "mean":0.820727, + "std":0.000364 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.749671, + "std":0.001142 + }, + "oos_f1":{ + "mean":0.646312, + "std":0.000424 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.001667, + "std":0.002887 + }, + "llm_call_rate":{ + "mean":0.989515, + "std":0.009886 + }, + "llm_cost_usd_per_1k":{ + "mean":0.365617, + "std":0.003652 + }, + "error_recovery_rate":{ + "mean":0.750171, + "std":0.004181 + }, + "recoverable_caught":{ + "mean":0.998938, + "std":0.001482 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2405.333333, + "std":112.828779 + } + } + }, + "entropy":{ + "tau":{ + "mean":-0.261598, + "std":0.079567, + "n":2 + }, + "feasible":[ + true, + false, + true + ], + "validation_coverage":{ + "mean":0.015161, + "std":0.014168 + }, + "validation_selective_risk":{ + "mean":0.005747, + "std":0.008128, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.0488, + "std":0.001538, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.010182, + "std":0.009838 + }, + "selective_risk":{ + "mean":0.076852, + "std":0.061544, + "n":2 + }, + "accuracy_8":{ + "mean":0.820667, + "std":0.000458 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.749671, + "std":0.001142 + }, + "oos_f1":{ + "mean":0.646312, + "std":0.000424 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.001333, + "std":0.002309 + }, + "llm_call_rate":{ + "mean":0.989818, + "std":0.009838 + }, + "llm_cost_usd_per_1k":{ + "mean":0.365729, + "std":0.003634 + }, + "error_recovery_rate":{ + "mean":0.750071, + "std":0.004026 + }, + "recoverable_caught":{ + "mean":0.998806, + "std":0.001708 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2405.0, + "std":112.263084 + } + } + }, + "margin":{ + "tau":{ + "mean":2.04137, + "std":0.66376 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.034839, + "std":0.015417 + }, + "validation_selective_risk":{ + "mean":0.012316, + "std":0.010775 + }, + "validation_risk_upper95":{ + "mean":0.048334, + "std":0.002002 + }, + "test":{ + "coverage":{ + "mean":0.027879, + "std":0.01747 + }, + "selective_risk":{ + "mean":0.072965, + "std":0.047992 + }, + "accuracy_8":{ + "mean":0.820545, + "std":0.000481 + }, + "oos_recall":{ + "mean":0.567667, + "std":0.000577 + }, + "oos_recall_wilson95":[ + { + "mean":0.53676, + "std":0.00058 + }, + { + "mean":0.598055, + "std":0.00057 + } + ], + "oos_precision":{ + "mean":0.751555, + "std":0.003019 + }, + "oos_f1":{ + "mean":0.646792, + "std":0.000749 + }, + "oos_misroute_rate":{ + "mean":0.432333, + "std":0.000577 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.006, + "std":0.005568 + }, + "llm_call_rate":{ + "mean":0.972121, + "std":0.01747 + }, + "llm_cost_usd_per_1k":{ + "mean":0.359185, + "std":0.006454 + }, + "error_recovery_rate":{ + "mean":0.749338, + "std":0.005318 + }, + "recoverable_caught":{ + "mean":0.997822, + "std":0.00189 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2402.666667, + "std":113.443084 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.610094, + "std":0.023516, + "n":2 + }, + "feasible":[ + false, + true, + true + ], + "validation_coverage":{ + "mean":0.029247, + "std":0.026486 + }, + "validation_selective_risk":{ + "mean":0.013839, + "std":0.006945, + "n":2 + }, + "validation_risk_upper95":{ + "mean":0.042508, + "std":0.004921, + "n":2 + }, + "test":{ + "coverage":{ + "mean":0.02497, + "std":0.022873 + }, + "selective_risk":{ + "mean":0.095044, + "std":0.002724, + "n":2 + }, + "accuracy_8":{ + "mean":0.820667, + "std":0.000105 + }, + "oos_recall":{ + "mean":0.567667, + "std":0.000577 + }, + "oos_recall_wilson95":[ + { + "mean":0.53676, + "std":0.00058 + }, + { + "mean":0.598055, + "std":0.00057 + } + ], + "oos_precision":{ + "mean":0.751892, + "std":0.003855 + }, + "oos_f1":{ + "mean":0.646915, + "std":0.001089 + }, + "oos_misroute_rate":{ + "mean":0.432333, + "std":0.000577 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.005, + "std":0.005 + }, + "llm_call_rate":{ + "mean":0.97503, + "std":0.022873 + }, + "llm_cost_usd_per_1k":{ + "mean":0.360262, + "std":0.008454 + }, + "error_recovery_rate":{ + "mean":0.749338, + "std":0.005318 + }, + "recoverable_caught":{ + "mean":0.997822, + "std":0.00189 + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435 + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058 + }, + "recovered_errors":{ + "mean":2402.666667, + "std":113.443084 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.582909, + "std":0.024735, + "values":[ + 0.562364, + 0.576, + 0.610364 + ] + }, + "accuracy_8":{ + "mean":0.417091, + "std":0.024735, + "values":[ + 0.437636, + 0.424, + 0.389636 + ] + }, + "oos_recall":{ + "mean":0.017333, + "std":0.007095, + "values":[ + 0.025, + 0.011, + 0.016 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.011005, + "std":0.005507, + "values":[ + 0.01699, + 0.006153, + 0.009872 + ] + }, + { + "mean":0.027356, + "std":0.008629, + "values":[ + 0.036646, + 0.01959, + 0.025832 + ] + } + ], + "oos_precision":{ + "mean":0.536179, + "std":0.093577, + "values":[ + 0.510204, + 0.458333, + 0.64 + ] + }, + "oos_f1":{ + "mean":0.033456, + "std":0.013233, + "values":[ + 0.047664, + 0.021484, + 0.03122 + ] + }, + "oos_misroute_rate":{ + "mean":0.982667, + "std":0.007095, + "values":[ + 0.975, + 0.989, + 0.984 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.982667, + "std":0.007095, + "values":[ + 0.975, + 0.989, + 0.984 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435, + "values":[ + 3093, + 3168, + 3357 + ] + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058, + "values":[ + 2320, + 2364, + 2540 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.417091, + "std":0.024735, + "values":[ + 0.437636, + 0.424, + 0.389636 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.854909, + "std":0.00411, + "values":[ + 0.859455, + 0.853818, + 0.851455 + ] + }, + "oos_recall":{ + "mean":0.571, + "std":0.002646, + "values":[ + 0.574, + 0.57, + 0.569 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.54011, + "std":0.002659, + "values":[ + 0.543125, + 0.539105, + 0.5381 + ] + }, + { + "mean":0.601347, + "std":0.002612, + "values":[ + 0.604308, + 0.60036, + 0.599372 + ] + } + ], + "oos_precision":{ + "mean":0.838153, + "std":0.012436, + "values":[ + 0.849112, + 0.840708, + 0.824638 + ] + }, + "oos_f1":{ + "mean":0.679239, + "std":0.005797, + "values":[ + 0.684964, + 0.67938, + 0.673373 + ] + }, + "oos_misroute_rate":{ + "mean":0.429, + "std":0.002646, + "values":[ + 0.426, + 0.43, + 0.431 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.582909, + "std":0.024735, + "values":[ + 0.562364, + 0.576, + 0.610364 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.215199, + "std":0.009164, + "values":[ + 0.207568, + 0.212665, + 0.225363 + ] + }, + "error_recovery_rate":{ + "mean":0.750974, + "std":0.005265, + "values":[ + 0.750081, + 0.746212, + 0.756628 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435, + "values":[ + 3093, + 3168, + 3357 + ] + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058, + "values":[ + 2320, + 2364, + 2540 + ] + }, + "recovered_errors":{ + "mean":2408.0, + "std":116.413058, + "values":[ + 2320, + 2364, + 2540 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.309432, + "std":0.027806, + "values":[ + 0.285073, + 0.303497, + 0.339727 + ] + }, + "oos_auroc":{ + "mean":0.626827, + "std":0.028635, + "values":[ + 0.655063, + 0.627607, + 0.59781 + ] + }, + "oos_auprc":{ + "mean":0.049292, + "std":0.008642, + "values":[ + 0.058446, + 0.048156, + 0.041274 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.382036, + "std":0.030285, + "values":[ + 0.358633, + 0.371235, + 0.416241 + ] + }, + "oos_auroc":{ + "mean":0.626271, + "std":0.013557, + "values":[ + 0.641721, + 0.616362, + 0.620729 + ] + }, + "oos_auprc":{ + "mean":0.252584, + "std":0.015859, + "values":[ + 0.270822, + 0.242038, + 0.244891 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.316008, + "std":0.026655, + "values":[ + 0.292777, + 0.310136, + 0.345109 + ] + }, + "oos_auroc":{ + "mean":0.662003, + "std":0.02613, + "values":[ + 0.691383, + 0.653263, + 0.641363 + ] + }, + "oos_auprc":{ + "mean":0.073606, + "std":0.027321, + "values":[ + 0.105137, + 0.05694, + 0.058742 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.385837, + "std":0.029689, + "values":[ + 0.363322, + 0.374706, + 0.419483 + ] + }, + "oos_auroc":{ + "mean":0.653792, + "std":0.019812, + "values":[ + 0.674976, + 0.635722, + 0.650677 + ] + }, + "oos_auprc":{ + "mean":0.289128, + "std":0.038405, + "values":[ + 0.332154, + 0.258312, + 0.276918 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.302951, + "std":0.027287, + "values":[ + 0.284151, + 0.290454, + 0.334248 + ] + }, + "oos_auroc":{ + "mean":0.601588, + "std":0.032908, + "values":[ + 0.60916, + 0.63005, + 0.565553 + ] + }, + "oos_auprc":{ + "mean":0.041971, + "std":0.004423, + "values":[ + 0.04329, + 0.045584, + 0.037038 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.38097, + "std":0.029687, + "values":[ + 0.36249, + 0.365207, + 0.415213 + ] + }, + "oos_auroc":{ + "mean":0.589918, + "std":0.012587, + "values":[ + 0.600427, + 0.575968, + 0.593358 + ] + }, + "oos_auprc":{ + "mean":0.215822, + "std":0.010581, + "values":[ + 0.224686, + 0.204108, + 0.218673 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.291633, + "std":0.022804, + "values":[ + 0.276059, + 0.281032, + 0.317807 + ] + }, + "oos_auroc":{ + "mean":0.650058, + "std":0.015363, + "values":[ + 0.660333, + 0.657443, + 0.632397 + ] + }, + "oos_auprc":{ + "mean":0.056116, + "std":0.008242, + "values":[ + 0.06478, + 0.055195, + 0.048373 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.366584, + "std":0.02742, + "values":[ + 0.350238, + 0.351273, + 0.39824 + ] + }, + "oos_auroc":{ + "mean":0.624161, + "std":0.023137, + "values":[ + 0.647734, + 0.601486, + 0.623263 + ] + }, + "oos_auprc":{ + "mean":0.249447, + "std":0.026098, + "values":[ + 0.277887, + 0.226597, + 0.243856 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0, + "values":[ + 0.820727, + 0.820727, + 0.820727 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0, + "values":[ + 0.748353, + 0.748353, + 0.748353 + ] + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0, + "values":[ + 0.645821, + 0.645821, + 0.645821 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0, + "values":[ + 0.369492, + 0.369492, + 0.369492 + ] + }, + "error_recovery_rate":{ + "mean":0.750974, + "std":0.005265, + "values":[ + 0.750081, + 0.746212, + 0.756628 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435, + "values":[ + 3093, + 3168, + 3357 + ] + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058, + "values":[ + 2320, + 2364, + 2540 + ] + }, + "recovered_errors":{ + "mean":2408.0, + "std":116.413058, + "values":[ + 2320, + 2364, + 2540 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp", + "msp_t", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.080174, + "std":0.555253, + "values":[ + 0.962084, + 0.593466, + 1.684973 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.036022, + "std":0.016707, + "values":[ + 0.018387, + 0.051613, + 0.038065 + ] + }, + "validation_selective_risk":{ + "mean":0.0119, + "std":0.010345, + "values":[ + 0.0, + 0.01875, + 0.016949 + ] + }, + "validation_risk_upper95":{ + "mean":0.047077, + "std":0.002492, + "values":[ + 0.045315, + 0.045987, + 0.049928 + ] + }, + "test":{ + "coverage":{ + "mean":0.028485, + "std":0.016547, + "values":[ + 0.011818, + 0.044909, + 0.028727 + ] + }, + "selective_risk":{ + "mean":0.077161, + "std":0.040823, + "values":[ + 0.030769, + 0.093117, + 0.107595 + ] + }, + "accuracy_8":{ + "mean":0.820667, + "std":0.000458, + "values":[ + 0.821091, + 0.820727, + 0.820182 + ] + }, + "oos_recall":{ + "mean":0.567667, + "std":0.000577, + "values":[ + 0.568, + 0.567, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.53676, + "std":0.00058, + "values":[ + 0.537095, + 0.536091, + 0.537095 + ] + }, + { + "mean":0.598055, + "std":0.00057, + "values":[ + 0.598384, + 0.597397, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.75222, + "std":0.003273, + "values":[ + 0.75033, + 0.756, + 0.75033 + ] + }, + "oos_f1":{ + "mean":0.647038, + "std":0.000833, + "values":[ + 0.646557, + 0.648, + 0.646557 + ] + }, + "oos_misroute_rate":{ + "mean":0.432333, + "std":0.000577, + "values":[ + 0.432, + 0.433, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.005667, + "std":0.005132, + "values":[ + 0.0, + 0.01, + 0.007 + ] + }, + "llm_call_rate":{ + "mean":0.971515, + "std":0.016547, + "values":[ + 0.988182, + 0.955091, + 0.971273 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.358961, + "std":0.006113, + "values":[ + 0.365119, + 0.352893, + 0.358871 + ] + }, + "error_recovery_rate":{ + "mean":0.74923, + "std":0.005299, + "values":[ + 0.749758, + 0.743687, + 0.754245 + ] + }, + "recoverable_caught":{ + "mean":0.997678, + "std":0.001641, + "values":[ + 0.999569, + 0.996616, + 0.99685 + ] + }, + "small_errors":{ + "mean":3206.0, + "std":136.040435, + "values":[ + 3093, + 3168, + 3357 + ] + }, + "recoverable_errors":{ + "mean":2408.0, + "std":116.413058, + "values":[ + 2320, + 2364, + 2540 + ] + }, + "recovered_errors":{ + "mean":2402.333333, + "std":113.808318, + "values":[ + 2319, + 2356, + 2532 + ] + } + } + } + } + } + } + } + }, + "modernbert/k10":{ + "model":"modernbert", + "k":10, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "ModernBERT-base-k10-seed42", + "ModernBERT-base-k10-seed43", + "ModernBERT-base-k10-seed44" + ], + "logits_sha256":[ + "6fff9cd41ff6f9c828b9337b3c190f6e4cb1e41dff62667329f5e0647f81afba", + "f08779235eb7af011ab4c6402e86fe393d5cb9cc0d341ecfb01992fe97d52004", + "6c20ced25748c4ed64688269d0ab4fe74f0b32ae9af565c8ccd4cf6ed72140d9" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":0.892542, + "std":0.022943, + "values":[ + 0.91842, + 0.884512, + 0.874692 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.022366, + "std":0.004325, + "values":[ + 0.024725, + 0.017374, + 0.025 + ] + }, + "ece_8_summed":{ + "mean":0.028488, + "std":0.007061, + "values":[ + 0.020673, + 0.030383, + 0.034409 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.028102, + "std":0.002998, + "values":[ + 0.025778, + 0.031486, + 0.027041 + ] + }, + "ece_8_summed":{ + "mean":0.027741, + "std":0.008972, + "values":[ + 0.02055, + 0.037795, + 0.024879 + ] + } + } + } + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.920753, + "std":0.01407, + "values":[ + 0.929355, + 0.928387, + 0.904516 + ] + }, + "summed":{ + "mean":0.924839, + "std":0.012296, + "values":[ + 0.931613, + 0.932258, + 0.910645 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.920753, + "std":0.01407 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.180061, + "std":0.011099 + }, + "accuracy_8":{ + "mean":0.819939, + "std":0.011099 + }, + "oos_recall":{ + "mean":0.288, + "std":0.053926 + }, + "oos_recall_wilson95":[ + { + "mean":0.260943, + "std":0.052286 + }, + { + "mean":0.316679, + "std":0.055153 + } + ], + "oos_precision":{ + "mean":0.909912, + "std":0.023365 + }, + "oos_f1":{ + "mean":0.436164, + "std":0.063829 + }, + "oos_misroute_rate":{ + "mean":0.712, + "std":0.053926 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.712, + "std":0.053926 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.819939, + "std":0.011099 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.917576, + "std":0.0029 + }, + "oos_recall":{ + "mean":0.632667, + "std":0.014503 + }, + "oos_recall_wilson95":[ + { + "mean":0.602342, + "std":0.014693 + }, + { + "mean":0.661976, + "std":0.014202 + } + ], + "oos_precision":{ + "mean":0.955228, + "std":0.006663 + }, + "oos_f1":{ + "mean":0.76112, + "std":0.010971 + }, + "oos_misroute_rate":{ + "mean":0.367333, + "std":0.014503 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.180061, + "std":0.011099 + }, + "llm_cost_usd_per_1k":{ + "mean":0.066333, + "std":0.004109 + }, + "error_recovery_rate":{ + "mean":0.541704, + "std":0.01368 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":537.0, + "std":45.90207 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.011532, + "std":0.002931 + }, + "oos_auroc":{ + "mean":0.862158, + "std":0.024161 + }, + "oos_auprc":{ + "mean":0.17518, + "std":0.041945 + } + }, + "test":{ + "aurc":{ + "mean":0.037842, + "std":0.002225 + }, + "oos_auroc":{ + "mean":0.878535, + "std":0.012268 + }, + "oos_auprc":{ + "mean":0.581657, + "std":0.033855 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.011356, + "std":0.002563 + }, + "oos_auroc":{ + "mean":0.880434, + "std":0.021496 + }, + "oos_auprc":{ + "mean":0.212544, + "std":0.039679 + } + }, + "test":{ + "aurc":{ + "mean":0.036739, + "std":0.002276 + }, + "oos_auroc":{ + "mean":0.894022, + "std":0.011985 + }, + "oos_auprc":{ + "mean":0.624238, + "std":0.033484 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.014539, + "std":0.004082 + }, + "oos_auroc":{ + "mean":0.807733, + "std":0.02688 + }, + "oos_auprc":{ + "mean":0.098098, + "std":0.012763 + } + }, + "test":{ + "aurc":{ + "mean":0.045883, + "std":0.003408 + }, + "oos_auroc":{ + "mean":0.825807, + "std":0.015384 + }, + "oos_auprc":{ + "mean":0.4322, + "std":0.023044 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.011719, + "std":0.003082 + }, + "oos_auroc":{ + "mean":0.856798, + "std":0.024613 + }, + "oos_auprc":{ + "mean":0.167864, + "std":0.039334 + } + }, + "test":{ + "aurc":{ + "mean":0.038399, + "std":0.002345 + }, + "oos_auroc":{ + "mean":0.873833, + "std":0.01243 + }, + "oos_auprc":{ + "mean":0.572439, + "std":0.033181 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"entropy", + "hybrid":{ + "tau":{ + "mean":-1.750068, + "std":0.066052 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.779677, + "std":0.040357 + }, + "validation_selective_risk":{ + "mean":0.01503, + "std":9.6e-05 + }, + "validation_risk_upper95":{ + "mean":0.01968, + "std":7.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.681697, + "std":0.033642 + }, + "selective_risk":{ + "mean":0.033128, + "std":0.004468 + }, + "accuracy_8":{ + "mean":0.878061, + "std":0.003437 + }, + "oos_recall":{ + "mean":0.553333, + "std":0.005859 + }, + "oos_recall_wilson95":[ + { + "mean":0.522375, + "std":0.005877 + }, + { + "mean":0.583883, + "std":0.005797 + } + ], + "oos_precision":{ + "mean":0.885673, + "std":0.01701 + }, + "oos_f1":{ + "mean":0.681031, + "std":0.000837 + }, + "oos_misroute_rate":{ + "mean":0.446667, + "std":0.005859 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.069, + "std":0.016523 + }, + "llm_call_rate":{ + "mean":0.318303, + "std":0.033642 + }, + "llm_cost_usd_per_1k":{ + "mean":0.117339, + "std":0.012466 + }, + "error_recovery_rate":{ + "mean":0.479278, + "std":0.025486 + }, + "recoverable_caught":{ + "mean":0.884347, + "std":0.025256 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":475.666667, + "std":53.668737 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.589549, + "std":0.026948 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.766022, + "std":0.04865 + }, + "validation_selective_risk":{ + "mean":0.015153, + "std":0.000157 + }, + "validation_risk_upper95":{ + "mean":0.019869, + "std":1.4e-05 + }, + "test":{ + "coverage":{ + "mean":0.677333, + "std":0.042763 + }, + "selective_risk":{ + "mean":0.037427, + "std":0.005787 + }, + "accuracy_8":{ + "mean":0.876182, + "std":0.004345 + }, + "oos_recall":{ + "mean":0.548333, + "std":0.010066 + }, + "oos_recall_wilson95":[ + { + "mean":0.517366, + "std":0.010089 + }, + { + "mean":0.578931, + "std":0.009966 + } + ], + "oos_precision":{ + "mean":0.884196, + "std":0.015513 + }, + "oos_f1":{ + "mean":0.676794, + "std":0.00698 + }, + "oos_misroute_rate":{ + "mean":0.451667, + "std":0.010066 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.085333, + "std":0.021779 + }, + "llm_call_rate":{ + "mean":0.322667, + "std":0.042763 + }, + "llm_cost_usd_per_1k":{ + "mean":0.118974, + "std":0.01584 + }, + "error_recovery_rate":{ + "mean":0.474047, + "std":0.031912 + }, + "recoverable_caught":{ + "mean":0.874486, + "std":0.036899 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":470.666667, + "std":58.943476 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.750068, + "std":0.066052 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.779677, + "std":0.040357 + }, + "validation_selective_risk":{ + "mean":0.01503, + "std":9.6e-05 + }, + "validation_risk_upper95":{ + "mean":0.01968, + "std":7.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.681697, + "std":0.033642 + }, + "selective_risk":{ + "mean":0.033128, + "std":0.004468 + }, + "accuracy_8":{ + "mean":0.878061, + "std":0.003437 + }, + "oos_recall":{ + "mean":0.553333, + "std":0.005859 + }, + "oos_recall_wilson95":[ + { + "mean":0.522375, + "std":0.005877 + }, + { + "mean":0.583883, + "std":0.005797 + } + ], + "oos_precision":{ + "mean":0.885673, + "std":0.01701 + }, + "oos_f1":{ + "mean":0.681031, + "std":0.000837 + }, + "oos_misroute_rate":{ + "mean":0.446667, + "std":0.005859 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.069, + "std":0.016523 + }, + "llm_call_rate":{ + "mean":0.318303, + "std":0.033642 + }, + "llm_cost_usd_per_1k":{ + "mean":0.117339, + "std":0.012466 + }, + "error_recovery_rate":{ + "mean":0.479278, + "std":0.025486 + }, + "recoverable_caught":{ + "mean":0.884347, + "std":0.025256 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":475.666667, + "std":53.668737 + } + } + }, + "margin":{ + "tau":{ + "mean":2.062235, + "std":0.148484 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.718065, + "std":0.038251 + }, + "validation_selective_risk":{ + "mean":0.01497, + "std":0.00015 + }, + "validation_risk_upper95":{ + "mean":0.019834, + "std":5.7e-05 + }, + "test":{ + "coverage":{ + "mean":0.641697, + "std":0.041457 + }, + "selective_risk":{ + "mean":0.041548, + "std":0.004661 + }, + "accuracy_8":{ + "mean":0.872242, + "std":0.005147 + }, + "oos_recall":{ + "mean":0.542333, + "std":0.004726 + }, + "oos_recall_wilson95":[ + { + "mean":0.511352, + "std":0.004732 + }, + { + "mean":0.572991, + "std":0.004684 + } + ], + "oos_precision":{ + "mean":0.870319, + "std":0.019877 + }, + "oos_f1":{ + "mean":0.6682, + "std":0.007662 + }, + "oos_misroute_rate":{ + "mean":0.457667, + "std":0.004726 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.099333, + "std":0.012583 + }, + "llm_call_rate":{ + "mean":0.358303, + "std":0.041457 + }, + "llm_cost_usd_per_1k":{ + "mean":0.132167, + "std":0.015361 + }, + "error_recovery_rate":{ + "mean":0.470345, + "std":0.032716 + }, + "recoverable_caught":{ + "mean":0.867633, + "std":0.038685 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":467.0, + "std":59.228372 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.683124, + "std":0.056492 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.757527, + "std":0.052564 + }, + "validation_selective_risk":{ + "mean":0.015034, + "std":0.000266 + }, + "validation_risk_upper95":{ + "mean":0.019764, + "std":0.000122 + }, + "test":{ + "coverage":{ + "mean":0.670727, + "std":0.048325 + }, + "selective_risk":{ + "mean":0.037065, + "std":0.007159 + }, + "accuracy_8":{ + "mean":0.876061, + "std":0.004211 + }, + "oos_recall":{ + "mean":0.549, + "std":0.008888 + }, + "oos_recall_wilson95":[ + { + "mean":0.518033, + "std":0.00891 + }, + { + "mean":0.579592, + "std":0.008798 + } + ], + "oos_precision":{ + "mean":0.882979, + "std":0.017675 + }, + "oos_f1":{ + "mean":0.676933, + "std":0.00568 + }, + "oos_misroute_rate":{ + "mean":0.451, + "std":0.008888 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.086, + "std":0.024637 + }, + "llm_call_rate":{ + "mean":0.329273, + "std":0.048325 + }, + "llm_cost_usd_per_1k":{ + "mean":0.121417, + "std":0.017892 + }, + "error_recovery_rate":{ + "mean":0.475556, + "std":0.035922 + }, + "recoverable_caught":{ + "mean":0.877145, + "std":0.044208 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":472.333333, + "std":63.058174 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp", + "entropy", + "entropy" + ], + "hybrid":{ + "tau":{ + "mean":-1.903141, + "std":1.891129 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.922151, + "std":0.027365 + }, + "validation_selective_risk":{ + "mean":0.043143, + "std":8.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.049839, + "std":9.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.851333, + "std":0.037808 + }, + "selective_risk":{ + "mean":0.096747, + "std":0.014966 + }, + "accuracy_8":{ + "mean":0.864667, + "std":0.006299 + }, + "oos_recall":{ + "mean":0.476667, + "std":0.030827 + }, + "oos_recall_wilson95":[ + { + "mean":0.445898, + "std":0.030594 + }, + { + "mean":0.507614, + "std":0.030825 + } + ], + "oos_precision":{ + "mean":0.915021, + "std":0.012928 + }, + "oos_f1":{ + "mean":0.626394, + "std":0.027089 + }, + "oos_misroute_rate":{ + "mean":0.523333, + "std":0.030827 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.296333, + "std":0.064042 + }, + "llm_call_rate":{ + "mean":0.148667, + "std":0.037808 + }, + "llm_cost_usd_per_1k":{ + "mean":0.054748, + "std":0.013944 + }, + "error_recovery_rate":{ + "mean":0.30883, + "std":0.062966 + }, + "recoverable_caught":{ + "mean":0.568811, + "std":0.103722 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":307.333333, + "std":76.291109 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.291864, + "std":0.042827 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.913118, + "std":0.037909 + }, + "validation_selective_risk":{ + "mean":0.043098, + "std":6.1e-05 + }, + "validation_risk_upper95":{ + "mean":0.049828, + "std":0.00011 + }, + "test":{ + "coverage":{ + "mean":0.844485, + "std":0.044589 + }, + "selective_risk":{ + "mean":0.095283, + "std":0.015579 + }, + "accuracy_8":{ + "mean":0.864424, + "std":0.005881 + }, + "oos_recall":{ + "mean":0.474667, + "std":0.032316 + }, + "oos_recall_wilson95":[ + { + "mean":0.443915, + "std":0.03206 + }, + { + "mean":0.505612, + "std":0.032325 + } + ], + "oos_precision":{ + "mean":0.910697, + "std":0.01709 + }, + "oos_f1":{ + "mean":0.623593, + "std":0.028506 + }, + "oos_misroute_rate":{ + "mean":0.525333, + "std":0.032316 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.295, + "std":0.073082 + }, + "llm_call_rate":{ + "mean":0.155515, + "std":0.044589 + }, + "llm_cost_usd_per_1k":{ + "mean":0.05728, + "std":0.016462 + }, + "error_recovery_rate":{ + "mean":0.310117, + "std":0.065187 + }, + "recoverable_caught":{ + "mean":0.57113, + "std":0.107686 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":308.666667, + "std":78.589652 + } + } + }, + "entropy":{ + "tau":{ + "mean":-2.981986, + "std":0.12604 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.921075, + "std":0.026613 + }, + "validation_selective_risk":{ + "mean":0.043192, + "std":8e-05 + }, + "validation_risk_upper95":{ + "mean":0.049896, + "std":2.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.850061, + "std":0.037284 + }, + "selective_risk":{ + "mean":0.095902, + "std":0.01504 + }, + "accuracy_8":{ + "mean":0.864364, + "std":0.005994 + }, + "oos_recall":{ + "mean":0.478, + "std":0.031575 + }, + "oos_recall_wilson95":[ + { + "mean":0.447224, + "std":0.031337 + }, + { + "mean":0.508944, + "std":0.031572 + } + ], + "oos_precision":{ + "mean":0.915787, + "std":0.01421 + }, + "oos_f1":{ + "mean":0.627722, + "std":0.028103 + }, + "oos_misroute_rate":{ + "mean":0.522, + "std":0.031575 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.289667, + "std":0.064663 + }, + "llm_call_rate":{ + "mean":0.149939, + "std":0.037284 + }, + "llm_cost_usd_per_1k":{ + "mean":0.055212, + "std":0.013752 + }, + "error_recovery_rate":{ + "mean":0.309192, + "std":0.062702 + }, + "recoverable_caught":{ + "mean":0.569497, + "std":0.10331 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":307.666667, + "std":75.956128 + } + } + }, + "margin":{ + "tau":{ + "mean":0.793377, + "std":0.219877 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.866129, + "std":0.05213 + }, + "validation_selective_risk":{ + "mean":0.04295, + "std":0.000117 + }, + "validation_risk_upper95":{ + "mean":0.049866, + "std":0.000112 + }, + "test":{ + "coverage":{ + "mean":0.815273, + "std":0.047992 + }, + "selective_risk":{ + "mean":0.103807, + "std":0.009002 + }, + "accuracy_8":{ + "mean":0.85903, + "std":0.004721 + }, + "oos_recall":{ + "mean":0.461333, + "std":0.026388 + }, + "oos_recall_wilson95":[ + { + "mean":0.430672, + "std":0.026137 + }, + { + "mean":0.492291, + "std":0.026437 + } + ], + "oos_precision":{ + "mean":0.898865, + "std":0.015741 + }, + "oos_f1":{ + "mean":0.609407, + "std":0.023767 + }, + "oos_misroute_rate":{ + "mean":0.538667, + "std":0.026388 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.332, + "std":0.052431 + }, + "llm_call_rate":{ + "mean":0.184727, + "std":0.047992 + }, + "llm_cost_usd_per_1k":{ + "mean":0.06811, + "std":0.017736 + }, + "error_recovery_rate":{ + "mean":0.301756, + "std":0.058089 + }, + "recoverable_caught":{ + "mean":0.555622, + "std":0.093481 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":300.666667, + "std":73.118625 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.350626, + "std":0.059539 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.912366, + "std":0.039489 + }, + "validation_selective_risk":{ + "mean":0.043012, + "std":0.000187 + }, + "validation_risk_upper95":{ + "mean":0.049739, + "std":5.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.844061, + "std":0.043911 + }, + "selective_risk":{ + "mean":0.095808, + "std":0.014408 + }, + "accuracy_8":{ + "mean":0.864848, + "std":0.004994 + }, + "oos_recall":{ + "mean":0.474667, + "std":0.032316 + }, + "oos_recall_wilson95":[ + { + "mean":0.443915, + "std":0.03206 + }, + { + "mean":0.505612, + "std":0.032325 + } + ], + "oos_precision":{ + "mean":0.913678, + "std":0.017933 + }, + "oos_f1":{ + "mean":0.624276, + "std":0.028401 + }, + "oos_misroute_rate":{ + "mean":0.525333, + "std":0.032316 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.299, + "std":0.070086 + }, + "llm_call_rate":{ + "mean":0.155939, + "std":0.043911 + }, + "llm_cost_usd_per_1k":{ + "mean":0.057446, + "std":0.016213 + }, + "error_recovery_rate":{ + "mean":0.311406, + "std":0.063042 + }, + "recoverable_caught":{ + "mean":0.573501, + "std":0.103312 + }, + "small_errors":{ + "mean":990.333333, + "std":61.0437 + }, + "recoverable_errors":{ + "mean":537.0, + "std":45.90207 + }, + "recovered_errors":{ + "mean":310.0, + "std":77.0 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.924839, + "std":0.012296 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.18497, + "std":0.006941 + }, + "accuracy_8":{ + "mean":0.81503, + "std":0.006941 + }, + "oos_recall":{ + "mean":0.231333, + "std":0.031786 + }, + "oos_recall_wilson95":[ + { + "mean":0.206323, + "std":0.030458 + }, + { + "mean":0.2584, + "std":0.032871 + } + ], + "oos_precision":{ + "mean":0.931216, + "std":0.021158 + }, + "oos_f1":{ + "mean":0.370089, + "std":0.042115 + }, + "oos_misroute_rate":{ + "mean":0.768667, + "std":0.031786 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.768667, + "std":0.031786 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.81503, + "std":0.006941 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.914545, + "std":0.002204 + }, + "oos_recall":{ + "mean":0.612333, + "std":0.006807 + }, + "oos_recall_wilson95":[ + { + "mean":0.581762, + "std":0.006879 + }, + { + "mean":0.642045, + "std":0.006683 + } + ], + "oos_precision":{ + "mean":0.955324, + "std":0.008506 + }, + "oos_f1":{ + "mean":0.746285, + "std":0.005871 + }, + "oos_misroute_rate":{ + "mean":0.387667, + "std":0.006807 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.18497, + "std":0.006941 + }, + "llm_cost_usd_per_1k":{ + "mean":0.068122, + "std":0.002581 + }, + "error_recovery_rate":{ + "mean":0.537833, + "std":0.008889 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":547.333333, + "std":28.005952 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.008386, + "std":0.001832 + }, + "oos_auroc":{ + "mean":0.907758, + "std":0.014876 + }, + "oos_auprc":{ + "mean":0.282515, + "std":0.052391 + } + }, + "test":{ + "aurc":{ + "mean":0.034411, + "std":0.001732 + }, + "oos_auroc":{ + "mean":0.91427, + "std":0.009241 + }, + "oos_auprc":{ + "mean":0.692128, + "std":0.016569 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.008876, + "std":0.00188 + }, + "oos_auroc":{ + "mean":0.916553, + "std":0.012545 + }, + "oos_auprc":{ + "mean":0.336138, + "std":0.052541 + } + }, + "test":{ + "aurc":{ + "mean":0.034623, + "std":0.00185 + }, + "oos_auroc":{ + "mean":0.92223, + "std":0.008197 + }, + "oos_auprc":{ + "mean":0.727928, + "std":0.015805 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.008577, + "std":0.002167 + }, + "oos_auroc":{ + "mean":0.886271, + "std":0.019905 + }, + "oos_auprc":{ + "mean":0.1776, + "std":0.041251 + } + }, + "test":{ + "aurc":{ + "mean":0.035842, + "std":0.001874 + }, + "oos_auroc":{ + "mean":0.892507, + "std":0.01337 + }, + "oos_auprc":{ + "mean":0.581481, + "std":0.033677 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.00842, + "std":0.00193 + }, + "oos_auroc":{ + "mean":0.90362, + "std":0.015844 + }, + "oos_auprc":{ + "mean":0.269483, + "std":0.055242 + } + }, + "test":{ + "aurc":{ + "mean":0.034291, + "std":0.001742 + }, + "oos_auroc":{ + "mean":0.910584, + "std":0.010288 + }, + "oos_auprc":{ + "mean":0.678916, + "std":0.021765 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "margin", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":0.961059, + "std":0.490732 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.860323, + "std":0.029334 + }, + "validation_selective_risk":{ + "mean":0.01537, + "std":0.000128 + }, + "validation_risk_upper95":{ + "mean":0.01981, + "std":5.8e-05 + }, + "test":{ + "coverage":{ + "mean":0.761212, + "std":0.037625 + }, + "selective_risk":{ + "mean":0.05093, + "std":0.011219 + }, + "accuracy_8":{ + "mean":0.879515, + "std":0.003909 + }, + "oos_recall":{ + "mean":0.534667, + "std":0.014572 + }, + "oos_recall_wilson95":[ + { + "mean":0.503686, + "std":0.014571 + }, + { + "mean":0.565382, + "std":0.01446 + } + ], + "oos_precision":{ + "mean":0.905161, + "std":0.020736 + }, + "oos_f1":{ + "mean":0.672058, + "std":0.01063 + }, + "oos_misroute_rate":{ + "mean":0.465333, + "std":0.014572 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.153667, + "std":0.047248 + }, + "llm_call_rate":{ + "mean":0.238788, + "std":0.037625 + }, + "llm_cost_usd_per_1k":{ + "mean":0.087936, + "std":0.01391 + }, + "error_recovery_rate":{ + "mean":0.454739, + "std":0.022639 + }, + "recoverable_caught":{ + "mean":0.84563, + "std":0.043454 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":463.0, + "std":37.0 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.660261, + "std":0.019472 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.855914, + "std":0.026168 + }, + "validation_selective_risk":{ + "mean":0.015325, + "std":0.000121 + }, + "validation_risk_upper95":{ + "mean":0.019771, + "std":8e-05 + }, + "test":{ + "coverage":{ + "mean":0.749455, + "std":0.027151 + }, + "selective_risk":{ + "mean":0.044782, + "std":0.004159 + }, + "accuracy_8":{ + "mean":0.88097, + "std":0.004644 + }, + "oos_recall":{ + "mean":0.543333, + "std":0.004726 + }, + "oos_recall_wilson95":[ + { + "mean":0.512353, + "std":0.004732 + }, + { + "mean":0.573982, + "std":0.004683 + } + ], + "oos_precision":{ + "mean":0.902836, + "std":0.02031 + }, + "oos_f1":{ + "mean":0.678345, + "std":0.007298 + }, + "oos_misroute_rate":{ + "mean":0.456667, + "std":0.004726 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.127333, + "std":0.013429 + }, + "llm_call_rate":{ + "mean":0.250545, + "std":0.027151 + }, + "llm_cost_usd_per_1k":{ + "mean":0.092261, + "std":0.010063 + }, + "error_recovery_rate":{ + "mean":0.468821, + "std":0.015201 + }, + "recoverable_caught":{ + "mean":0.871606, + "std":0.019929 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":477.333333, + "std":33.080709 + } + } + }, + "entropy":{ + "tau":{ + "mean":-0.992119, + "std":0.039835 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.832258, + "std":0.033806 + }, + "validation_selective_risk":{ + "mean":0.015241, + "std":0.000247 + }, + "validation_risk_upper95":{ + "mean":0.019748, + "std":0.000219 + }, + "test":{ + "coverage":{ + "mean":0.721879, + "std":0.036691 + }, + "selective_risk":{ + "mean":0.037049, + "std":0.006782 + }, + "accuracy_8":{ + "mean":0.880788, + "std":0.004384 + }, + "oos_recall":{ + "mean":0.555667, + "std":0.004509 + }, + "oos_recall_wilson95":[ + { + "mean":0.524715, + "std":0.004523 + }, + { + "mean":0.586192, + "std":0.004461 + } + ], + "oos_precision":{ + "mean":0.896629, + "std":0.020494 + }, + "oos_f1":{ + "mean":0.686032, + "std":0.004508 + }, + "oos_misroute_rate":{ + "mean":0.444333, + "std":0.004509 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.087667, + "std":0.020526 + }, + "llm_call_rate":{ + "mean":0.278121, + "std":0.036691 + }, + "llm_cost_usd_per_1k":{ + "mean":0.102432, + "std":0.013577 + }, + "error_recovery_rate":{ + "mean":0.479641, + "std":0.016077 + }, + "recoverable_caught":{ + "mean":0.891801, + "std":0.026036 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":488.333333, + "std":33.857545 + } + } + }, + "margin":{ + "tau":{ + "mean":1.732785, + "std":0.207569 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.850753, + "std":0.03141 + }, + "validation_selective_risk":{ + "mean":0.015419, + "std":2e-05 + }, + "validation_risk_upper95":{ + "mean":0.019893, + "std":8.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.756121, + "std":0.036148 + }, + "selective_risk":{ + "mean":0.051303, + "std":0.0103 + }, + "accuracy_8":{ + "mean":0.879273, + "std":0.002799 + }, + "oos_recall":{ + "mean":0.530667, + "std":0.01097 + }, + "oos_recall_wilson95":[ + { + "mean":0.499681, + "std":0.010965 + }, + { + "mean":0.561417, + "std":0.01089 + } + ], + "oos_precision":{ + "mean":0.903305, + "std":0.016144 + }, + "oos_f1":{ + "mean":0.668454, + "std":0.007805 + }, + "oos_misroute_rate":{ + "mean":0.469333, + "std":0.01097 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.162, + "std":0.037802 + }, + "llm_call_rate":{ + "mean":0.243879, + "std":0.036148 + }, + "llm_cost_usd_per_1k":{ + "mean":0.089831, + "std":0.013375 + }, + "error_recovery_rate":{ + "mean":0.457454, + "std":0.021388 + }, + "recoverable_caught":{ + "mean":0.850753, + "std":0.043389 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":465.666667, + "std":34.53018 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.725211, + "std":0.041767 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.855699, + "std":0.028839 + }, + "validation_selective_risk":{ + "mean":0.015329, + "std":0.000103 + }, + "validation_risk_upper95":{ + "mean":0.019777, + "std":0.000104 + }, + "test":{ + "coverage":{ + "mean":0.751515, + "std":0.032338 + }, + "selective_risk":{ + "mean":0.04555, + "std":0.006128 + }, + "accuracy_8":{ + "mean":0.880667, + "std":0.003093 + }, + "oos_recall":{ + "mean":0.540667, + "std":0.006506 + }, + "oos_recall_wilson95":[ + { + "mean":0.509684, + "std":0.006514 + }, + { + "mean":0.571338, + "std":0.006449 + } + ], + "oos_precision":{ + "mean":0.903912, + "std":0.017547 + }, + "oos_f1":{ + "mean":0.676542, + "std":0.005257 + }, + "oos_misroute_rate":{ + "mean":0.459333, + "std":0.006506 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.131667, + "std":0.018824 + }, + "llm_call_rate":{ + "mean":0.248485, + "std":0.032338 + }, + "llm_cost_usd_per_1k":{ + "mean":0.091511, + "std":0.011977 + }, + "error_recovery_rate":{ + "mean":0.465377, + "std":0.022311 + }, + "recoverable_caught":{ + "mean":0.865139, + "std":0.034217 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":474.0, + "std":39.949969 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "margin", + "msp", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":0.461076, + "std":0.067367 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.940215, + "std":0.022466 + }, + "validation_selective_risk":{ + "mean":0.043114, + "std":7.6e-05 + }, + "validation_risk_upper95":{ + "mean":0.049739, + "std":2.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.882485, + "std":0.032791 + }, + "selective_risk":{ + "mean":0.112, + "std":0.0149 + }, + "accuracy_8":{ + "mean":0.860545, + "std":0.005894 + }, + "oos_recall":{ + "mean":0.435333, + "std":0.035388 + }, + "oos_recall_wilson95":[ + { + "mean":0.404961, + "std":0.035002 + }, + { + "mean":0.4662, + "std":0.035504 + } + ], + "oos_precision":{ + "mean":0.933486, + "std":0.012943 + }, + "oos_f1":{ + "mean":0.593046, + "std":0.030579 + }, + "oos_misroute_rate":{ + "mean":0.564667, + "std":0.035388 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.400667, + "std":0.075295 + }, + "llm_call_rate":{ + "mean":0.117515, + "std":0.032791 + }, + "llm_cost_usd_per_1k":{ + "mean":0.043229, + "std":0.012077 + }, + "error_recovery_rate":{ + "mean":0.288799, + "std":0.059844 + }, + "recoverable_caught":{ + "mean":0.536768, + "std":0.109764 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":295.0, + "std":70.171219 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.42448, + "std":0.029824 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.939785, + "std":0.022196 + }, + "validation_selective_risk":{ + "mean":0.043134, + "std":8.7e-05 + }, + "validation_risk_upper95":{ + "mean":0.049761, + "std":1.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.875455, + "std":0.027813 + }, + "selective_risk":{ + "mean":0.107374, + "std":0.014496 + }, + "accuracy_8":{ + "mean":0.863333, + "std":0.006995 + }, + "oos_recall":{ + "mean":0.452667, + "std":0.039552 + }, + "oos_recall_wilson95":[ + { + "mean":0.422121, + "std":0.039111 + }, + { + "mean":0.483575, + "std":0.039689 + } + ], + "oos_precision":{ + "mean":0.935343, + "std":0.015811 + }, + "oos_f1":{ + "mean":0.609389, + "std":0.037056 + }, + "oos_misroute_rate":{ + "mean":0.547333, + "std":0.039552 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.369333, + "std":0.071452 + }, + "llm_call_rate":{ + "mean":0.124545, + "std":0.027813 + }, + "llm_cost_usd_per_1k":{ + "mean":0.04581, + "std":0.010249 + }, + "error_recovery_rate":{ + "mean":0.30591, + "std":0.046103 + }, + "recoverable_caught":{ + "mean":0.569131, + "std":0.088067 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":311.666667, + "std":53.45403 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.539212, + "std":0.080232 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.936129, + "std":0.024687 + }, + "validation_selective_risk":{ + "mean":0.043186, + "std":0.000175 + }, + "validation_risk_upper95":{ + "mean":0.049831, + "std":0.0001 + }, + "test":{ + "coverage":{ + "mean":0.860848, + "std":0.036107 + }, + "selective_risk":{ + "mean":0.098587, + "std":0.017533 + }, + "accuracy_8":{ + "mean":0.868121, + "std":0.005308 + }, + "oos_recall":{ + "mean":0.483333, + "std":0.03453 + }, + "oos_recall_wilson95":[ + { + "mean":0.452532, + "std":0.034298 + }, + { + "mean":0.514262, + "std":0.034498 + } + ], + "oos_precision":{ + "mean":0.937801, + "std":0.015654 + }, + "oos_f1":{ + "mean":0.637282, + "std":0.02896 + }, + "oos_misroute_rate":{ + "mean":0.516667, + "std":0.03453 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.314333, + "std":0.084678 + }, + "llm_call_rate":{ + "mean":0.139152, + "std":0.036107 + }, + "llm_cost_usd_per_1k":{ + "mean":0.051179, + "std":0.01331 + }, + "error_recovery_rate":{ + "mean":0.33433, + "std":0.048096 + }, + "recoverable_caught":{ + "mean":0.621933, + "std":0.091523 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":340.666667, + "std":56.659804 + } + } + }, + "margin":{ + "tau":{ + "mean":0.621615, + "std":0.171692 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.93828, + "std":0.02315 + }, + "validation_selective_risk":{ + "mean":0.043204, + "std":0.0001 + }, + "validation_risk_upper95":{ + "mean":0.049842, + "std":0.000113 + }, + "test":{ + "coverage":{ + "mean":0.888606, + "std":0.027617 + }, + "selective_risk":{ + "mean":0.117984, + "std":0.011919 + }, + "accuracy_8":{ + "mean":0.856242, + "std":0.004798 + }, + "oos_recall":{ + "mean":0.412667, + "std":0.028729 + }, + "oos_recall_wilson95":[ + { + "mean":0.382579, + "std":0.028277 + }, + { + "mean":0.443422, + "std":0.02896 + } + ], + "oos_precision":{ + "mean":0.926737, + "std":0.017353 + }, + "oos_f1":{ + "mean":0.570658, + "std":0.028437 + }, + "oos_misroute_rate":{ + "mean":0.587333, + "std":0.028729 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.437667, + "std":0.062501 + }, + "llm_call_rate":{ + "mean":0.111394, + "std":0.027617 + }, + "llm_cost_usd_per_1k":{ + "mean":0.041001, + "std":0.010205 + }, + "error_recovery_rate":{ + "mean":0.265733, + "std":0.042308 + }, + "recoverable_caught":{ + "mean":0.494169, + "std":0.078904 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":271.0, + "std":50.477718 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.472258, + "std":0.040119 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.938065, + "std":0.023288 + }, + "validation_selective_risk":{ + "mean":0.043214, + "std":4.3e-05 + }, + "validation_risk_upper95":{ + "mean":0.049853, + "std":6.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.874909, + "std":0.029 + }, + "selective_risk":{ + "mean":0.10527, + "std":0.014724 + }, + "accuracy_8":{ + "mean":0.863576, + "std":0.006721 + }, + "oos_recall":{ + "mean":0.454667, + "std":0.037846 + }, + "oos_recall_wilson95":[ + { + "mean":0.424096, + "std":0.037437 + }, + { + "mean":0.485584, + "std":0.037966 + } + ], + "oos_precision":{ + "mean":0.93496, + "std":0.015666 + }, + "oos_f1":{ + "mean":0.611185, + "std":0.035357 + }, + "oos_misroute_rate":{ + "mean":0.545333, + "std":0.037846 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.36, + "std":0.069907 + }, + "llm_call_rate":{ + "mean":0.125091, + "std":0.029 + }, + "llm_cost_usd_per_1k":{ + "mean":0.046024, + "std":0.010696 + }, + "error_recovery_rate":{ + "mean":0.305544, + "std":0.046898 + }, + "recoverable_caught":{ + "mean":0.568428, + "std":0.089309 + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035 + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952 + }, + "recovered_errors":{ + "mean":311.333333, + "std":54.500765 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.18497, + "std":0.006941, + "values":[ + 0.177091, + 0.187636, + 0.190182 + ] + }, + "accuracy_8":{ + "mean":0.81503, + "std":0.006941, + "values":[ + 0.822909, + 0.812364, + 0.809818 + ] + }, + "oos_recall":{ + "mean":0.231333, + "std":0.031786, + "values":[ + 0.267, + 0.206, + 0.221 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.206323, + "std":0.030458, + "values":[ + 0.24051, + 0.182081, + 0.196378 + ] + }, + { + "mean":0.2584, + "std":0.032871, + "values":[ + 0.295273, + 0.232169, + 0.247758 + ] + } + ], + "oos_precision":{ + "mean":0.931216, + "std":0.021158, + "values":[ + 0.953571, + 0.911504, + 0.928571 + ] + }, + "oos_f1":{ + "mean":0.370089, + "std":0.042115, + "values":[ + 0.417188, + 0.336052, + 0.357027 + ] + }, + "oos_misroute_rate":{ + "mean":0.768667, + "std":0.031786, + "values":[ + 0.733, + 0.794, + 0.779 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.768667, + "std":0.031786, + "values":[ + 0.733, + 0.794, + 0.779 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035, + "values":[ + 974, + 1032, + 1046 + ] + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952, + "values":[ + 515, + 564, + 563 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.81503, + "std":0.006941, + "values":[ + 0.822909, + 0.812364, + 0.809818 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.914545, + "std":0.002204, + "values":[ + 0.916545, + 0.914909, + 0.912182 + ] + }, + "oos_recall":{ + "mean":0.612333, + "std":0.006807, + "values":[ + 0.62, + 0.607, + 0.61 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.581762, + "std":0.006879, + "values":[ + 0.58951, + 0.576373, + 0.579403 + ] + }, + { + "mean":0.642045, + "std":0.006683, + "values":[ + 0.649571, + 0.636808, + 0.639755 + ] + } + ], + "oos_precision":{ + "mean":0.955324, + "std":0.008506, + "values":[ + 0.958269, + 0.961965, + 0.945736 + ] + }, + "oos_f1":{ + "mean":0.746285, + "std":0.005871, + "values":[ + 0.752884, + 0.744329, + 0.741641 + ] + }, + "oos_misroute_rate":{ + "mean":0.387667, + "std":0.006807, + "values":[ + 0.38, + 0.393, + 0.39 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.18497, + "std":0.006941, + "values":[ + 0.177091, + 0.187636, + 0.190182 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.068122, + "std":0.002581, + "values":[ + 0.065194, + 0.069104, + 0.070067 + ] + }, + "error_recovery_rate":{ + "mean":0.537833, + "std":0.008889, + "values":[ + 0.528747, + 0.546512, + 0.538241 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035, + "values":[ + 974, + 1032, + 1046 + ] + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952, + "values":[ + 515, + 564, + 563 + ] + }, + "recovered_errors":{ + "mean":547.333333, + "std":28.005952, + "values":[ + 515, + 564, + 563 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.008386, + "std":0.001832, + "values":[ + 0.007045, + 0.00764, + 0.010473 + ] + }, + "oos_auroc":{ + "mean":0.907758, + "std":0.014876, + "values":[ + 0.90997, + 0.921403, + 0.8919 + ] + }, + "oos_auprc":{ + "mean":0.282515, + "std":0.052391, + "values":[ + 0.285363, + 0.333424, + 0.228758 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.034411, + "std":0.001732, + "values":[ + 0.03296, + 0.033944, + 0.036328 + ] + }, + "oos_auroc":{ + "mean":0.91427, + "std":0.009241, + "values":[ + 0.915537, + 0.922812, + 0.904461 + ] + }, + "oos_auprc":{ + "mean":0.692128, + "std":0.016569, + "values":[ + 0.698749, + 0.704363, + 0.673272 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.008876, + "std":0.00188, + "values":[ + 0.007499, + 0.00811, + 0.011018 + ] + }, + "oos_auroc":{ + "mean":0.916553, + "std":0.012545, + "values":[ + 0.91723, + 0.928747, + 0.903683 + ] + }, + "oos_auprc":{ + "mean":0.336138, + "std":0.052541, + "values":[ + 0.32534, + 0.393238, + 0.289834 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.034623, + "std":0.00185, + "values":[ + 0.033202, + 0.033953, + 0.036714 + ] + }, + "oos_auroc":{ + "mean":0.92223, + "std":0.008197, + "values":[ + 0.923118, + 0.929947, + 0.913625 + ] + }, + "oos_auprc":{ + "mean":0.727928, + "std":0.015805, + "values":[ + 0.729602, + 0.742829, + 0.711352 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.008577, + "std":0.002167, + "values":[ + 0.007014, + 0.007666, + 0.011051 + ] + }, + "oos_auroc":{ + "mean":0.886271, + "std":0.019905, + "values":[ + 0.892593, + 0.902247, + 0.863973 + ] + }, + "oos_auprc":{ + "mean":0.1776, + "std":0.041251, + "values":[ + 0.20416, + 0.198563, + 0.130077 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.035842, + "std":0.001874, + "values":[ + 0.034225, + 0.035404, + 0.037895 + ] + }, + "oos_auroc":{ + "mean":0.892507, + "std":0.01337, + "values":[ + 0.896247, + 0.903608, + 0.877665 + ] + }, + "oos_auprc":{ + "mean":0.581481, + "std":0.033677, + "values":[ + 0.596065, + 0.605407, + 0.54297 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.00842, + "std":0.00193, + "values":[ + 0.007026, + 0.007611, + 0.010623 + ] + }, + "oos_auroc":{ + "mean":0.90362, + "std":0.015844, + "values":[ + 0.907217, + 0.917357, + 0.886287 + ] + }, + "oos_auprc":{ + "mean":0.269483, + "std":0.055242, + "values":[ + 0.283412, + 0.316426, + 0.20861 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.034291, + "std":0.001742, + "values":[ + 0.03282, + 0.033838, + 0.036214 + ] + }, + "oos_auroc":{ + "mean":0.910584, + "std":0.010288, + "values":[ + 0.912781, + 0.919596, + 0.899376 + ] + }, + "oos_auprc":{ + "mean":0.678916, + "std":0.021765, + "values":[ + 0.688885, + 0.69391, + 0.653952 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "margin", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":0.961059, + "std":0.490732, + "values":[ + 0.680271, + 1.5277, + 0.675207 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.860323, + "std":0.029334, + "values":[ + 0.877097, + 0.877419, + 0.826452 + ] + }, + "validation_selective_risk":{ + "mean":0.01537, + "std":0.000128, + "values":[ + 0.015447, + 0.015441, + 0.015222 + ] + }, + "validation_risk_upper95":{ + "mean":0.01981, + "std":5.8e-05, + "values":[ + 0.019846, + 0.019839, + 0.019743 + ] + }, + "test":{ + "coverage":{ + "mean":0.761212, + "std":0.037625, + "values":[ + 0.775455, + 0.789636, + 0.718545 + ] + }, + "selective_risk":{ + "mean":0.05093, + "std":0.011219, + "values":[ + 0.05041, + 0.062399, + 0.03998 + ] + }, + "accuracy_8":{ + "mean":0.879515, + "std":0.003909, + "values":[ + 0.883818, + 0.878545, + 0.876182 + ] + }, + "oos_recall":{ + "mean":0.534667, + "std":0.014572, + "values":[ + 0.541, + 0.518, + 0.545 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.503686, + "std":0.014571, + "values":[ + 0.510016, + 0.48702, + 0.514022 + ] + }, + { + "mean":0.565382, + "std":0.01446, + "values":[ + 0.57167, + 0.548842, + 0.575634 + ] + } + ], + "oos_precision":{ + "mean":0.905161, + "std":0.020736, + "values":[ + 0.921635, + 0.911972, + 0.881877 + ] + }, + "oos_f1":{ + "mean":0.672058, + "std":0.01063, + "values":[ + 0.68179, + 0.660714, + 0.673671 + ] + }, + "oos_misroute_rate":{ + "mean":0.465333, + "std":0.014572, + "values":[ + 0.459, + 0.482, + 0.455 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.153667, + "std":0.047248, + "values":[ + 0.144, + 0.205, + 0.112 + ] + }, + "llm_call_rate":{ + "mean":0.238788, + "std":0.037625, + "values":[ + 0.224545, + 0.210364, + 0.281455 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.087936, + "std":0.01391, + "values":[ + 0.082638, + 0.077453, + 0.103716 + ] + }, + "error_recovery_rate":{ + "mean":0.454739, + "std":0.022639, + "values":[ + 0.441478, + 0.44186, + 0.48088 + ] + }, + "recoverable_caught":{ + "mean":0.84563, + "std":0.043454, + "values":[ + 0.834951, + 0.808511, + 0.893428 + ] + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035, + "values":[ + 974, + 1032, + 1046 + ] + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952, + "values":[ + 515, + 564, + 563 + ] + }, + "recovered_errors":{ + "mean":463.0, + "std":37.0, + "values":[ + 430, + 456, + 503 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "margin", + "msp", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":0.461076, + "std":0.067367, + "values":[ + 0.530907, + 0.396479, + 0.455842 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.940215, + "std":0.022466, + "values":[ + 0.95, + 0.956129, + 0.914516 + ] + }, + "validation_selective_risk":{ + "mean":0.043114, + "std":7.6e-05, + "values":[ + 0.043124, + 0.043185, + 0.043034 + ] + }, + "validation_risk_upper95":{ + "mean":0.049739, + "std":2.3e-05, + "values":[ + 0.049712, + 0.049754, + 0.04975 + ] + }, + "test":{ + "coverage":{ + "mean":0.882485, + "std":0.032791, + "values":[ + 0.903818, + 0.898909, + 0.844727 + ] + }, + "selective_risk":{ + "mean":0.112, + "std":0.0149, + "values":[ + 0.117481, + 0.123382, + 0.095136 + ] + }, + "accuracy_8":{ + "mean":0.860545, + "std":0.005894, + "values":[ + 0.859455, + 0.855273, + 0.866909 + ] + }, + "oos_recall":{ + "mean":0.435333, + "std":0.035388, + "values":[ + 0.424, + 0.407, + 0.475 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.404961, + "std":0.035002, + "values":[ + 0.393718, + 0.376963, + 0.444203 + ] + }, + { + "mean":0.4662, + "std":0.035504, + "values":[ + 0.454864, + 0.437749, + 0.505988 + ] + } + ], + "oos_precision":{ + "mean":0.933486, + "std":0.012943, + "values":[ + 0.946429, + 0.933486, + 0.920543 + ] + }, + "oos_f1":{ + "mean":0.593046, + "std":0.030579, + "values":[ + 0.585635, + 0.566852, + 0.626649 + ] + }, + "oos_misroute_rate":{ + "mean":0.564667, + "std":0.035388, + "values":[ + 0.576, + 0.593, + 0.525 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.400667, + "std":0.075295, + "values":[ + 0.438, + 0.45, + 0.314 + ] + }, + "llm_call_rate":{ + "mean":0.117515, + "std":0.032791, + "values":[ + 0.096182, + 0.101091, + 0.155273 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.043229, + "std":0.012077, + "values":[ + 0.035377, + 0.037175, + 0.057136 + ] + }, + "error_recovery_rate":{ + "mean":0.288799, + "std":0.059844, + "values":[ + 0.243326, + 0.266473, + 0.356597 + ] + }, + "recoverable_caught":{ + "mean":0.536768, + "std":0.109764, + "values":[ + 0.460194, + 0.487589, + 0.662522 + ] + }, + "small_errors":{ + "mean":1017.333333, + "std":38.175035, + "values":[ + 974, + 1032, + 1046 + ] + }, + "recoverable_errors":{ + "mean":547.333333, + "std":28.005952, + "values":[ + 515, + 564, + 563 + ] + }, + "recovered_errors":{ + "mean":295.0, + "std":70.171219, + "values":[ + 237, + 275, + 373 + ] + } + } + } + } + } + } + } + }, + "modernbert/k100":{ + "model":"modernbert", + "k":100, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "ModernBERT-base-k100-seed42", + "ModernBERT-base-k100-seed43", + "ModernBERT-base-k100-seed44" + ], + "logits_sha256":[ + "208f372ab9096fa4f95a006328c8ef766cb65144005d9df8f4bad1c391a100a9", + "58c3bd83838784f23d7e9977921b4a021797799b9a514b3c4cd449c641cd5257", + "37ebd645a97a654308f9f50cf4501b6b1c5bfed5bbd064fe591d8587a4873966" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":1.252213, + "std":0.007164, + "values":[ + 1.252711, + 1.259115, + 1.244812 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.039473, + "std":0.002033, + "values":[ + 0.038343, + 0.04182, + 0.038255 + ] + }, + "ece_8_summed":{ + "mean":0.040883, + "std":0.001155, + "values":[ + 0.039993, + 0.042189, + 0.040467 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.024498, + "std":0.002868, + "values":[ + 0.022617, + 0.027799, + 0.023077 + ] + }, + "ece_8_summed":{ + "mean":0.025542, + "std":0.00053, + "values":[ + 0.02515, + 0.026145, + 0.025331 + ] + } + } + } + }, + "selected_aggregation":"argmax", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.983441, + "std":0.002781, + "values":[ + 0.980968, + 0.986452, + 0.982903 + ] + }, + "summed":{ + "mean":0.982688, + "std":0.002266, + "values":[ + 0.980323, + 0.984839, + 0.982903 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.983441, + "std":0.002781 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.081273, + "std":0.001273 + }, + "accuracy_8":{ + "mean":0.918727, + "std":0.001273 + }, + "oos_recall":{ + "mean":0.610667, + "std":0.003786 + }, + "oos_recall_wilson95":[ + { + "mean":0.580077, + "std":0.003825 + }, + { + "mean":0.640409, + "std":0.003718 + } + ], + "oos_precision":{ + "mean":0.977584, + "std":0.001671 + }, + "oos_f1":{ + "mean":0.75174, + "std":0.003284 + }, + "oos_misroute_rate":{ + "mean":0.389333, + "std":0.003786 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.389333, + "std":0.003786 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.918727, + "std":0.001273 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.951455, + "std":0.000545 + }, + "oos_recall":{ + "mean":0.756, + "std":0.006 + }, + "oos_recall_wilson95":[ + { + "mean":0.728435, + "std":0.006197 + }, + { + "mean":0.781605, + "std":0.005757 + } + ], + "oos_precision":{ + "mean":0.986949, + "std":0.001245 + }, + "oos_f1":{ + "mean":0.856163, + "std":0.003667 + }, + "oos_misroute_rate":{ + "mean":0.244, + "std":0.006 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.081273, + "std":0.001273 + }, + "llm_cost_usd_per_1k":{ + "mean":0.029987, + "std":0.000472 + }, + "error_recovery_rate":{ + "mean":0.402533, + "std":0.015072 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":180.0, + "std":9.539392 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.001617, + "std":0.00045 + }, + "oos_auroc":{ + "mean":0.868824, + "std":0.027477 + }, + "oos_auprc":{ + "mean":0.188392, + "std":0.016463 + } + }, + "test":{ + "aurc":{ + "mean":0.009717, + "std":0.00039 + }, + "oos_auroc":{ + "mean":0.912429, + "std":0.005125 + }, + "oos_auprc":{ + "mean":0.698739, + "std":0.006223 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.001599, + "std":0.000462 + }, + "oos_auroc":{ + "mean":0.874583, + "std":0.028575 + }, + "oos_auprc":{ + "mean":0.227325, + "std":0.026379 + } + }, + "test":{ + "aurc":{ + "mean":0.009648, + "std":0.000402 + }, + "oos_auroc":{ + "mean":0.918048, + "std":0.004998 + }, + "oos_auprc":{ + "mean":0.730276, + "std":0.009479 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.001693, + "std":0.000335 + }, + "oos_auroc":{ + "mean":0.846224, + "std":0.023069 + }, + "oos_auprc":{ + "mean":0.149501, + "std":0.012854 + } + }, + "test":{ + "aurc":{ + "mean":0.010344, + "std":0.000436 + }, + "oos_auroc":{ + "mean":0.893344, + "std":0.005211 + }, + "oos_auprc":{ + "mean":0.622529, + "std":0.003516 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.001631, + "std":0.000488 + }, + "oos_auroc":{ + "mean":0.876236, + "std":0.028686 + }, + "oos_auprc":{ + "mean":0.209201, + "std":0.021882 + } + }, + "test":{ + "aurc":{ + "mean":0.009631, + "std":0.000363 + }, + "oos_auroc":{ + "mean":0.918824, + "std":0.004975 + }, + "oos_auprc":{ + "mean":0.72043, + "std":0.007304 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "entropy", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":-0.969815, + "std":2.014441 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.997204, + "std":0.003539 + }, + "validation_selective_risk":{ + "mean":0.01499, + "std":0.001254 + }, + "validation_risk_upper95":{ + "mean":0.01903, + "std":0.001407 + }, + "test":{ + "coverage":{ + "mean":0.987273, + "std":0.015872 + }, + "selective_risk":{ + "mean":0.07345, + "std":0.00857 + }, + "accuracy_8":{ + "mean":0.921273, + "std":0.00142 + }, + "oos_recall":{ + "mean":0.624333, + "std":0.013317 + }, + "oos_recall_wilson95":[ + { + "mean":0.593902, + "std":0.013481 + }, + { + "mean":0.653813, + "std":0.013051 + } + ], + "oos_precision":{ + "mean":0.977074, + "std":0.001917 + }, + "oos_f1":{ + "mean":0.761791, + "std":0.009383 + }, + "oos_misroute_rate":{ + "mean":0.375667, + "std":0.013317 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.345333, + "std":0.055293 + }, + "llm_call_rate":{ + "mean":0.012727, + "std":0.015872 + }, + "llm_cost_usd_per_1k":{ + "mean":0.004693, + "std":0.005853 + }, + "error_recovery_rate":{ + "mean":0.052317, + "std":0.061375 + }, + "recoverable_caught":{ + "mean":0.126368, + "std":0.144723 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":23.666667, + "std":28.041636 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.312232, + "std":0.131797 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.996237, + "std":0.003568 + }, + "validation_selective_risk":{ + "mean":0.014897, + "std":0.001168 + }, + "validation_risk_upper95":{ + "mean":0.018929, + "std":0.001314 + }, + "test":{ + "coverage":{ + "mean":0.980182, + "std":0.01595 + }, + "selective_risk":{ + "mean":0.069157, + "std":0.008729 + }, + "accuracy_8":{ + "mean":0.922303, + "std":0.002273 + }, + "oos_recall":{ + "mean":0.628, + "std":0.014 + }, + "oos_recall_wilson95":[ + { + "mean":0.597615, + "std":0.014168 + }, + { + "mean":0.657406, + "std":0.013725 + } + ], + "oos_precision":{ + "mean":0.977183, + "std":0.00162 + }, + "oos_f1":{ + "mean":0.76455, + "std":0.01039 + }, + "oos_misroute_rate":{ + "mean":0.372, + "std":0.014 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.321, + "std":0.054028 + }, + "llm_call_rate":{ + "mean":0.019818, + "std":0.01595 + }, + "llm_cost_usd_per_1k":{ + "mean":0.007313, + "std":0.005882 + }, + "error_recovery_rate":{ + "mean":0.075724, + "std":0.058969 + }, + "recoverable_caught":{ + "mean":0.185554, + "std":0.142361 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":34.0, + "std":26.664583 + } + } + }, + "entropy":{ + "tau":{ + "mean":-2.544619, + "std":0.659319 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.995806, + "std":0.004194 + }, + "validation_selective_risk":{ + "mean":0.014904, + "std":0.001175 + }, + "validation_risk_upper95":{ + "mean":0.018938, + "std":0.001322 + }, + "test":{ + "coverage":{ + "mean":0.977576, + "std":0.018282 + }, + "selective_risk":{ + "mean":0.068265, + "std":0.009495 + }, + "accuracy_8":{ + "mean":0.922303, + "std":0.002222 + }, + "oos_recall":{ + "mean":0.628667, + "std":0.013796 + }, + "oos_recall_wilson95":[ + { + "mean":0.59829, + "std":0.013963 + }, + { + "mean":0.658059, + "std":0.013523 + } + ], + "oos_precision":{ + "mean":0.977726, + "std":0.002056 + }, + "oos_f1":{ + "mean":0.765207, + "std":0.009983 + }, + "oos_misroute_rate":{ + "mean":0.371333, + "std":0.013796 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.315, + "std":0.060225 + }, + "llm_call_rate":{ + "mean":0.022424, + "std":0.018282 + }, + "llm_cost_usd_per_1k":{ + "mean":0.00827, + "std":0.006738 + }, + "error_recovery_rate":{ + "mean":0.086879, + "std":0.064794 + }, + "recoverable_caught":{ + "mean":0.21303, + "std":0.156227 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":39.0, + "std":29.308702 + } + } + }, + "margin":{ + "tau":{ + "mean":0.136325, + "std":0.163459 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.996129, + "std":0.005369 + }, + "validation_selective_risk":{ + "mean":0.015007, + "std":0.001267 + }, + "validation_risk_upper95":{ + "mean":0.019052, + "std":0.001423 + }, + "test":{ + "coverage":{ + "mean":0.989939, + "std":0.01251 + }, + "selective_risk":{ + "mean":0.075681, + "std":0.006033 + }, + "accuracy_8":{ + "mean":0.920121, + "std":0.000584 + }, + "oos_recall":{ + "mean":0.617667, + "std":0.007572 + }, + "oos_recall_wilson95":[ + { + "mean":0.587153, + "std":0.007654 + }, + { + "mean":0.647279, + "std":0.007432 + } + ], + "oos_precision":{ + "mean":0.974783, + "std":0.003832 + }, + "oos_f1":{ + "mean":0.756154, + "std":0.004932 + }, + "oos_misroute_rate":{ + "mean":0.382333, + "std":0.007572 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.359667, + "std":0.037581 + }, + "llm_call_rate":{ + "mean":0.010061, + "std":0.01251 + }, + "llm_cost_usd_per_1k":{ + "mean":0.003713, + "std":0.00462 + }, + "error_recovery_rate":{ + "mean":0.033179, + "std":0.038951 + }, + "recoverable_caught":{ + "mean":0.080141, + "std":0.092167 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":15.0, + "std":17.776389 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.22775, + "std":0.088487 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.996774, + "std":0.003399 + }, + "validation_selective_risk":{ + "mean":0.014997, + "std":0.001261 + }, + "validation_risk_upper95":{ + "mean":0.019039, + "std":0.001415 + }, + "test":{ + "coverage":{ + "mean":0.980121, + "std":0.015338 + }, + "selective_risk":{ + "mean":0.068982, + "std":0.008458 + }, + "accuracy_8":{ + "mean":0.922424, + "std":0.002044 + }, + "oos_recall":{ + "mean":0.629, + "std":0.014 + }, + "oos_recall_wilson95":[ + { + "mean":0.598627, + "std":0.01417 + }, + { + "mean":0.658385, + "std":0.013723 + } + ], + "oos_precision":{ + "mean":0.977727, + "std":0.002183 + }, + "oos_f1":{ + "mean":0.765457, + "std":0.010355 + }, + "oos_misroute_rate":{ + "mean":0.371, + "std":0.014 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.319, + "std":0.053226 + }, + "llm_call_rate":{ + "mean":0.019879, + "std":0.015338 + }, + "llm_cost_usd_per_1k":{ + "mean":0.007334, + "std":0.005655 + }, + "error_recovery_rate":{ + "mean":0.077208, + "std":0.058549 + }, + "recoverable_caught":{ + "mean":0.189215, + "std":0.140992 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":34.666667, + "std":26.501572 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "entropy", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":-2.199503, + "std":1.914925 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.016559, + "std":0.002781 + }, + "validation_risk_upper95":{ + "mean":0.020763, + "std":0.003093 + }, + "test":{ + "coverage":{ + "mean":0.998424, + "std":0.000639 + }, + "selective_risk":{ + "mean":0.080066, + "std":0.000842 + }, + "accuracy_8":{ + "mean":0.919273, + "std":0.001311 + }, + "oos_recall":{ + "mean":0.613667, + "std":0.002082 + }, + "oos_recall_wilson95":[ + { + "mean":0.583108, + "std":0.002104 + }, + { + "mean":0.643356, + "std":0.002044 + } + ], + "oos_precision":{ + "mean":0.977178, + "std":0.002412 + }, + "oos_f1":{ + "mean":0.75389, + "std":0.001995 + }, + "oos_misroute_rate":{ + "mean":0.386333, + "std":0.002082 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.383, + "std":0.001732 + }, + "llm_call_rate":{ + "mean":0.001576, + "std":0.000639 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000582, + "std":0.000236 + }, + "error_recovery_rate":{ + "mean":0.008189, + "std":0.00565 + }, + "recoverable_caught":{ + "mean":0.02038, + "std":0.014392 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":3.666667, + "std":2.516611 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.180987, + "std":0.017005 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.016559, + "std":0.002781 + }, + "validation_risk_upper95":{ + "mean":0.020763, + "std":0.003093 + }, + "test":{ + "coverage":{ + "mean":0.996667, + "std":0.001728 + }, + "selective_risk":{ + "mean":0.078807, + "std":0.001523 + }, + "accuracy_8":{ + "mean":0.919697, + "std":0.001389 + }, + "oos_recall":{ + "mean":0.616, + "std":0.005292 + }, + "oos_recall_wilson95":[ + { + "mean":0.585467, + "std":0.00535 + }, + { + "mean":0.645645, + "std":0.005193 + } + ], + "oos_precision":{ + "mean":0.977256, + "std":0.002484 + }, + "oos_f1":{ + "mean":0.755666, + "std":0.004505 + }, + "oos_misroute_rate":{ + "mean":0.384, + "std":0.005292 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.374667, + "std":0.009074 + }, + "llm_call_rate":{ + "mean":0.003333, + "std":0.001728 + }, + "llm_cost_usd_per_1k":{ + "mean":0.00123, + "std":0.000633 + }, + "error_recovery_rate":{ + "mean":0.01494, + "std":0.006975 + }, + "recoverable_caught":{ + "mean":0.037182, + "std":0.017793 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":6.666667, + "std":3.05505 + } + } + }, + "entropy":{ + "tau":{ + "mean":-3.23616, + "std":0.119982 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.016559, + "std":0.002781 + }, + "validation_risk_upper95":{ + "mean":0.020763, + "std":0.003093 + }, + "test":{ + "coverage":{ + "mean":0.996545, + "std":0.002691 + }, + "selective_risk":{ + "mean":0.078692, + "std":0.002518 + }, + "accuracy_8":{ + "mean":0.92, + "std":0.002025 + }, + "oos_recall":{ + "mean":0.617333, + "std":0.008386 + }, + "oos_recall_wilson95":[ + { + "mean":0.586817, + "std":0.008482 + }, + { + "mean":0.646952, + "std":0.008227 + } + ], + "oos_precision":{ + "mean":0.977814, + "std":0.00286 + }, + "oos_f1":{ + "mean":0.756825, + "std":0.006848 + }, + "oos_misroute_rate":{ + "mean":0.382667, + "std":0.008386 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.374333, + "std":0.014224 + }, + "llm_call_rate":{ + "mean":0.003455, + "std":0.002691 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001274, + "std":0.000989 + }, + "error_recovery_rate":{ + "mean":0.017993, + "std":0.012081 + }, + "recoverable_caught":{ + "mean":0.045142, + "std":0.030938 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":8.0, + "std":5.291503 + } + } + }, + "margin":{ + "tau":{ + "mean":0.005776, + "std":0.005131 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.016559, + "std":0.002781 + }, + "validation_risk_upper95":{ + "mean":0.020763, + "std":0.003093 + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.000458 + }, + "selective_risk":{ + "mean":0.081125, + "std":0.00141 + }, + "accuracy_8":{ + "mean":0.918788, + "std":0.001315 + }, + "oos_recall":{ + "mean":0.611333, + "std":0.004041 + }, + "oos_recall_wilson95":[ + { + "mean":0.580751, + "std":0.004083 + }, + { + "mean":0.641064, + "std":0.003968 + } + ], + "oos_precision":{ + "mean":0.977089, + "std":0.000984 + }, + "oos_f1":{ + "mean":0.752097, + "std":0.00324 + }, + "oos_misroute_rate":{ + "mean":0.388667, + "std":0.004041 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.388667, + "std":0.004041 + }, + "llm_call_rate":{ + "mean":0.000424, + "std":0.000458 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000157, + "std":0.000168 + }, + "error_recovery_rate":{ + "mean":0.001505, + "std":0.001303 + }, + "recoverable_caught":{ + "mean":0.00382, + "std":0.003309 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":0.666667, + "std":0.57735 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.133238, + "std":0.010698 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.016559, + "std":0.002781 + }, + "validation_risk_upper95":{ + "mean":0.020763, + "std":0.003093 + }, + "test":{ + "coverage":{ + "mean":0.996303, + "std":0.002003 + }, + "selective_risk":{ + "mean":0.078591, + "std":0.002066 + }, + "accuracy_8":{ + "mean":0.919879, + "std":0.001568 + }, + "oos_recall":{ + "mean":0.617, + "std":0.006083 + }, + "oos_recall_wilson95":[ + { + "mean":0.586478, + "std":0.006151 + }, + { + "mean":0.646626, + "std":0.005968 + } + ], + "oos_precision":{ + "mean":0.977289, + "std":0.002516 + }, + "oos_f1":{ + "mean":0.756426, + "std":0.005148 + }, + "oos_misroute_rate":{ + "mean":0.383, + "std":0.006083 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.373333, + "std":0.010693 + }, + "llm_call_rate":{ + "mean":0.003697, + "std":0.002003 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001363, + "std":0.000735 + }, + "error_recovery_rate":{ + "mean":0.017199, + "std":0.008671 + }, + "recoverable_caught":{ + "mean":0.042907, + "std":0.022266 + }, + "small_errors":{ + "mean":447.0, + "std":7.0 + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392 + }, + "recovered_errors":{ + "mean":7.666667, + "std":3.785939 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.982688, + "std":0.002266 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.088364, + "std":0.000315 + }, + "accuracy_8":{ + "mean":0.911636, + "std":0.000315 + }, + "oos_recall":{ + "mean":0.567333, + "std":0.006658 + }, + "oos_recall_wilson95":[ + { + "mean":0.536427, + "std":0.00669 + }, + { + "mean":0.597724, + "std":0.006575 + } + ], + "oos_precision":{ + "mean":0.980992, + "std":0.001499 + }, + "oos_f1":{ + "mean":0.718886, + "std":0.004984 + }, + "oos_misroute_rate":{ + "mean":0.432667, + "std":0.006658 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.432667, + "std":0.006658 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.911636, + "std":0.000315 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.947273, + "std":0.000656 + }, + "oos_recall":{ + "mean":0.732667, + "std":0.005033 + }, + "oos_recall_wilson95":[ + { + "mean":0.704385, + "std":0.005177 + }, + { + "mean":0.759167, + "std":0.004851 + } + ], + "oos_precision":{ + "mean":0.986094, + "std":0.001527 + }, + "oos_f1":{ + "mean":0.84069, + "std":0.003329 + }, + "oos_misroute_rate":{ + "mean":0.267333, + "std":0.005033 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.088364, + "std":0.000315 + }, + "llm_cost_usd_per_1k":{ + "mean":0.032601, + "std":0.000119 + }, + "error_recovery_rate":{ + "mean":0.4033, + "std":0.006063 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":196.0, + "std":2.645751 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.001439, + "std":0.000407 + }, + "oos_auroc":{ + "mean":0.909499, + "std":0.021576 + }, + "oos_auprc":{ + "mean":0.297135, + "std":0.037876 + } + }, + "test":{ + "aurc":{ + "mean":0.009474, + "std":0.000399 + }, + "oos_auroc":{ + "mean":0.940986, + "std":0.004103 + }, + "oos_auprc":{ + "mean":0.78488, + "std":0.007115 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.001446, + "std":0.000409 + }, + "oos_auroc":{ + "mean":0.910747, + "std":0.021879 + }, + "oos_auprc":{ + "mean":0.314064, + "std":0.040639 + } + }, + "test":{ + "aurc":{ + "mean":0.009521, + "std":0.00042 + }, + "oos_auroc":{ + "mean":0.942523, + "std":0.004036 + }, + "oos_auprc":{ + "mean":0.798546, + "std":0.008309 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.001406, + "std":0.000429 + }, + "oos_auroc":{ + "mean":0.904701, + "std":0.021268 + }, + "oos_auprc":{ + "mean":0.276268, + "std":0.029029 + } + }, + "test":{ + "aurc":{ + "mean":0.009854, + "std":0.000405 + }, + "oos_auroc":{ + "mean":0.936941, + "std":0.004835 + }, + "oos_auprc":{ + "mean":0.761592, + "std":0.010998 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.00149, + "std":0.000444 + }, + "oos_auroc":{ + "mean":0.913494, + "std":0.02262 + }, + "oos_auprc":{ + "mean":0.327829, + "std":0.047872 + } + }, + "test":{ + "aurc":{ + "mean":0.009705, + "std":0.000391 + }, + "oos_auroc":{ + "mean":0.944372, + "std":0.003929 + }, + "oos_auprc":{ + "mean":0.800547, + "std":0.008042 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "margin", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":0.313659, + "std":0.257516 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.997312, + "std":0.003597 + }, + "validation_selective_risk":{ + "mean":0.015526, + "std":0.000337 + }, + "validation_risk_upper95":{ + "mean":0.01963, + "std":0.000379 + }, + "test":{ + "coverage":{ + "mean":0.977818, + "std":0.01877 + }, + "selective_risk":{ + "mean":0.074299, + "std":0.010991 + }, + "accuracy_8":{ + "mean":0.916485, + "std":0.003259 + }, + "oos_recall":{ + "mean":0.591, + "std":0.020881 + }, + "oos_recall_wilson95":[ + { + "mean":0.560254, + "std":0.021023 + }, + { + "mean":0.621049, + "std":0.020578 + } + ], + "oos_precision":{ + "mean":0.979612, + "std":0.004844 + }, + "oos_f1":{ + "mean":0.737068, + "std":0.015968 + }, + "oos_misroute_rate":{ + "mean":0.409, + "std":0.020881 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.354667, + "std":0.066561 + }, + "llm_call_rate":{ + "mean":0.022182, + "std":0.01877 + }, + "llm_cost_usd_per_1k":{ + "mean":0.008175, + "std":0.006918 + }, + "error_recovery_rate":{ + "mean":0.082169, + "std":0.061169 + }, + "recoverable_caught":{ + "mean":0.204051, + "std":0.151006 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":40.0, + "std":29.816103 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.418728, + "std":0.117196 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.997097, + "std":0.003489 + }, + "validation_selective_risk":{ + "mean":0.015529, + "std":0.000342 + }, + "validation_risk_upper95":{ + "mean":0.019634, + "std":0.000384 + }, + "test":{ + "coverage":{ + "mean":0.980121, + "std":0.015184 + }, + "selective_risk":{ + "mean":0.073789, + "std":0.010099 + }, + "accuracy_8":{ + "mean":0.917818, + "std":0.002834 + }, + "oos_recall":{ + "mean":0.599667, + "std":0.019088 + }, + "oos_recall_wilson95":[ + { + "mean":0.568988, + "std":0.019242 + }, + { + "mean":0.629582, + "std":0.018787 + } + ], + "oos_precision":{ + "mean":0.979907, + "std":0.004756 + }, + "oos_f1":{ + "mean":0.743883, + "std":0.014246 + }, + "oos_misroute_rate":{ + "mean":0.400333, + "std":0.019088 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.349667, + "std":0.061695 + }, + "llm_call_rate":{ + "mean":0.019879, + "std":0.015184 + }, + "llm_cost_usd_per_1k":{ + "mean":0.00733, + "std":0.005603 + }, + "error_recovery_rate":{ + "mean":0.086302, + "std":0.053765 + }, + "recoverable_caught":{ + "mean":0.214247, + "std":0.132784 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":42.0, + "std":26.229754 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.460893, + "std":0.307002 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.996667, + "std":0.003981 + }, + "validation_selective_risk":{ + "mean":0.015536, + "std":0.000346 + }, + "validation_risk_upper95":{ + "mean":0.019642, + "std":0.000389 + }, + "test":{ + "coverage":{ + "mean":0.977091, + "std":0.018387 + }, + "selective_risk":{ + "mean":0.072672, + "std":0.011835 + }, + "accuracy_8":{ + "mean":0.917818, + "std":0.004021 + }, + "oos_recall":{ + "mean":0.601, + "std":0.026058 + }, + "oos_recall_wilson95":[ + { + "mean":0.570347, + "std":0.026265 + }, + { + "mean":0.63088, + "std":0.025651 + } + ], + "oos_precision":{ + "mean":0.980449, + "std":0.002345 + }, + "oos_f1":{ + "mean":0.744972, + "std":0.020023 + }, + "oos_misroute_rate":{ + "mean":0.399, + "std":0.026058 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.341, + "std":0.074277 + }, + "llm_call_rate":{ + "mean":0.022909, + "std":0.018387 + }, + "llm_cost_usd_per_1k":{ + "mean":0.008444, + "std":0.006777 + }, + "error_recovery_rate":{ + "mean":0.096543, + "std":0.070568 + }, + "recoverable_caught":{ + "mean":0.240059, + "std":0.17513 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":47.0, + "std":34.394767 + } + } + }, + "margin":{ + "tau":{ + "mean":0.21588, + "std":0.279559 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.997097, + "std":0.003489 + }, + "validation_selective_risk":{ + "mean":0.015529, + "std":0.000342 + }, + "validation_risk_upper95":{ + "mean":0.019634, + "std":0.000384 + }, + "test":{ + "coverage":{ + "mean":0.983091, + "std":0.020315 + }, + "selective_risk":{ + "mean":0.07807, + "std":0.011756 + }, + "accuracy_8":{ + "mean":0.914909, + "std":0.002886 + }, + "oos_recall":{ + "mean":0.582333, + "std":0.020033 + }, + "oos_recall_wilson95":[ + { + "mean":0.551525, + "std":0.020175 + }, + { + "mean":0.612511, + "std":0.019738 + } + ], + "oos_precision":{ + "mean":0.97937, + "std":0.004551 + }, + "oos_f1":{ + "mean":0.730212, + "std":0.014484 + }, + "oos_misroute_rate":{ + "mean":0.417667, + "std":0.020033 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.376333, + "std":0.070359 + }, + "llm_call_rate":{ + "mean":0.016909, + "std":0.020315 + }, + "llm_cost_usd_per_1k":{ + "mean":0.00623, + "std":0.007488 + }, + "error_recovery_rate":{ + "mean":0.061636, + "std":0.066153 + }, + "recoverable_caught":{ + "mean":0.152238, + "std":0.162272 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":30.0, + "std":32.233523 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.346698, + "std":0.10345 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.997204, + "std":0.00378 + }, + "validation_selective_risk":{ + "mean":0.015528, + "std":0.000338 + }, + "validation_risk_upper95":{ + "mean":0.019632, + "std":0.000379 + }, + "test":{ + "coverage":{ + "mean":0.979212, + "std":0.017178 + }, + "selective_risk":{ + "mean":0.075021, + "std":0.010037 + }, + "accuracy_8":{ + "mean":0.916303, + "std":0.002989 + }, + "oos_recall":{ + "mean":0.592333, + "std":0.020429 + }, + "oos_recall_wilson95":[ + { + "mean":0.561597, + "std":0.020573 + }, + { + "mean":0.622363, + "std":0.020128 + } + ], + "oos_precision":{ + "mean":0.980708, + "std":0.003102 + }, + "oos_f1":{ + "mean":0.738431, + "std":0.015805 + }, + "oos_misroute_rate":{ + "mean":0.407667, + "std":0.020429 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.355, + "std":0.064156 + }, + "llm_call_rate":{ + "mean":0.020788, + "std":0.017178 + }, + "llm_cost_usd_per_1k":{ + "mean":0.007662, + "std":0.006332 + }, + "error_recovery_rate":{ + "mean":0.080112, + "std":0.060448 + }, + "recoverable_caught":{ + "mean":0.198992, + "std":0.149336 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":39.0, + "std":29.46184 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "margin", + "margin", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":0.137719, + "std":0.195342 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.017312, + "std":0.002266 + }, + "validation_risk_upper95":{ + "mean":0.021602, + "std":0.002509 + }, + "test":{ + "coverage":{ + "mean":0.994909, + "std":0.004902 + }, + "selective_risk":{ + "mean":0.084416, + "std":0.00423 + }, + "accuracy_8":{ + "mean":0.913636, + "std":0.002405 + }, + "oos_recall":{ + "mean":0.578, + "std":0.011533 + }, + "oos_recall_wilson95":[ + { + "mean":0.547153, + "std":0.011602 + }, + { + "mean":0.60825, + "std":0.011375 + } + ], + "oos_precision":{ + "mean":0.98131, + "std":0.001877 + }, + "oos_f1":{ + "mean":0.727458, + "std":0.009493 + }, + "oos_misroute_rate":{ + "mean":0.422, + "std":0.011533 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.409333, + "std":0.023245 + }, + "llm_call_rate":{ + "mean":0.005091, + "std":0.004902 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001875, + "std":0.001805 + }, + "error_recovery_rate":{ + "mean":0.026044, + "std":0.025982 + }, + "recoverable_caught":{ + "mean":0.065219, + "std":0.065913 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":12.666667, + "std":12.66228 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.307924, + "std":0.04922 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.017312, + "std":0.002266 + }, + "validation_risk_upper95":{ + "mean":0.021602, + "std":0.002509 + }, + "test":{ + "coverage":{ + "mean":0.994606, + "std":0.004746 + }, + "selective_risk":{ + "mean":0.083773, + "std":0.003818 + }, + "accuracy_8":{ + "mean":0.914545, + "std":0.001818 + }, + "oos_recall":{ + "mean":0.583333, + "std":0.00611 + }, + "oos_recall_wilson95":[ + { + "mean":0.552516, + "std":0.006151 + }, + { + "mean":0.613513, + "std":0.006022 + } + ], + "oos_precision":{ + "mean":0.980936, + "std":0.002613 + }, + "oos_f1":{ + "mean":0.731594, + "std":0.00524 + }, + "oos_misroute_rate":{ + "mean":0.416667, + "std":0.00611 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.405333, + "std":0.019348 + }, + "llm_call_rate":{ + "mean":0.005394, + "std":0.004746 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001984, + "std":0.001749 + }, + "error_recovery_rate":{ + "mean":0.032914, + "std":0.020822 + }, + "recoverable_caught":{ + "mean":0.082105, + "std":0.053139 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":16.0, + "std":10.148892 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.741034, + "std":0.074299 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.017312, + "std":0.002266 + }, + "validation_risk_upper95":{ + "mean":0.021602, + "std":0.002509 + }, + "test":{ + "coverage":{ + "mean":0.994364, + "std":0.004468 + }, + "selective_risk":{ + "mean":0.083856, + "std":0.003407 + }, + "accuracy_8":{ + "mean":0.914364, + "std":0.002049 + }, + "oos_recall":{ + "mean":0.582333, + "std":0.010599 + }, + "oos_recall_wilson95":[ + { + "mean":0.551513, + "std":0.010666 + }, + { + "mean":0.612524, + "std":0.010451 + } + ], + "oos_precision":{ + "mean":0.980904, + "std":0.002588 + }, + "oos_f1":{ + "mean":0.730773, + "std":0.008618 + }, + "oos_misroute_rate":{ + "mean":0.417667, + "std":0.010599 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.405667, + "std":0.020033 + }, + "llm_call_rate":{ + "mean":0.005636, + "std":0.004468 + }, + "llm_cost_usd_per_1k":{ + "mean":0.002077, + "std":0.001647 + }, + "error_recovery_rate":{ + "mean":0.032888, + "std":0.024947 + }, + "recoverable_caught":{ + "mean":0.082185, + "std":0.063634 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":16.0, + "std":12.165525 + } + } + }, + "margin":{ + "tau":{ + "mean":0.019183, + "std":0.012443 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.017312, + "std":0.002266 + }, + "validation_risk_upper95":{ + "mean":0.021602, + "std":0.002509 + }, + "test":{ + "coverage":{ + "mean":0.998424, + "std":0.00129 + }, + "selective_risk":{ + "mean":0.087349, + "std":0.001075 + }, + "accuracy_8":{ + "mean":0.912, + "std":0.000656 + }, + "oos_recall":{ + "mean":0.569, + "std":0.007211 + }, + "oos_recall_wilson95":[ + { + "mean":0.538102, + "std":0.007247 + }, + { + "mean":0.59937, + "std":0.00712 + } + ], + "oos_precision":{ + "mean":0.981048, + "std":0.001462 + }, + "oos_f1":{ + "mean":0.720235, + "std":0.005389 + }, + "oos_misroute_rate":{ + "mean":0.431, + "std":0.007211 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.427, + "std":0.009 + }, + "llm_call_rate":{ + "mean":0.001576, + "std":0.00129 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000581, + "std":0.000475 + }, + "error_recovery_rate":{ + "mean":0.007563, + "std":0.008353 + }, + "recoverable_caught":{ + "mean":0.018587, + "std":0.020521 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":3.666667, + "std":4.041452 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.249366, + "std":0.027794 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.017312, + "std":0.002266 + }, + "validation_risk_upper95":{ + "mean":0.021602, + "std":0.002509 + }, + "test":{ + "coverage":{ + "mean":0.995818, + "std":0.003963 + }, + "selective_risk":{ + "mean":0.085563, + "std":0.00276 + }, + "accuracy_8":{ + "mean":0.913333, + "std":0.001766 + }, + "oos_recall":{ + "mean":0.576667, + "std":0.007506 + }, + "oos_recall_wilson95":[ + { + "mean":0.545809, + "std":0.007548 + }, + { + "mean":0.606937, + "std":0.007405 + } + ], + "oos_precision":{ + "mean":0.981275, + "std":0.001781 + }, + "oos_f1":{ + "mean":0.726415, + "std":0.006249 + }, + "oos_misroute_rate":{ + "mean":0.423333, + "std":0.007506 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.415333, + "std":0.013429 + }, + "llm_call_rate":{ + "mean":0.004182, + "std":0.003963 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001539, + "std":0.001457 + }, + "error_recovery_rate":{ + "mean":0.021933, + "std":0.020552 + }, + "recoverable_caught":{ + "mean":0.054891, + "std":0.052185 + }, + "small_errors":{ + "mean":486.0, + "std":1.732051 + }, + "recoverable_errors":{ + "mean":196.0, + "std":2.645751 + }, + "recovered_errors":{ + "mean":10.666667, + "std":10.016653 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.081273, + "std":0.001273, + "values":[ + 0.082727, + 0.080727, + 0.080364 + ] + }, + "accuracy_8":{ + "mean":0.918727, + "std":0.001273, + "values":[ + 0.917273, + 0.919273, + 0.919636 + ] + }, + "oos_recall":{ + "mean":0.610667, + "std":0.003786, + "values":[ + 0.609, + 0.608, + 0.615 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.580077, + "std":0.003825, + "values":[ + 0.578393, + 0.577383, + 0.584455 + ] + }, + { + "mean":0.640409, + "std":0.003718, + "values":[ + 0.638773, + 0.63779, + 0.644665 + ] + } + ], + "oos_precision":{ + "mean":0.977584, + "std":0.001671, + "values":[ + 0.975962, + 0.977492, + 0.979299 + ] + }, + "oos_f1":{ + "mean":0.75174, + "std":0.003284, + "values":[ + 0.75, + 0.749692, + 0.755528 + ] + }, + "oos_misroute_rate":{ + "mean":0.389333, + "std":0.003786, + "values":[ + 0.391, + 0.392, + 0.385 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.389333, + "std":0.003786, + "values":[ + 0.391, + 0.392, + 0.385 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":447.0, + "std":7.0, + "values":[ + 455, + 444, + 442 + ] + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392, + "values":[ + 191, + 174, + 175 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.918727, + "std":0.001273, + "values":[ + 0.917273, + 0.919273, + 0.919636 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.951455, + "std":0.000545, + "values":[ + 0.952, + 0.950909, + 0.951455 + ] + }, + "oos_recall":{ + "mean":0.756, + "std":0.006, + "values":[ + 0.762, + 0.75, + 0.756 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.728435, + "std":0.006197, + "values":[ + 0.734634, + 0.722239, + 0.728433 + ] + }, + { + "mean":0.781605, + "std":0.005757, + "values":[ + 0.787361, + 0.775847, + 0.781608 + ] + } + ], + "oos_precision":{ + "mean":0.986949, + "std":0.001245, + "values":[ + 0.987047, + 0.988142, + 0.985658 + ] + }, + "oos_f1":{ + "mean":0.856163, + "std":0.003667, + "values":[ + 0.860045, + 0.852757, + 0.855688 + ] + }, + "oos_misroute_rate":{ + "mean":0.244, + "std":0.006, + "values":[ + 0.238, + 0.25, + 0.244 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.081273, + "std":0.001273, + "values":[ + 0.082727, + 0.080727, + 0.080364 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.029987, + "std":0.000472, + "values":[ + 0.030527, + 0.029788, + 0.029647 + ] + }, + "error_recovery_rate":{ + "mean":0.402533, + "std":0.015072, + "values":[ + 0.41978, + 0.391892, + 0.395928 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":447.0, + "std":7.0, + "values":[ + 455, + 444, + 442 + ] + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392, + "values":[ + 191, + 174, + 175 + ] + }, + "recovered_errors":{ + "mean":180.0, + "std":9.539392, + "values":[ + 191, + 174, + 175 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.001617, + "std":0.00045, + "values":[ + 0.001271, + 0.001456, + 0.002125 + ] + }, + "oos_auroc":{ + "mean":0.868824, + "std":0.027477, + "values":[ + 0.89231, + 0.838607, + 0.875557 + ] + }, + "oos_auprc":{ + "mean":0.188392, + "std":0.016463, + "values":[ + 0.205653, + 0.186655, + 0.172866 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.009717, + "std":0.00039, + "values":[ + 0.010036, + 0.009832, + 0.009282 + ] + }, + "oos_auroc":{ + "mean":0.912429, + "std":0.005125, + "values":[ + 0.916852, + 0.906813, + 0.913623 + ] + }, + "oos_auprc":{ + "mean":0.698739, + "std":0.006223, + "values":[ + 0.704751, + 0.692324, + 0.699142 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.001599, + "std":0.000462, + "values":[ + 0.001245, + 0.00143, + 0.002121 + ] + }, + "oos_auroc":{ + "mean":0.874583, + "std":0.028575, + "values":[ + 0.899723, + 0.843507, + 0.88052 + ] + }, + "oos_auprc":{ + "mean":0.227325, + "std":0.026379, + "values":[ + 0.24883, + 0.235254, + 0.19789 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.009648, + "std":0.000402, + "values":[ + 0.010022, + 0.009699, + 0.009222 + ] + }, + "oos_auroc":{ + "mean":0.918048, + "std":0.004998, + "values":[ + 0.922767, + 0.912811, + 0.918565 + ] + }, + "oos_auprc":{ + "mean":0.730276, + "std":0.009479, + "values":[ + 0.741216, + 0.72512, + 0.724494 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.001693, + "std":0.000335, + "values":[ + 0.001562, + 0.001444, + 0.002074 + ] + }, + "oos_auroc":{ + "mean":0.846224, + "std":0.023069, + "values":[ + 0.86265, + 0.81985, + 0.856173 + ] + }, + "oos_auprc":{ + "mean":0.149501, + "std":0.012854, + "values":[ + 0.156494, + 0.134666, + 0.157341 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.010344, + "std":0.000436, + "values":[ + 0.010573, + 0.010617, + 0.009841 + ] + }, + "oos_auroc":{ + "mean":0.893344, + "std":0.005211, + "values":[ + 0.897526, + 0.887506, + 0.894999 + ] + }, + "oos_auprc":{ + "mean":0.622529, + "std":0.003516, + "values":[ + 0.626092, + 0.619062, + 0.622434 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.001631, + "std":0.000488, + "values":[ + 0.001252, + 0.001459, + 0.002183 + ] + }, + "oos_auroc":{ + "mean":0.876236, + "std":0.028686, + "values":[ + 0.90171, + 0.845163, + 0.881833 + ] + }, + "oos_auprc":{ + "mean":0.209201, + "std":0.021882, + "values":[ + 0.228672, + 0.213411, + 0.18552 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.009631, + "std":0.000363, + "values":[ + 0.009944, + 0.009717, + 0.009233 + ] + }, + "oos_auroc":{ + "mean":0.918824, + "std":0.004975, + "values":[ + 0.923417, + 0.913539, + 0.919516 + ] + }, + "oos_auprc":{ + "mean":0.72043, + "std":0.007304, + "values":[ + 0.728828, + 0.715561, + 0.7169 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "entropy", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":-0.969815, + "std":2.014441, + "values":[ + 0.296954, + -3.292709, + 0.08631 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.997204, + "std":0.003539, + "values":[ + 0.993226, + 1.0, + 0.998387 + ] + }, + "validation_selective_risk":{ + "mean":0.01499, + "std":0.001254, + "values":[ + 0.015589, + 0.013548, + 0.015832 + ] + }, + "validation_risk_upper95":{ + "mean":0.01903, + "std":0.001407, + "values":[ + 0.01971, + 0.017413, + 0.019968 + ] + }, + "test":{ + "coverage":{ + "mean":0.987273, + "std":0.015872, + "values":[ + 0.969091, + 0.998364, + 0.994364 + ] + }, + "selective_risk":{ + "mean":0.07345, + "std":0.00857, + "values":[ + 0.063602, + 0.079221, + 0.077528 + ] + }, + "accuracy_8":{ + "mean":0.921273, + "std":0.00142, + "values":[ + 0.922909, + 0.920364, + 0.920545 + ] + }, + "oos_recall":{ + "mean":0.624333, + "std":0.013317, + "values":[ + 0.639, + 0.613, + 0.621 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.593902, + "std":0.013481, + "values":[ + 0.608752, + 0.582434, + 0.590522 + ] + }, + { + "mean":0.653813, + "std":0.013051, + "values":[ + 0.668185, + 0.642701, + 0.650552 + ] + } + ], + "oos_precision":{ + "mean":0.977074, + "std":0.001917, + "values":[ + 0.975573, + 0.979233, + 0.976415 + ] + }, + "oos_f1":{ + "mean":0.761791, + "std":0.009383, + "values":[ + 0.772205, + 0.753998, + 0.759169 + ] + }, + "oos_misroute_rate":{ + "mean":0.375667, + "std":0.013317, + "values":[ + 0.361, + 0.387, + 0.379 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.345333, + "std":0.055293, + "values":[ + 0.282, + 0.384, + 0.37 + ] + }, + "llm_call_rate":{ + "mean":0.012727, + "std":0.015872, + "values":[ + 0.030909, + 0.001636, + 0.005636 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.004693, + "std":0.005853, + "values":[ + 0.011398, + 0.000607, + 0.002073 + ] + }, + "error_recovery_rate":{ + "mean":0.052317, + "std":0.061375, + "values":[ + 0.123077, + 0.013514, + 0.020362 + ] + }, + "recoverable_caught":{ + "mean":0.126368, + "std":0.144723, + "values":[ + 0.293194, + 0.034483, + 0.051429 + ] + }, + "small_errors":{ + "mean":447.0, + "std":7.0, + "values":[ + 455, + 444, + 442 + ] + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392, + "values":[ + 191, + 174, + 175 + ] + }, + "recovered_errors":{ + "mean":23.666667, + "std":28.041636, + "values":[ + 56, + 6, + 9 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "entropy", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":-2.199503, + "std":1.914925, + "values":[ + -3.317417, + -3.292709, + 0.011616 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "validation_selective_risk":{ + "mean":0.016559, + "std":0.002781, + "values":[ + 0.019032, + 0.013548, + 0.017097 + ] + }, + "validation_risk_upper95":{ + "mean":0.020763, + "std":0.003093, + "values":[ + 0.023508, + 0.017413, + 0.021369 + ] + }, + "test":{ + "coverage":{ + "mean":0.998424, + "std":0.000639, + "values":[ + 0.997818, + 0.998364, + 0.999091 + ] + }, + "selective_risk":{ + "mean":0.080066, + "std":0.000842, + "values":[ + 0.080904, + 0.079221, + 0.080073 + ] + }, + "accuracy_8":{ + "mean":0.919273, + "std":0.001311, + "values":[ + 0.917818, + 0.920364, + 0.919636 + ] + }, + "oos_recall":{ + "mean":0.613667, + "std":0.002082, + "values":[ + 0.612, + 0.613, + 0.616 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.583108, + "std":0.002104, + "values":[ + 0.581423, + 0.582434, + 0.585466 + ] + }, + { + "mean":0.643356, + "std":0.002044, + "values":[ + 0.641719, + 0.642701, + 0.645646 + ] + } + ], + "oos_precision":{ + "mean":0.977178, + "std":0.002412, + "values":[ + 0.974522, + 0.979233, + 0.977778 + ] + }, + "oos_f1":{ + "mean":0.75389, + "std":0.001995, + "values":[ + 0.751843, + 0.753998, + 0.755828 + ] + }, + "oos_misroute_rate":{ + "mean":0.386333, + "std":0.002082, + "values":[ + 0.388, + 0.387, + 0.384 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.383, + "std":0.001732, + "values":[ + 0.381, + 0.384, + 0.384 + ] + }, + "llm_call_rate":{ + "mean":0.001576, + "std":0.000639, + "values":[ + 0.002182, + 0.001636, + 0.000909 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.000582, + "std":0.000236, + "values":[ + 0.000804, + 0.000607, + 0.000334 + ] + }, + "error_recovery_rate":{ + "mean":0.008189, + "std":0.00565, + "values":[ + 0.008791, + 0.013514, + 0.002262 + ] + }, + "recoverable_caught":{ + "mean":0.02038, + "std":0.014392, + "values":[ + 0.020942, + 0.034483, + 0.005714 + ] + }, + "small_errors":{ + "mean":447.0, + "std":7.0, + "values":[ + 455, + 444, + 442 + ] + }, + "recoverable_errors":{ + "mean":180.0, + "std":9.539392, + "values":[ + 191, + 174, + 175 + ] + }, + "recovered_errors":{ + "mean":3.666667, + "std":2.516611, + "values":[ + 4, + 6, + 1 + ] + } + } + } + } + } + } + } + }, + "modernbert/k25":{ + "model":"modernbert", + "k":25, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "ModernBERT-base-k25-seed42", + "ModernBERT-base-k25-seed43", + "ModernBERT-base-k25-seed44" + ], + "logits_sha256":[ + "da2c8b7a2fcefcda47211bec4d2248e7ca5288eda05cc2572ade0945abea36ff", + "ac640bffd502ea71d7dd5b5f9cc18d7068b6b8b1b950acbf3a2fd58a3c50f1b6", + "cd9b8473e1b5b0d4597feb76bdf99d27464910b85b536eb7169193676c94ce68" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":0.961702, + "std":0.019986, + "values":[ + 0.984631, + 0.947971, + 0.952503 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.022247, + "std":0.004348, + "values":[ + 0.024075, + 0.017284, + 0.025383 + ] + }, + "ece_8_summed":{ + "mean":0.035044, + "std":0.002948, + "values":[ + 0.033703, + 0.033005, + 0.038424 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.02806, + "std":0.003865, + "values":[ + 0.025323, + 0.026376, + 0.032482 + ] + }, + "ece_8_summed":{ + "mean":0.040317, + "std":0.004833, + "values":[ + 0.035235, + 0.04086, + 0.044855 + ] + } + } + } + }, + "selected_aggregation":[ + "summed", + "argmax", + "summed" + ], + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.957419, + "std":0.002435, + "values":[ + 0.959677, + 0.954839, + 0.957742 + ] + }, + "summed":{ + "mean":0.958387, + "std":0.003177, + "values":[ + 0.960968, + 0.954839, + 0.959355 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.957419, + "std":0.002435 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.132364, + "std":0.009291 + }, + "accuracy_8":{ + "mean":0.867636, + "std":0.009291 + }, + "oos_recall":{ + "mean":0.398667, + "std":0.047501 + }, + "oos_recall_wilson95":[ + { + "mean":0.368861, + "std":0.046789 + }, + { + "mean":0.429248, + "std":0.047849 + } + ], + "oos_precision":{ + "mean":0.958738, + "std":0.013765 + }, + "oos_f1":{ + "mean":0.562176, + "std":0.047712 + }, + "oos_misroute_rate":{ + "mean":0.601333, + "std":0.047501 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.601333, + "std":0.047501 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.867636, + "std":0.009291 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.930121, + "std":0.004022 + }, + "oos_recall":{ + "mean":0.664333, + "std":0.023116 + }, + "oos_recall_wilson95":[ + { + "mean":0.634511, + "std":0.023546 + }, + { + "mean":0.692898, + "std":0.022509 + } + ], + "oos_precision":{ + "mean":0.975576, + "std":0.002164 + }, + "oos_f1":{ + "mean":0.790248, + "std":0.015566 + }, + "oos_misroute_rate":{ + "mean":0.335667, + "std":0.023116 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.132364, + "std":0.009291 + }, + "llm_cost_usd_per_1k":{ + "mean":0.048776, + "std":0.00342 + }, + "error_recovery_rate":{ + "mean":0.471746, + "std":0.007118 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":343.666667, + "std":29.022979 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.004024, + "std":0.001015 + }, + "oos_auroc":{ + "mean":0.898714, + "std":0.007123 + }, + "oos_auprc":{ + "mean":0.246874, + "std":0.025209 + } + }, + "test":{ + "aurc":{ + "mean":0.021011, + "std":0.002011 + }, + "oos_auroc":{ + "mean":0.911455, + "std":0.00279 + }, + "oos_auprc":{ + "mean":0.685688, + "std":0.002273 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.004011, + "std":0.001029 + }, + "oos_auroc":{ + "mean":0.909801, + "std":0.005703 + }, + "oos_auprc":{ + "mean":0.287332, + "std":0.030559 + } + }, + "test":{ + "aurc":{ + "mean":0.020452, + "std":0.001797 + }, + "oos_auroc":{ + "mean":0.923524, + "std":0.002898 + }, + "oos_auprc":{ + "mean":0.725399, + "std":0.002986 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.00487, + "std":0.001367 + }, + "oos_auroc":{ + "mean":0.867361, + "std":0.013258 + }, + "oos_auprc":{ + "mean":0.153517, + "std":0.00541 + } + }, + "test":{ + "aurc":{ + "mean":0.024244, + "std":0.002699 + }, + "oos_auroc":{ + "mean":0.878586, + "std":0.002702 + }, + "oos_auprc":{ + "mean":0.549232, + "std":0.004139 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.004047, + "std":0.001034 + }, + "oos_auroc":{ + "mean":0.897241, + "std":0.007678 + }, + "oos_auprc":{ + "mean":0.244357, + "std":0.025391 + } + }, + "test":{ + "aurc":{ + "mean":0.021125, + "std":0.002081 + }, + "oos_auroc":{ + "mean":0.910003, + "std":0.002098 + }, + "oos_auprc":{ + "mean":0.681629, + "std":0.003494 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp", + "entropy", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":-0.305304, + "std":1.466578 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.931505, + "std":0.002148 + }, + "validation_selective_risk":{ + "mean":0.015584, + "std":3.6e-05 + }, + "validation_risk_upper95":{ + "mean":0.019853, + "std":4.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.850061, + "std":0.0088 + }, + "selective_risk":{ + "mean":0.05331, + "std":0.005032 + }, + "accuracy_8":{ + "mean":0.895394, + "std":0.002068 + }, + "oos_recall":{ + "mean":0.534333, + "std":0.013051 + }, + "oos_recall_wilson95":[ + { + "mean":0.503351, + "std":0.013058 + }, + { + "mean":0.565053, + "std":0.012944 + } + ], + "oos_precision":{ + "mean":0.950219, + "std":0.003091 + }, + "oos_f1":{ + "mean":0.683957, + "std":0.010639 + }, + "oos_misroute_rate":{ + "mean":0.465667, + "std":0.013051 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.195333, + "std":0.027025 + }, + "llm_call_rate":{ + "mean":0.149939, + "std":0.0088 + }, + "llm_cost_usd_per_1k":{ + "mean":0.055238, + "std":0.003229 + }, + "error_recovery_rate":{ + "mean":0.321603, + "std":0.009179 + }, + "recoverable_caught":{ + "mean":0.681744, + "std":0.01719 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":234.333333, + "std":21.594752 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.516448, + "std":0.052193 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.93043, + "std":0.004007 + }, + "validation_selective_risk":{ + "mean":0.015602, + "std":6.7e-05 + }, + "validation_risk_upper95":{ + "mean":0.019877, + "std":8.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.853152, + "std":0.006172 + }, + "selective_risk":{ + "mean":0.054963, + "std":0.006225 + }, + "accuracy_8":{ + "mean":0.894909, + "std":0.002886 + }, + "oos_recall":{ + "mean":0.530667, + "std":0.015011 + }, + "oos_recall_wilson95":[ + { + "mean":0.499686, + "std":0.015019 + }, + { + "mean":0.561413, + "std":0.014889 + } + ], + "oos_precision":{ + "mean":0.952166, + "std":0.000673 + }, + "oos_f1":{ + "mean":0.681423, + "std":0.012132 + }, + "oos_misroute_rate":{ + "mean":0.469333, + "std":0.015011 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.204333, + "std":0.02635 + }, + "llm_call_rate":{ + "mean":0.146848, + "std":0.006172 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0541, + "std":0.002265 + }, + "error_recovery_rate":{ + "mean":0.316005, + "std":0.001725 + }, + "recoverable_caught":{ + "mean":0.67, + "std":0.013791 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":230.0, + "std":15.099669 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.759441, + "std":0.284108 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.92914, + "std":0.001777 + }, + "validation_selective_risk":{ + "mean":0.015623, + "std":3e-05 + }, + "validation_risk_upper95":{ + "mean":0.019904, + "std":3.8e-05 + }, + "test":{ + "coverage":{ + "mean":0.839333, + "std":0.014483 + }, + "selective_risk":{ + "mean":0.04829, + "std":0.008624 + }, + "accuracy_8":{ + "mean":0.895758, + "std":0.002338 + }, + "oos_recall":{ + "mean":0.538, + "std":0.01044 + }, + "oos_recall_wilson95":[ + { + "mean":0.507017, + "std":0.010453 + }, + { + "mean":0.568692, + "std":0.010348 + } + ], + "oos_precision":{ + "mean":0.948922, + "std":0.005362 + }, + "oos_f1":{ + "mean":0.686621, + "std":0.00754 + }, + "oos_misroute_rate":{ + "mean":0.462, + "std":0.01044 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.168333, + "std":0.033292 + }, + "llm_call_rate":{ + "mean":0.160667, + "std":0.014483 + }, + "llm_cost_usd_per_1k":{ + "mean":0.059182, + "std":0.005345 + }, + "error_recovery_rate":{ + "mean":0.341265, + "std":0.013381 + }, + "recoverable_caught":{ + "mean":0.723807, + "std":0.039608 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":248.0, + "std":8.544004 + } + } + }, + "margin":{ + "tau":{ + "mean":1.249927, + "std":0.111859 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.908817, + "std":0.008694 + }, + "validation_selective_risk":{ + "mean":0.015499, + "std":8.2e-05 + }, + "validation_risk_upper95":{ + "mean":0.019818, + "std":8.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.843333, + "std":0.009632 + }, + "selective_risk":{ + "mean":0.060479, + "std":0.005976 + }, + "accuracy_8":{ + "mean":0.894424, + "std":0.006153 + }, + "oos_recall":{ + "mean":0.533333, + "std":0.014742 + }, + "oos_recall_wilson95":[ + { + "mean":0.502352, + "std":0.014753 + }, + { + "mean":0.564059, + "std":0.014619 + } + ], + "oos_precision":{ + "mean":0.951177, + "std":0.0065 + }, + "oos_f1":{ + "mean":0.683399, + "std":0.013504 + }, + "oos_misroute_rate":{ + "mean":0.466667, + "std":0.014742 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.234, + "std":0.015524 + }, + "llm_call_rate":{ + "mean":0.156667, + "std":0.009632 + }, + "llm_cost_usd_per_1k":{ + "mean":0.057735, + "std":0.003539 + }, + "error_recovery_rate":{ + "mean":0.315187, + "std":0.014362 + }, + "recoverable_caught":{ + "mean":0.668035, + "std":0.025211 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":229.666667, + "std":22.27854 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.538175, + "std":0.039781 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.929032, + "std":0.005624 + }, + "validation_selective_risk":{ + "mean":0.015509, + "std":0.000108 + }, + "validation_risk_upper95":{ + "mean":0.019776, + "std":0.000107 + }, + "test":{ + "coverage":{ + "mean":0.853818, + "std":0.006735 + }, + "selective_risk":{ + "mean":0.0557, + "std":0.006696 + }, + "accuracy_8":{ + "mean":0.894727, + "std":0.003102 + }, + "oos_recall":{ + "mean":0.529667, + "std":0.016773 + }, + "oos_recall_wilson95":[ + { + "mean":0.498688, + "std":0.01678 + }, + { + "mean":0.560418, + "std":0.016637 + } + ], + "oos_precision":{ + "mean":0.95266, + "std":0.001406 + }, + "oos_f1":{ + "mean":0.6807, + "std":0.013517 + }, + "oos_misroute_rate":{ + "mean":0.470333, + "std":0.016773 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.208333, + "std":0.028989 + }, + "llm_call_rate":{ + "mean":0.146182, + "std":0.006735 + }, + "llm_cost_usd_per_1k":{ + "mean":0.053854, + "std":0.002473 + }, + "error_recovery_rate":{ + "mean":0.313291, + "std":0.002938 + }, + "recoverable_caught":{ + "mean":0.664268, + "std":0.015974 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":228.0, + "std":14.525839 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp", + "msp", + "entropy" + ], + "hybrid":{ + "tau":{ + "mean":-1.223336, + "std":2.323956 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.999247, + "std":0.001304 + }, + "validation_selective_risk":{ + "mean":0.041968, + "std":0.001521 + }, + "validation_risk_upper95":{ + "mean":0.048304, + "std":0.001626 + }, + "test":{ + "coverage":{ + "mean":0.99503, + "std":0.003592 + }, + "selective_risk":{ + "mean":0.128699, + "std":0.009209 + }, + "accuracy_8":{ + "mean":0.869697, + "std":0.008531 + }, + "oos_recall":{ + "mean":0.409333, + "std":0.046479 + }, + "oos_recall_wilson95":[ + { + "mean":0.379353, + "std":0.045852 + }, + { + "mean":0.440008, + "std":0.046751 + } + ], + "oos_precision":{ + "mean":0.959423, + "std":0.013204 + }, + "oos_f1":{ + "mean":0.572828, + "std":0.045066 + }, + "oos_misroute_rate":{ + "mean":0.590667, + "std":0.046479 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.579, + "std":0.051029 + }, + "llm_call_rate":{ + "mean":0.00497, + "std":0.003592 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001829, + "std":0.001323 + }, + "error_recovery_rate":{ + "mean":0.016323, + "std":0.009528 + }, + "recoverable_caught":{ + "mean":0.034581, + "std":0.019966 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":12.0, + "std":7.549834 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.103284, + "std":0.026182 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.999247, + "std":0.001304 + }, + "validation_selective_risk":{ + "mean":0.041968, + "std":0.001521 + }, + "validation_risk_upper95":{ + "mean":0.048304, + "std":0.001626 + }, + "test":{ + "coverage":{ + "mean":0.995333, + "std":0.004087 + }, + "selective_risk":{ + "mean":0.128962, + "std":0.009404 + }, + "accuracy_8":{ + "mean":0.869455, + "std":0.008659 + }, + "oos_recall":{ + "mean":0.408333, + "std":0.047385 + }, + "oos_recall_wilson95":[ + { + "mean":0.378372, + "std":0.04674 + }, + { + "mean":0.438997, + "std":0.047667 + } + ], + "oos_precision":{ + "mean":0.958523, + "std":0.011949 + }, + "oos_f1":{ + "mean":0.571653, + "std":0.046054 + }, + "oos_misroute_rate":{ + "mean":0.591667, + "std":0.047385 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.580333, + "std":0.052548 + }, + "llm_call_rate":{ + "mean":0.004667, + "std":0.004087 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001718, + "std":0.001504 + }, + "error_recovery_rate":{ + "mean":0.014514, + "std":0.012344 + }, + "recoverable_caught":{ + "mean":0.030771, + "std":0.025964 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":10.666667, + "std":9.504385 + } + } + }, + "entropy":{ + "tau":{ + "mean":-3.778412, + "std":0.111415 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.99914, + "std":0.00149 + }, + "validation_selective_risk":{ + "mean":0.041973, + "std":0.001528 + }, + "validation_risk_upper95":{ + "mean":0.048309, + "std":0.001633 + }, + "test":{ + "coverage":{ + "mean":0.99703, + "std":0.001821 + }, + "selective_risk":{ + "mean":0.130268, + "std":0.009014 + }, + "accuracy_8":{ + "mean":0.86903, + "std":0.009007 + }, + "oos_recall":{ + "mean":0.405, + "std":0.046936 + }, + "oos_recall_wilson95":[ + { + "mean":0.375089, + "std":0.046275 + }, + { + "mean":0.435638, + "std":0.047238 + } + ], + "oos_precision":{ + "mean":0.96172, + "std":0.013201 + }, + "oos_f1":{ + "mean":0.56901, + "std":0.046618 + }, + "oos_misroute_rate":{ + "mean":0.595, + "std":0.046936 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.589, + "std":0.047634 + }, + "llm_call_rate":{ + "mean":0.00297, + "std":0.001821 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001091, + "std":0.000668 + }, + "error_recovery_rate":{ + "mean":0.010961, + "std":0.003799 + }, + "recoverable_caught":{ + "mean":0.023246, + "std":0.008035 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":8.0, + "std":3.0 + } + } + }, + "margin":{ + "tau":{ + "mean":0.017375, + "std":0.018385 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.99828, + "std":0.00298 + }, + "validation_selective_risk":{ + "mean":0.04201, + "std":0.001578 + }, + "validation_risk_upper95":{ + "mean":0.048352, + "std":0.001691 + }, + "test":{ + "coverage":{ + "mean":0.996485, + "std":0.004365 + }, + "selective_risk":{ + "mean":0.130594, + "std":0.007898 + }, + "accuracy_8":{ + "mean":0.868667, + "std":0.008372 + }, + "oos_recall":{ + "mean":0.403333, + "std":0.043016 + }, + "oos_recall_wilson95":[ + { + "mean":0.373433, + "std":0.042396 + }, + { + "mean":0.433973, + "std":0.043305 + } + ], + "oos_precision":{ + "mean":0.95938, + "std":0.012751 + }, + "oos_f1":{ + "mean":0.567086, + "std":0.042715 + }, + "oos_misroute_rate":{ + "mean":0.596667, + "std":0.043016 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.591, + "std":0.040286 + }, + "llm_call_rate":{ + "mean":0.003515, + "std":0.004365 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001297, + "std":0.001611 + }, + "error_recovery_rate":{ + "mean":0.008764, + "std":0.01146 + }, + "recoverable_caught":{ + "mean":0.018446, + "std":0.023989 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":6.666667, + "std":8.962886 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.10886, + "std":0.02711 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.999247, + "std":0.001304 + }, + "validation_selective_risk":{ + "mean":0.041968, + "std":0.001521 + }, + "validation_risk_upper95":{ + "mean":0.048304, + "std":0.001626 + }, + "test":{ + "coverage":{ + "mean":0.995394, + "std":0.00401 + }, + "selective_risk":{ + "mean":0.129014, + "std":0.009455 + }, + "accuracy_8":{ + "mean":0.869455, + "std":0.008659 + }, + "oos_recall":{ + "mean":0.408333, + "std":0.047385 + }, + "oos_recall_wilson95":[ + { + "mean":0.378372, + "std":0.04674 + }, + { + "mean":0.438997, + "std":0.047667 + } + ], + "oos_precision":{ + "mean":0.958523, + "std":0.011949 + }, + "oos_f1":{ + "mean":0.571653, + "std":0.046054 + }, + "oos_misroute_rate":{ + "mean":0.591667, + "std":0.047385 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.580667, + "std":0.052729 + }, + "llm_call_rate":{ + "mean":0.004606, + "std":0.00401 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001696, + "std":0.001475 + }, + "error_recovery_rate":{ + "mean":0.014514, + "std":0.012344 + }, + "recoverable_caught":{ + "mean":0.030771, + "std":0.025964 + }, + "small_errors":{ + "mean":728.0, + "std":51.097945 + }, + "recoverable_errors":{ + "mean":343.666667, + "std":29.022979 + }, + "recovered_errors":{ + "mean":10.666667, + "std":9.504385 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.958387, + "std":0.003177 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.140303, + "std":0.009282 + }, + "accuracy_8":{ + "mean":0.859697, + "std":0.009282 + }, + "oos_recall":{ + "mean":0.346333, + "std":0.042852 + }, + "oos_recall_wilson95":[ + { + "mean":0.317568, + "std":0.041884 + }, + { + "mean":0.376274, + "std":0.043493 + } + ], + "oos_precision":{ + "mean":0.964455, + "std":0.012813 + }, + "oos_f1":{ + "mean":0.508717, + "std":0.046445 + }, + "oos_misroute_rate":{ + "mean":0.653667, + "std":0.042852 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.653667, + "std":0.042852 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.859697, + "std":0.009282 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.926303, + "std":0.00283 + }, + "oos_recall":{ + "mean":0.642333, + "std":0.016197 + }, + "oos_recall_wilson95":[ + { + "mean":0.612145, + "std":0.016442 + }, + { + "mean":0.671433, + "std":0.015828 + } + ], + "oos_precision":{ + "mean":0.975282, + "std":0.004092 + }, + "oos_f1":{ + "mean":0.774437, + "std":0.010405 + }, + "oos_misroute_rate":{ + "mean":0.357667, + "std":0.016197 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.140303, + "std":0.009282 + }, + "llm_cost_usd_per_1k":{ + "mean":0.051694, + "std":0.003414 + }, + "error_recovery_rate":{ + "mean":0.474084, + "std":0.014822 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":366.333333, + "std":35.697806 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.003233, + "std":0.000871 + }, + "oos_auroc":{ + "mean":0.936188, + "std":0.006669 + }, + "oos_auprc":{ + "mean":0.366123, + "std":0.01667 + } + }, + "test":{ + "aurc":{ + "mean":0.020178, + "std":0.002066 + }, + "oos_auroc":{ + "mean":0.941852, + "std":0.002103 + }, + "oos_auprc":{ + "mean":0.776397, + "std":0.005438 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.003344, + "std":0.000935 + }, + "oos_auroc":{ + "mean":0.939197, + "std":0.006912 + }, + "oos_auprc":{ + "mean":0.401722, + "std":0.010339 + } + }, + "test":{ + "aurc":{ + "mean":0.02009, + "std":0.001993 + }, + "oos_auroc":{ + "mean":0.946223, + "std":0.002316 + }, + "oos_auprc":{ + "mean":0.799242, + "std":0.008926 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.003207, + "std":0.000723 + }, + "oos_auroc":{ + "mean":0.927512, + "std":0.006909 + }, + "oos_auprc":{ + "mean":0.310618, + "std":0.009685 + } + }, + "test":{ + "aurc":{ + "mean":0.021031, + "std":0.002329 + }, + "oos_auroc":{ + "mean":0.930886, + "std":0.001631 + }, + "oos_auprc":{ + "mean":0.706424, + "std":0.001176 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.003228, + "std":0.000868 + }, + "oos_auroc":{ + "mean":0.93539, + "std":0.006829 + }, + "oos_auprc":{ + "mean":0.361177, + "std":0.020886 + } + }, + "test":{ + "aurc":{ + "mean":0.020121, + "std":0.002033 + }, + "oos_auroc":{ + "mean":0.941052, + "std":0.001814 + }, + "oos_auprc":{ + "mean":0.772924, + "std":0.00613 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.657306, + "std":0.059267 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.949355, + "std":0.001707 + }, + "validation_selective_risk":{ + "mean":0.01563, + "std":2.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.019861, + "std":3.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.868485, + "std":0.009261 + }, + "selective_risk":{ + "mean":0.058806, + "std":0.012849 + }, + "accuracy_8":{ + "mean":0.895394, + "std":0.00626 + }, + "oos_recall":{ + "mean":0.52, + "std":0.021 + }, + "oos_recall_wilson95":[ + { + "mean":0.489035, + "std":0.020971 + }, + { + "mean":0.550812, + "std":0.020868 + } + ], + "oos_precision":{ + "mean":0.958168, + "std":0.002534 + }, + "oos_f1":{ + "mean":0.673995, + "std":0.018271 + }, + "oos_misroute_rate":{ + "mean":0.48, + "std":0.021 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.234333, + "std":0.047543 + }, + "llm_call_rate":{ + "mean":0.131515, + "std":0.009261 + }, + "llm_cost_usd_per_1k":{ + "mean":0.048402, + "std":0.003412 + }, + "error_recovery_rate":{ + "mean":0.338122, + "std":0.017827 + }, + "recoverable_caught":{ + "mean":0.714441, + "std":0.059123 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":260.333333, + "std":5.686241 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.63691, + "std":0.067756 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.949247, + "std":0.001834 + }, + "validation_selective_risk":{ + "mean":0.015632, + "std":3e-05 + }, + "validation_risk_upper95":{ + "mean":0.019863, + "std":3.8e-05 + }, + "test":{ + "coverage":{ + "mean":0.867515, + "std":0.009886 + }, + "selective_risk":{ + "mean":0.058235, + "std":0.013034 + }, + "accuracy_8":{ + "mean":0.895333, + "std":0.006231 + }, + "oos_recall":{ + "mean":0.52, + "std":0.021 + }, + "oos_recall_wilson95":[ + { + "mean":0.489035, + "std":0.020971 + }, + { + "mean":0.550812, + "std":0.020868 + } + ], + "oos_precision":{ + "mean":0.959369, + "std":0.001575 + }, + "oos_f1":{ + "mean":0.674285, + "std":0.018056 + }, + "oos_misroute_rate":{ + "mean":0.48, + "std":0.021 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.231333, + "std":0.048542 + }, + "llm_call_rate":{ + "mean":0.132485, + "std":0.009886 + }, + "llm_cost_usd_per_1k":{ + "mean":0.048757, + "std":0.003643 + }, + "error_recovery_rate":{ + "mean":0.339392, + "std":0.017008 + }, + "recoverable_caught":{ + "mean":0.717092, + "std":0.057624 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":261.333333, + "std":5.507571 + } + } + }, + "entropy":{ + "tau":{ + "mean":-0.992699, + "std":0.136758 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.942151, + "std":0.002818 + }, + "validation_selective_risk":{ + "mean":0.015635, + "std":0.000154 + }, + "validation_risk_upper95":{ + "mean":0.019884, + "std":0.000165 + }, + "test":{ + "coverage":{ + "mean":0.84903, + "std":0.01019 + }, + "selective_risk":{ + "mean":0.049917, + "std":0.009678 + }, + "accuracy_8":{ + "mean":0.896242, + "std":0.004319 + }, + "oos_recall":{ + "mean":0.531333, + "std":0.012342 + }, + "oos_recall_wilson95":[ + { + "mean":0.500349, + "std":0.012346 + }, + { + "mean":0.562078, + "std":0.012244 + } + ], + "oos_precision":{ + "mean":0.953357, + "std":0.006382 + }, + "oos_f1":{ + "mean":0.682311, + "std":0.0106 + }, + "oos_misroute_rate":{ + "mean":0.468667, + "std":0.012342 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.182667, + "std":0.034646 + }, + "llm_call_rate":{ + "mean":0.15097, + "std":0.01019 + }, + "llm_cost_usd_per_1k":{ + "mean":0.055565, + "std":0.003753 + }, + "error_recovery_rate":{ + "mean":0.358889, + "std":0.009458 + }, + "recoverable_caught":{ + "mean":0.757827, + "std":0.041133 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":276.666667, + "std":12.66228 + } + } + }, + "margin":{ + "tau":{ + "mean":1.421763, + "std":0.225615 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.943763, + "std":0.005372 + }, + "validation_selective_risk":{ + "mean":0.015609, + "std":0.000113 + }, + "validation_risk_upper95":{ + "mean":0.01985, + "std":0.000115 + }, + "test":{ + "coverage":{ + "mean":0.866606, + "std":0.005976 + }, + "selective_risk":{ + "mean":0.059131, + "std":0.010846 + }, + "accuracy_8":{ + "mean":0.895333, + "std":0.006196 + }, + "oos_recall":{ + "mean":0.52, + "std":0.020298 + }, + "oos_recall_wilson95":[ + { + "mean":0.489034, + "std":0.020263 + }, + { + "mean":0.550813, + "std":0.020177 + } + ], + "oos_precision":{ + "mean":0.961096, + "std":0.004271 + }, + "oos_f1":{ + "mean":0.674738, + "std":0.01811 + }, + "oos_misroute_rate":{ + "mean":0.48, + "std":0.020298 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.242, + "std":0.040841 + }, + "llm_call_rate":{ + "mean":0.133394, + "std":0.005976 + }, + "llm_cost_usd_per_1k":{ + "mean":0.049093, + "std":0.002214 + }, + "error_recovery_rate":{ + "mean":0.3379, + "std":0.014909 + }, + "recoverable_caught":{ + "mean":0.713393, + "std":0.043077 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":260.666667, + "std":18.823744 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.657306, + "std":0.059267 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.949355, + "std":0.001707 + }, + "validation_selective_risk":{ + "mean":0.01563, + "std":2.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.019861, + "std":3.6e-05 + }, + "test":{ + "coverage":{ + "mean":0.868485, + "std":0.009261 + }, + "selective_risk":{ + "mean":0.058806, + "std":0.012849 + }, + "accuracy_8":{ + "mean":0.895394, + "std":0.00626 + }, + "oos_recall":{ + "mean":0.52, + "std":0.021 + }, + "oos_recall_wilson95":[ + { + "mean":0.489035, + "std":0.020971 + }, + { + "mean":0.550812, + "std":0.020868 + } + ], + "oos_precision":{ + "mean":0.958168, + "std":0.002534 + }, + "oos_f1":{ + "mean":0.673995, + "std":0.018271 + }, + "oos_misroute_rate":{ + "mean":0.48, + "std":0.021 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.234333, + "std":0.047543 + }, + "llm_call_rate":{ + "mean":0.131515, + "std":0.009261 + }, + "llm_cost_usd_per_1k":{ + "mean":0.048402, + "std":0.003412 + }, + "error_recovery_rate":{ + "mean":0.338122, + "std":0.017827 + }, + "recoverable_caught":{ + "mean":0.714441, + "std":0.059123 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":260.333333, + "std":5.686241 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp_t", + "margin", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":0.084543, + "std":0.113018 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.999247, + "std":0.001304 + }, + "validation_selective_risk":{ + "mean":0.041, + "std":0.002168 + }, + "validation_risk_upper95":{ + "mean":0.047271, + "std":0.002317 + }, + "test":{ + "coverage":{ + "mean":0.998242, + "std":0.002273 + }, + "selective_risk":{ + "mean":0.139163, + "std":0.008204 + }, + "accuracy_8":{ + "mean":0.860545, + "std":0.008733 + }, + "oos_recall":{ + "mean":0.350333, + "std":0.041016 + }, + "oos_recall_wilson95":[ + { + "mean":0.321466, + "std":0.040123 + }, + { + "mean":0.380346, + "std":0.041595 + } + ], + "oos_precision":{ + "mean":0.965923, + "std":0.010692 + }, + "oos_f1":{ + "mean":0.513296, + "std":0.043778 + }, + "oos_misroute_rate":{ + "mean":0.649667, + "std":0.041016 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.646667, + "std":0.037899 + }, + "llm_call_rate":{ + "mean":0.001758, + "std":0.002273 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000644, + "std":0.000833 + }, + "error_recovery_rate":{ + "mean":0.005892, + "std":0.006076 + }, + "recoverable_caught":{ + "mean":0.012255, + "std":0.012378 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":4.666667, + "std":5.033223 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.2365, + "std":0.031262 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.999032, + "std":0.001676 + }, + "validation_selective_risk":{ + "mean":0.04101, + "std":0.002183 + }, + "validation_risk_upper95":{ + "mean":0.047282, + "std":0.002334 + }, + "test":{ + "coverage":{ + "mean":0.995515, + "std":0.005299 + }, + "selective_risk":{ + "mean":0.137363, + "std":0.006305 + }, + "accuracy_8":{ + "mean":0.862303, + "std":0.006843 + }, + "oos_recall":{ + "mean":0.36, + "std":0.032047 + }, + "oos_recall_wilson95":[ + { + "mean":0.330884, + "std":0.031388 + }, + { + "mean":0.390187, + "std":0.03246 + } + ], + "oos_precision":{ + "mean":0.966107, + "std":0.010886 + }, + "oos_f1":{ + "mean":0.523988, + "std":0.033558 + }, + "oos_misroute_rate":{ + "mean":0.64, + "std":0.032047 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.635333, + "std":0.028746 + }, + "llm_call_rate":{ + "mean":0.004485, + "std":0.005299 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001647, + "std":0.001947 + }, + "error_recovery_rate":{ + "mean":0.018664, + "std":0.018679 + }, + "recoverable_caught":{ + "mean":0.038598, + "std":0.037652 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":15.0, + "std":15.716234 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.91012, + "std":0.060319 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.999247, + "std":0.001304 + }, + "validation_selective_risk":{ + "mean":0.041, + "std":0.002168 + }, + "validation_risk_upper95":{ + "mean":0.047271, + "std":0.002317 + }, + "test":{ + "coverage":{ + "mean":0.993333, + "std":0.007147 + }, + "selective_risk":{ + "mean":0.136138, + "std":0.006204 + }, + "accuracy_8":{ + "mean":0.863091, + "std":0.006909 + }, + "oos_recall":{ + "mean":0.363667, + "std":0.035218 + }, + "oos_recall_wilson95":[ + { + "mean":0.334482, + "std":0.034513 + }, + { + "mean":0.393895, + "std":0.035654 + } + ], + "oos_precision":{ + "mean":0.966499, + "std":0.010451 + }, + "oos_f1":{ + "mean":0.527807, + "std":0.036546 + }, + "oos_misroute_rate":{ + "mean":0.636333, + "std":0.035218 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.628333, + "std":0.033247 + }, + "llm_call_rate":{ + "mean":0.006667, + "std":0.007147 + }, + "llm_cost_usd_per_1k":{ + "mean":0.00245, + "std":0.002624 + }, + "error_recovery_rate":{ + "mean":0.025969, + "std":0.023012 + }, + "recoverable_caught":{ + "mean":0.053936, + "std":0.046226 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":20.666667, + "std":19.502137 + } + } + }, + "margin":{ + "tau":{ + "mean":0.013917, + "std":0.018036 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.999247, + "std":0.001304 + }, + "validation_selective_risk":{ + "mean":0.041, + "std":0.002168 + }, + "validation_risk_upper95":{ + "mean":0.047271, + "std":0.002317 + }, + "test":{ + "coverage":{ + "mean":0.998424, + "std":0.002414 + }, + "selective_risk":{ + "mean":0.139321, + "std":0.007958 + }, + "accuracy_8":{ + "mean":0.860364, + "std":0.008456 + }, + "oos_recall":{ + "mean":0.349333, + "std":0.039311 + }, + "oos_recall_wilson95":[ + { + "mean":0.320483, + "std":0.038447 + }, + { + "mean":0.379336, + "std":0.039874 + } + ], + "oos_precision":{ + "mean":0.965839, + "std":0.010693 + }, + "oos_f1":{ + "mean":0.512275, + "std":0.042051 + }, + "oos_misroute_rate":{ + "mean":0.650667, + "std":0.039311 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.647667, + "std":0.036171 + }, + "llm_call_rate":{ + "mean":0.001576, + "std":0.002414 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000578, + "std":0.000884 + }, + "error_recovery_rate":{ + "mean":0.004507, + "std":0.006643 + }, + "recoverable_caught":{ + "mean":0.009252, + "std":0.013508 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":3.666667, + "std":5.507571 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.239143, + "std":0.028079 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.99914, + "std":0.00149 + }, + "validation_selective_risk":{ + "mean":0.041005, + "std":0.002175 + }, + "validation_risk_upper95":{ + "mean":0.047276, + "std":0.002326 + }, + "test":{ + "coverage":{ + "mean":0.995818, + "std":0.004502 + }, + "selective_risk":{ + "mean":0.137381, + "std":0.006538 + }, + "accuracy_8":{ + "mean":0.862182, + "std":0.007184 + }, + "oos_recall":{ + "mean":0.359333, + "std":0.032716 + }, + "oos_recall_wilson95":[ + { + "mean":0.330234, + "std":0.03204 + }, + { + "mean":0.389509, + "std":0.033141 + } + ], + "oos_precision":{ + "mean":0.965982, + "std":0.011203 + }, + "oos_f1":{ + "mean":0.523252, + "std":0.034446 + }, + "oos_misroute_rate":{ + "mean":0.640667, + "std":0.032716 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.635667, + "std":0.028449 + }, + "llm_call_rate":{ + "mean":0.004182, + "std":0.004502 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001537, + "std":0.001653 + }, + "error_recovery_rate":{ + "mean":0.017913, + "std":0.015424 + }, + "recoverable_caught":{ + "mean":0.037139, + "std":0.031 + }, + "small_errors":{ + "mean":771.666667, + "std":51.052261 + }, + "recoverable_errors":{ + "mean":366.333333, + "std":35.697806 + }, + "recovered_errors":{ + "mean":14.333333, + "std":13.051181 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.137273, + "std":0.005216, + "values":[ + 0.131273, + 0.140727, + 0.139818 + ] + }, + "accuracy_8":{ + "mean":0.862727, + "std":0.005216, + "values":[ + 0.868727, + 0.859273, + 0.860182 + ] + }, + "oos_recall":{ + "mean":0.364333, + "std":0.030006, + "values":[ + 0.394, + 0.365, + 0.334 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.335119, + "std":0.029366, + "values":[ + 0.364175, + 0.33573, + 0.305452 + ] + }, + { + "mean":0.394586, + "std":0.030415, + "values":[ + 0.424636, + 0.395303, + 0.363819 + ] + } + ], + "oos_precision":{ + "mean":0.961816, + "std":0.01706, + "values":[ + 0.965686, + 0.943152, + 0.976608 + ] + }, + "oos_f1":{ + "mean":0.527913, + "std":0.030978, + "values":[ + 0.559659, + 0.526316, + 0.497765 + ] + }, + "oos_misroute_rate":{ + "mean":0.635667, + "std":0.030006, + "values":[ + 0.606, + 0.635, + 0.666 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.635667, + "std":0.030006, + "values":[ + 0.606, + 0.635, + 0.666 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":755.0, + "std":28.687977, + "values":[ + 722, + 774, + 769 + ] + }, + "recoverable_errors":{ + "mean":354.666667, + "std":19.087518, + "values":[ + 333, + 369, + 362 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.862727, + "std":0.005216, + "values":[ + 0.868727, + 0.859273, + 0.860182 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.927212, + "std":0.001794, + "values":[ + 0.929273, + 0.926364, + 0.926 + ] + }, + "oos_recall":{ + "mean":0.647667, + "std":0.01464, + "values":[ + 0.661, + 0.65, + 0.632 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.617555, + "std":0.01486, + "values":[ + 0.631094, + 0.619914, + 0.601657 + ] + }, + { + "mean":0.676648, + "std":0.014309, + "values":[ + 0.689674, + 0.678938, + 0.661333 + ] + } + ], + "oos_precision":{ + "mean":0.974979, + "std":0.003944, + "values":[ + 0.970631, + 0.975976, + 0.978328 + ] + }, + "oos_f1":{ + "mean":0.778224, + "std":0.009432, + "values":[ + 0.786437, + 0.780312, + 0.767922 + ] + }, + "oos_misroute_rate":{ + "mean":0.352333, + "std":0.01464, + "values":[ + 0.339, + 0.35, + 0.368 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.137273, + "std":0.005216, + "values":[ + 0.131273, + 0.140727, + 0.139818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.05058, + "std":0.001922, + "values":[ + 0.048369, + 0.05185, + 0.051522 + ] + }, + "error_recovery_rate":{ + "mean":0.469568, + "std":0.007829, + "values":[ + 0.461219, + 0.476744, + 0.470741 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":755.0, + "std":28.687977, + "values":[ + 722, + 774, + 769 + ] + }, + "recoverable_errors":{ + "mean":354.666667, + "std":19.087518, + "values":[ + 333, + 369, + 362 + ] + }, + "recovered_errors":{ + "mean":354.666667, + "std":19.087518, + "values":[ + 333, + 369, + 362 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.003551, + "std":0.001421, + "values":[ + 0.002786, + 0.00519, + 0.002676 + ] + }, + "oos_auroc":{ + "mean":0.923623, + "std":0.028383, + "values":[ + 0.939083, + 0.890867, + 0.94092 + ] + }, + "oos_auprc":{ + "mean":0.337494, + "std":0.05931, + "values":[ + 0.385362, + 0.271144, + 0.355976 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.020372, + "std":0.002389, + "values":[ + 0.018415, + 0.023034, + 0.019668 + ] + }, + "oos_auroc":{ + "mean":0.932166, + "std":0.017366, + "values":[ + 0.940071, + 0.912254, + 0.944173 + ] + }, + "oos_auprc":{ + "mean":0.748839, + "std":0.053143, + "values":[ + 0.78009, + 0.687479, + 0.778948 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.003599, + "std":0.001376, + "values":[ + 0.002851, + 0.005188, + 0.00276 + ] + }, + "oos_auroc":{ + "mean":0.92987, + "std":0.023025, + "values":[ + 0.942253, + 0.903303, + 0.944053 + ] + }, + "oos_auprc":{ + "mean":0.365771, + "std":0.06875, + "values":[ + 0.413612, + 0.286988, + 0.396714 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.020094, + "std":0.001999, + "values":[ + 0.018413, + 0.022305, + 0.019564 + ] + }, + "oos_auroc":{ + "mean":0.93904, + "std":0.013014, + "values":[ + 0.944151, + 0.924246, + 0.948723 + ] + }, + "oos_auprc":{ + "mean":0.776896, + "std":0.047557, + "values":[ + 0.803085, + 0.722001, + 0.805603 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.004013, + "std":0.002112, + "values":[ + 0.002924, + 0.006447, + 0.002667 + ] + }, + "oos_auroc":{ + "mean":0.906038, + "std":0.043762, + "values":[ + 0.928943, + 0.855577, + 0.933593 + ] + }, + "oos_auprc":{ + "mean":0.254233, + "std":0.092829, + "values":[ + 0.315795, + 0.14746, + 0.299444 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.02216, + "std":0.004249, + "values":[ + 0.019138, + 0.027019, + 0.020324 + ] + }, + "oos_auroc":{ + "mean":0.913379, + "std":0.031471, + "values":[ + 0.930358, + 0.877065, + 0.932715 + ] + }, + "oos_auprc":{ + "mean":0.652898, + "std":0.093838, + "values":[ + 0.70741, + 0.544545, + 0.70674 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.003564, + "std":0.001449, + "values":[ + 0.002784, + 0.005236, + 0.002672 + ] + }, + "oos_auroc":{ + "mean":0.922414, + "std":0.029289, + "values":[ + 0.938813, + 0.8886, + 0.93983 + ] + }, + "oos_auprc":{ + "mean":0.331937, + "std":0.058797, + "values":[ + 0.38417, + 0.268259, + 0.34338 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.020403, + "std":0.002504, + "values":[ + 0.018385, + 0.023205, + 0.01962 + ] + }, + "oos_auroc":{ + "mean":0.931062, + "std":0.01804, + "values":[ + 0.93974, + 0.910323, + 0.943122 + ] + }, + "oos_auprc":{ + "mean":0.744873, + "std":0.054259, + "values":[ + 0.778621, + 0.682284, + 0.773715 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "entropy", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":-0.205695, + "std":1.552599, + "values":[ + 0.703851, + -1.998417, + 0.677481 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.941935, + "std":0.011179, + "values":[ + 0.948065, + 0.929032, + 0.94871 + ] + }, + "validation_selective_risk":{ + "mean":0.015639, + "std":1.3e-05, + "values":[ + 0.015652, + 0.015625, + 0.015641 + ] + }, + "validation_risk_upper95":{ + "mean":0.019889, + "std":1.6e-05, + "values":[ + 0.019888, + 0.019906, + 0.019874 + ] + }, + "test":{ + "coverage":{ + "mean":0.856606, + "std":0.012531, + "values":[ + 0.86, + 0.842727, + 0.867091 + ] + }, + "selective_risk":{ + "mean":0.052779, + "std":0.005235, + "values":[ + 0.046934, + 0.054369, + 0.057035 + ] + }, + "accuracy_8":{ + "mean":0.89697, + "std":0.003661, + "values":[ + 0.900364, + 0.893091, + 0.897455 + ] + }, + "oos_recall":{ + "mean":0.531333, + "std":0.010599, + "values":[ + 0.541, + 0.533, + 0.52 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.500348, + "std":0.010598, + "values":[ + 0.510016, + 0.50201, + 0.489017 + ] + }, + { + "mean":0.562079, + "std":0.010519, + "values":[ + 0.57167, + 0.563737, + 0.55083 + ] + } + ], + "oos_precision":{ + "mean":0.955093, + "std":0.00744, + "values":[ + 0.960924, + 0.946714, + 0.957643 + ] + }, + "oos_f1":{ + "mean":0.682764, + "std":0.009146, + "values":[ + 0.692258, + 0.682022, + 0.674012 + ] + }, + "oos_misroute_rate":{ + "mean":0.468667, + "std":0.010599, + "values":[ + 0.459, + 0.467, + 0.48 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.201667, + "std":0.026312, + "values":[ + 0.188, + 0.185, + 0.232 + ] + }, + "llm_call_rate":{ + "mean":0.143394, + "std":0.012531, + "values":[ + 0.14, + 0.157273, + 0.132909 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.052791, + "std":0.004633, + "values":[ + 0.051522, + 0.057927, + 0.048925 + ] + }, + "error_recovery_rate":{ + "mean":0.342815, + "std":0.01, + "values":[ + 0.351801, + 0.332041, + 0.344603 + ] + }, + "recoverable_caught":{ + "mean":0.730428, + "std":0.033172, + "values":[ + 0.762763, + 0.696477, + 0.732044 + ] + }, + "small_errors":{ + "mean":755.0, + "std":28.687977, + "values":[ + 722, + 774, + 769 + ] + }, + "recoverable_errors":{ + "mean":354.666667, + "std":19.087518, + "values":[ + 333, + 369, + 362 + ] + }, + "recovered_errors":{ + "mean":258.666667, + "std":5.686241, + "values":[ + 254, + 257, + 265 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp_t", + "msp", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":0.112426, + "std":0.104565, + "values":[ + 0.213919, + 0.11832, + 0.005039 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.999247, + "std":0.001304, + "values":[ + 1.0, + 0.997742, + 1.0 + ] + }, + "validation_selective_risk":{ + "mean":0.041, + "std":0.002168, + "values":[ + 0.039032, + 0.043324, + 0.040645 + ] + }, + "validation_risk_upper95":{ + "mean":0.047271, + "std":0.002317, + "values":[ + 0.045167, + 0.049755, + 0.046891 + ] + }, + "test":{ + "coverage":{ + "mean":0.99697, + "std":0.00447, + "values":[ + 0.999273, + 0.991818, + 0.999818 + ] + }, + "selective_risk":{ + "mean":0.135135, + "std":0.004605, + "values":[ + 0.13064, + 0.134922, + 0.139844 + ] + }, + "accuracy_8":{ + "mean":0.864182, + "std":0.004766, + "values":[ + 0.869455, + 0.862909, + 0.860182 + ] + }, + "oos_recall":{ + "mean":0.371333, + "std":0.033081, + "values":[ + 0.397, + 0.383, + 0.334 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.341983, + "std":0.032375, + "values":[ + 0.367124, + 0.353372, + 0.305452 + ] + }, + { + "mean":0.401669, + "std":0.033533, + "values":[ + 0.427664, + 0.413523, + 0.363819 + ] + } + ], + "oos_precision":{ + "mean":0.962741, + "std":0.01571, + "values":[ + 0.965937, + 0.945679, + 0.976608 + ] + }, + "oos_f1":{ + "mean":0.535227, + "std":0.033606, + "values":[ + 0.562721, + 0.545196, + 0.497765 + ] + }, + "oos_misroute_rate":{ + "mean":0.628667, + "std":0.033081, + "values":[ + 0.603, + 0.617, + 0.666 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.622667, + "std":0.037581, + "values":[ + 0.603, + 0.599, + 0.666 + ] + }, + "llm_call_rate":{ + "mean":0.00303, + "std":0.00447, + "values":[ + 0.000727, + 0.008182, + 0.000182 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.001115, + "std":0.001645, + "values":[ + 0.000267, + 0.003011, + 6.7e-05 + ] + }, + "error_recovery_rate":{ + "mean":0.01046, + "std":0.013604, + "values":[ + 0.00554, + 0.02584, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.022071, + "std":0.028466, + "values":[ + 0.012012, + 0.054201, + 0.0 + ] + }, + "small_errors":{ + "mean":755.0, + "std":28.687977, + "values":[ + 722, + 774, + 769 + ] + }, + "recoverable_errors":{ + "mean":354.666667, + "std":19.087518, + "values":[ + 333, + 369, + 362 + ] + }, + "recovered_errors":{ + "mean":8.0, + "std":10.583005, + "values":[ + 4, + 20, + 0 + ] + } + } + } + } + } + } + } + }, + "modernbert/k5":{ + "model":"modernbert", + "k":5, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "ModernBERT-base-k5-seed42", + "ModernBERT-base-k5-seed43", + "ModernBERT-base-k5-seed44" + ], + "logits_sha256":[ + "8ffbbf00d37abfc1fc2e3601e96ed26c71c6cb2452c8c88d32375674f22996c1", + "561ef2079fd97de9a221e029b2589c50ccd17a579423fa9503ef95fe2944730c", + "9c6b23ca81db2a58e2f444a13533edcf9687f6d4020ba6ec6ecc52915f26e784" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":0.915887, + "std":0.040688, + "values":[ + 0.909945, + 0.959219, + 0.878496 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.027955, + "std":0.006046, + "values":[ + 0.034406, + 0.027042, + 0.022417 + ] + }, + "ece_8_summed":{ + "mean":0.06161, + "std":0.002282, + "values":[ + 0.063592, + 0.062123, + 0.059115 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.028179, + "std":0.011525, + "values":[ + 0.021634, + 0.021417, + 0.041487 + ] + }, + "ece_8_summed":{ + "mean":0.037731, + "std":0.010644, + "values":[ + 0.03644, + 0.048961, + 0.02779 + ] + } + } + } + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.811828, + "std":0.038476, + "values":[ + 0.841613, + 0.768387, + 0.825484 + ] + }, + "summed":{ + "mean":0.829355, + "std":0.036456, + "values":[ + 0.857097, + 0.788065, + 0.842903 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.811828, + "std":0.038476 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.300545, + "std":0.041032 + }, + "accuracy_8":{ + "mean":0.699455, + "std":0.041032 + }, + "oos_recall":{ + "mean":0.147333, + "std":0.042501 + }, + "oos_recall_wilson95":[ + { + "mean":0.126927, + "std":0.039703 + }, + { + "mean":0.170439, + "std":0.044974 + } + ], + "oos_precision":{ + "mean":0.747489, + "std":0.104762 + }, + "oos_f1":{ + "mean":0.245633, + "std":0.064977 + }, + "oos_misroute_rate":{ + "mean":0.852667, + "std":0.042501 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.852667, + "std":0.042501 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.699455, + "std":0.041032 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.897091, + "std":0.011868 + }, + "oos_recall":{ + "mean":0.603333, + "std":0.01222 + }, + "oos_recall_wilson95":[ + { + "mean":0.572679, + "std":0.012329 + }, + { + "mean":0.633197, + "std":0.012017 + } + ], + "oos_precision":{ + "mean":0.9092, + "std":0.021425 + }, + "oos_f1":{ + "mean":0.725315, + "std":0.014763 + }, + "oos_misroute_rate":{ + "mean":0.396667, + "std":0.01222 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.300545, + "std":0.041032 + }, + "llm_cost_usd_per_1k":{ + "mean":0.110808, + "std":0.015177 + }, + "error_recovery_rate":{ + "mean":0.656747, + "std":0.014852 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":1087.0, + "std":163.685674 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.057405, + "std":0.019332 + }, + "oos_auroc":{ + "mean":0.760649, + "std":0.022838 + }, + "oos_auprc":{ + "mean":0.097326, + "std":0.010819 + } + }, + "test":{ + "aurc":{ + "mean":0.107562, + "std":0.026807 + }, + "oos_auroc":{ + "mean":0.788629, + "std":0.009608 + }, + "oos_auprc":{ + "mean":0.409598, + "std":0.019041 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.057529, + "std":0.018389 + }, + "oos_auroc":{ + "mean":0.777754, + "std":0.02568 + }, + "oos_auprc":{ + "mean":0.103805, + "std":0.014997 + } + }, + "test":{ + "aurc":{ + "mean":0.1061, + "std":0.026122 + }, + "oos_auroc":{ + "mean":0.806529, + "std":0.010325 + }, + "oos_auprc":{ + "mean":0.444192, + "std":0.017891 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.066772, + "std":0.022756 + }, + "oos_auroc":{ + "mean":0.70575, + "std":0.020776 + }, + "oos_auprc":{ + "mean":0.065422, + "std":0.002117 + } + }, + "test":{ + "aurc":{ + "mean":0.124597, + "std":0.030803 + }, + "oos_auroc":{ + "mean":0.724266, + "std":0.0126 + }, + "oos_auprc":{ + "mean":0.312467, + "std":0.010901 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.057615, + "std":0.019379 + }, + "oos_auroc":{ + "mean":0.756984, + "std":0.022468 + }, + "oos_auprc":{ + "mean":0.095939, + "std":0.009527 + } + }, + "test":{ + "aurc":{ + "mean":0.108162, + "std":0.02672 + }, + "oos_auroc":{ + "mean":0.785295, + "std":0.008286 + }, + "oos_auprc":{ + "mean":0.40569, + "std":0.018839 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "entropy", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":0.256583, + "std":0.91838 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.336237, + "std":0.106264 + }, + "validation_selective_risk":{ + "mean":0.012162, + "std":0.001471 + }, + "validation_risk_upper95":{ + "mean":0.019402, + "std":0.000168 + }, + "test":{ + "coverage":{ + "mean":0.286182, + "std":0.098865 + }, + "selective_risk":{ + "mean":0.028272, + "std":0.006827 + }, + "accuracy_8":{ + "mean":0.840182, + "std":0.010045 + }, + "oos_recall":{ + "mean":0.564, + "std":0.002646 + }, + "oos_recall_wilson95":[ + { + "mean":0.533078, + "std":0.002657 + }, + { + "mean":0.594432, + "std":0.002614 + } + ], + "oos_precision":{ + "mean":0.787308, + "std":0.017911 + }, + "oos_f1":{ + "mean":0.657122, + "std":0.004501 + }, + "oos_misroute_rate":{ + "mean":0.436, + "std":0.002646 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.019333, + "std":0.010786 + }, + "llm_call_rate":{ + "mean":0.713818, + "std":0.098865 + }, + "llm_cost_usd_per_1k":{ + "mean":0.263626, + "std":0.036568 + }, + "error_recovery_rate":{ + "mean":0.639482, + "std":0.018786 + }, + "recoverable_caught":{ + "mean":0.973662, + "std":0.013869 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":1059.666667, + "std":173.338782 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.79547, + "std":0.071266 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.333118, + "std":0.109166 + }, + "validation_selective_risk":{ + "mean":0.012303, + "std":0.001307 + }, + "validation_risk_upper95":{ + "mean":0.019649, + "std":0.000195 + }, + "test":{ + "coverage":{ + "mean":0.283333, + "std":0.102967 + }, + "selective_risk":{ + "mean":0.027194, + "std":0.007412 + }, + "accuracy_8":{ + "mean":0.840061, + "std":0.01114 + }, + "oos_recall":{ + "mean":0.565, + "std":0.002 + }, + "oos_recall_wilson95":[ + { + "mean":0.534082, + "std":0.002009 + }, + { + "mean":0.595421, + "std":0.001976 + } + ], + "oos_precision":{ + "mean":0.786936, + "std":0.02045 + }, + "oos_f1":{ + "mean":0.657663, + "std":0.006319 + }, + "oos_misroute_rate":{ + "mean":0.435, + "std":0.002 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.017667, + "std":0.010116 + }, + "llm_call_rate":{ + "mean":0.716667, + "std":0.102967 + }, + "llm_cost_usd_per_1k":{ + "mean":0.264683, + "std":0.038093 + }, + "error_recovery_rate":{ + "mean":0.640171, + "std":0.01764 + }, + "recoverable_caught":{ + "mean":0.97474, + "std":0.013153 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":1060.666667, + "std":171.934096 + } + } + }, + "entropy":{ + "tau":{ + "mean":-0.963769, + "std":0.143989 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.306667, + "std":0.079597 + }, + "validation_selective_risk":{ + "mean":0.011733, + "std":0.001091 + }, + "validation_risk_upper95":{ + "mean":0.019199, + "std":1.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.261273, + "std":0.077148 + }, + "selective_risk":{ + "mean":0.023537, + "std":0.005291 + }, + "accuracy_8":{ + "mean":0.839212, + "std":0.009288 + }, + "oos_recall":{ + "mean":0.567333, + "std":0.001528 + }, + "oos_recall_wilson95":[ + { + "mean":0.536425, + "std":0.001535 + }, + { + "mean":0.597726, + "std":0.001509 + } + ], + "oos_precision":{ + "mean":0.784904, + "std":0.016221 + }, + "oos_f1":{ + "mean":0.658575, + "std":0.006317 + }, + "oos_misroute_rate":{ + "mean":0.432667, + "std":0.001528 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.010667, + "std":0.004726 + }, + "llm_call_rate":{ + "mean":0.738727, + "std":0.077148 + }, + "llm_cost_usd_per_1k":{ + "mean":0.272846, + "std":0.028531 + }, + "error_recovery_rate":{ + "mean":0.643664, + "std":0.015352 + }, + "recoverable_caught":{ + "mean":0.980087, + "std":0.00885 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":1066.0, + "std":167.725967 + } + } + }, + "margin":{ + "tau":{ + "mean":3.43977, + "std":0.397724 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.297419, + "std":0.094754 + }, + "validation_selective_risk":{ + "mean":0.011551, + "std":0.001789 + }, + "validation_risk_upper95":{ + "mean":0.019178, + "std":0.000659 + }, + "test":{ + "coverage":{ + "mean":0.252182, + "std":0.087865 + }, + "selective_risk":{ + "mean":0.025929, + "std":0.002025 + }, + "accuracy_8":{ + "mean":0.838242, + "std":0.011015 + }, + "oos_recall":{ + "mean":0.562667, + "std":0.002309 + }, + "oos_recall_wilson95":[ + { + "mean":0.531739, + "std":0.002318 + }, + { + "mean":0.593115, + "std":0.002283 + } + ], + "oos_precision":{ + "mean":0.781416, + "std":0.017343 + }, + "oos_f1":{ + "mean":0.654171, + "std":0.004911 + }, + "oos_misroute_rate":{ + "mean":0.437333, + "std":0.002309 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.019, + "std":0.009539 + }, + "llm_call_rate":{ + "mean":0.747818, + "std":0.087865 + }, + "llm_cost_usd_per_1k":{ + "mean":0.276206, + "std":0.032489 + }, + "error_recovery_rate":{ + "mean":0.644198, + "std":0.021221 + }, + "recoverable_caught":{ + "mean":0.980739, + "std":0.010387 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":1067.0, + "std":169.51991 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.861118, + "std":0.03695 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.327097, + "std":0.106377 + }, + "validation_selective_risk":{ + "mean":0.01222, + "std":0.001257 + }, + "validation_risk_upper95":{ + "mean":0.019615, + "std":0.000167 + }, + "test":{ + "coverage":{ + "mean":0.278061, + "std":0.099445 + }, + "selective_risk":{ + "mean":0.026948, + "std":0.006302 + }, + "accuracy_8":{ + "mean":0.839576, + "std":0.010683 + }, + "oos_recall":{ + "mean":0.564333, + "std":0.002517 + }, + "oos_recall_wilson95":[ + { + "mean":0.533413, + "std":0.002527 + }, + { + "mean":0.594762, + "std":0.002487 + } + ], + "oos_precision":{ + "mean":0.78526, + "std":0.019209 + }, + "oos_f1":{ + "mean":0.656624, + "std":0.005072 + }, + "oos_misroute_rate":{ + "mean":0.435667, + "std":0.002517 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.018333, + "std":0.01097 + }, + "llm_call_rate":{ + "mean":0.721939, + "std":0.099445 + }, + "llm_cost_usd_per_1k":{ + "mean":0.266632, + "std":0.036787 + }, + "error_recovery_rate":{ + "mean":0.640722, + "std":0.019421 + }, + "recoverable_caught":{ + "mean":0.975517, + "std":0.012373 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":1061.666667, + "std":173.032752 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp", + "entropy", + "entropy" + ], + "hybrid":{ + "tau":{ + "mean":-1.153793, + "std":1.377153 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.530968, + "std":0.120216 + }, + "validation_selective_risk":{ + "mean":0.040949, + "std":0.001068 + }, + "validation_risk_upper95":{ + "mean":0.049892, + "std":4.4e-05 + }, + "test":{ + "coverage":{ + "mean":0.463515, + "std":0.112565 + }, + "selective_risk":{ + "mean":0.067432, + "std":0.007611 + }, + "accuracy_8":{ + "mean":0.842909, + "std":0.008689 + }, + "oos_recall":{ + "mean":0.545667, + "std":0.017214 + }, + "oos_recall_wilson95":[ + { + "mean":0.514702, + "std":0.017236 + }, + { + "mean":0.576282, + "std":0.017061 + } + ], + "oos_precision":{ + "mean":0.815777, + "std":0.024353 + }, + "oos_f1":{ + "mean":0.653552, + "std":0.006679 + }, + "oos_misroute_rate":{ + "mean":0.454333, + "std":0.017214 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.069667, + "std":0.034487 + }, + "llm_call_rate":{ + "mean":0.536485, + "std":0.112565 + }, + "llm_cost_usd_per_1k":{ + "mean":0.198062, + "std":0.041582 + }, + "error_recovery_rate":{ + "mean":0.590317, + "std":0.046776 + }, + "recoverable_caught":{ + "mean":0.898159, + "std":0.05378 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":982.0, + "std":203.501843 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.535471, + "std":0.087669 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.521613, + "std":0.127918 + }, + "validation_selective_risk":{ + "mean":0.04084, + "std":0.001118 + }, + "validation_risk_upper95":{ + "mean":0.049882, + "std":0.000101 + }, + "test":{ + "coverage":{ + "mean":0.454364, + "std":0.11799 + }, + "selective_risk":{ + "mean":0.067455, + "std":0.008109 + }, + "accuracy_8":{ + "mean":0.842485, + "std":0.009146 + }, + "oos_recall":{ + "mean":0.543, + "std":0.014933 + }, + "oos_recall_wilson95":[ + { + "mean":0.512028, + "std":0.014948 + }, + { + "mean":0.573643, + "std":0.014804 + } + ], + "oos_precision":{ + "mean":0.813382, + "std":0.0244 + }, + "oos_f1":{ + "mean":0.650904, + "std":0.004097 + }, + "oos_misroute_rate":{ + "mean":0.457, + "std":0.014933 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.073333, + "std":0.031342 + }, + "llm_call_rate":{ + "mean":0.545636, + "std":0.11799 + }, + "llm_cost_usd_per_1k":{ + "mean":0.201449, + "std":0.043596 + }, + "error_recovery_rate":{ + "mean":0.592032, + "std":0.047718 + }, + "recoverable_caught":{ + "mean":0.90072, + "std":0.054425 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":984.666667, + "std":202.574266 + } + } + }, + "entropy":{ + "tau":{ + "mean":-2.041897, + "std":0.164803 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.521828, + "std":0.106538 + }, + "validation_selective_risk":{ + "mean":0.040875, + "std":0.000964 + }, + "validation_risk_upper95":{ + "mean":0.049874, + "std":2.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.450242, + "std":0.093112 + }, + "selective_risk":{ + "mean":0.062999, + "std":8.5e-05 + }, + "accuracy_8":{ + "mean":0.843576, + "std":0.009597 + }, + "oos_recall":{ + "mean":0.552667, + "std":0.005508 + }, + "oos_recall_wilson95":[ + { + "mean":0.521707, + "std":0.005522 + }, + { + "mean":0.583223, + "std":0.005451 + } + ], + "oos_precision":{ + "mean":0.81282, + "std":0.020349 + }, + "oos_f1":{ + "mean":0.657833, + "std":0.003188 + }, + "oos_misroute_rate":{ + "mean":0.447333, + "std":0.005508 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.057333, + "std":0.016258 + }, + "llm_call_rate":{ + "mean":0.549758, + "std":0.093112 + }, + "llm_cost_usd_per_1k":{ + "mean":0.202958, + "std":0.034409 + }, + "error_recovery_rate":{ + "mean":0.598593, + "std":0.032798 + }, + "recoverable_caught":{ + "mean":0.91109, + "std":0.033406 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":994.0, + "std":185.84133 + } + } + }, + "margin":{ + "tau":{ + "mean":2.020664, + "std":0.371151 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.471613, + "std":0.118197 + }, + "validation_selective_risk":{ + "mean":0.040138, + "std":0.001325 + }, + "validation_risk_upper95":{ + "mean":0.049637, + "std":0.000139 + }, + "test":{ + "coverage":{ + "mean":0.415091, + "std":0.113121 + }, + "selective_risk":{ + "mean":0.073099, + "std":0.007633 + }, + "accuracy_8":{ + "mean":0.838606, + "std":0.008728 + }, + "oos_recall":{ + "mean":0.536333, + "std":0.016166 + }, + "oos_recall_wilson95":[ + { + "mean":0.505356, + "std":0.016173 + }, + { + "mean":0.567033, + "std":0.016035 + } + ], + "oos_precision":{ + "mean":0.805479, + "std":0.024435 + }, + "oos_f1":{ + "mean":0.643538, + "std":0.00415 + }, + "oos_misroute_rate":{ + "mean":0.463667, + "std":0.016166 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.087, + "std":0.039154 + }, + "llm_call_rate":{ + "mean":0.584909, + "std":0.113121 + }, + "llm_cost_usd_per_1k":{ + "mean":0.215994, + "std":0.041794 + }, + "error_recovery_rate":{ + "mean":0.592018, + "std":0.043377 + }, + "recoverable_caught":{ + "mean":0.900871, + "std":0.049815 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":984.666667, + "std":201.75315 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.607424, + "std":0.091906 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.518065, + "std":0.126809 + }, + "validation_selective_risk":{ + "mean":0.040732, + "std":0.000985 + }, + "validation_risk_upper95":{ + "mean":0.049796, + "std":0.000198 + }, + "test":{ + "coverage":{ + "mean":0.451636, + "std":0.117057 + }, + "selective_risk":{ + "mean":0.068425, + "std":0.008273 + }, + "accuracy_8":{ + "mean":0.842303, + "std":0.008431 + }, + "oos_recall":{ + "mean":0.542333, + "std":0.016073 + }, + "oos_recall_wilson95":[ + { + "mean":0.511362, + "std":0.016087 + }, + { + "mean":0.572981, + "std":0.015935 + } + ], + "oos_precision":{ + "mean":0.812718, + "std":0.022719 + }, + "oos_f1":{ + "mean":0.650214, + "std":0.005593 + }, + "oos_misroute_rate":{ + "mean":0.457667, + "std":0.016073 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.076333, + "std":0.03371 + }, + "llm_call_rate":{ + "mean":0.548364, + "std":0.117057 + }, + "llm_cost_usd_per_1k":{ + "mean":0.202458, + "std":0.043248 + }, + "error_recovery_rate":{ + "mean":0.591488, + "std":0.049124 + }, + "recoverable_caught":{ + "mean":0.899865, + "std":0.056717 + }, + "small_errors":{ + "mean":1653.0, + "std":225.674544 + }, + "recoverable_errors":{ + "mean":1087.0, + "std":163.685674 + }, + "recovered_errors":{ + "mean":984.0, + "std":205.065843 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.829355, + "std":0.036456 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.290061, + "std":0.033965 + }, + "accuracy_8":{ + "mean":0.709939, + "std":0.033965 + }, + "oos_recall":{ + "mean":0.11, + "std":0.04293 + }, + "oos_recall_wilson95":[ + { + "mean":0.092385, + "std":0.039513 + }, + { + "mean":0.1306, + "std":0.04602 + } + ], + "oos_precision":{ + "mean":0.772455, + "std":0.064439 + }, + "oos_f1":{ + "mean":0.191273, + "std":0.067505 + }, + "oos_misroute_rate":{ + "mean":0.89, + "std":0.04293 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.89, + "std":0.04293 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.709939, + "std":0.033965 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.895333, + "std":0.010897 + }, + "oos_recall":{ + "mean":0.590667, + "std":0.009452 + }, + "oos_recall_wilson95":[ + { + "mean":0.559903, + "std":0.00952 + }, + { + "mean":0.620736, + "std":0.009311 + } + ], + "oos_precision":{ + "mean":0.910301, + "std":0.023483 + }, + "oos_f1":{ + "mean":0.716429, + "std":0.013816 + }, + "oos_misroute_rate":{ + "mean":0.409333, + "std":0.009452 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.290061, + "std":0.033965 + }, + "llm_cost_usd_per_1k":{ + "mean":0.106908, + "std":0.012565 + }, + "error_recovery_rate":{ + "mean":0.638592, + "std":0.01399 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":1019.666667, + "std":130.936371 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.040797, + "std":0.014402 + }, + "oos_auroc":{ + "mean":0.824351, + "std":0.024404 + }, + "oos_auprc":{ + "mean":0.146982, + "std":0.045039 + } + }, + "test":{ + "aurc":{ + "mean":0.089409, + "std":0.021808 + }, + "oos_auroc":{ + "mean":0.830177, + "std":0.014527 + }, + "oos_auprc":{ + "mean":0.487184, + "std":0.030754 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.044194, + "std":0.015 + }, + "oos_auroc":{ + "mean":0.838524, + "std":0.028866 + }, + "oos_auprc":{ + "mean":0.185507, + "std":0.063336 + } + }, + "test":{ + "aurc":{ + "mean":0.091216, + "std":0.021719 + }, + "oos_auroc":{ + "mean":0.84594, + "std":0.016038 + }, + "oos_auprc":{ + "mean":0.543021, + "std":0.043922 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.040659, + "std":0.01432 + }, + "oos_auroc":{ + "mean":0.788748, + "std":0.018542 + }, + "oos_auprc":{ + "mean":0.091813, + "std":0.007456 + } + }, + "test":{ + "aurc":{ + "mean":0.092646, + "std":0.022292 + }, + "oos_auroc":{ + "mean":0.794558, + "std":0.01156 + }, + "oos_auprc":{ + "mean":0.391667, + "std":0.015806 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.041067, + "std":0.014411 + }, + "oos_auroc":{ + "mean":0.818861, + "std":0.020977 + }, + "oos_auprc":{ + "mean":0.135761, + "std":0.032385 + } + }, + "test":{ + "aurc":{ + "mean":0.089611, + "std":0.021915 + }, + "oos_auroc":{ + "mean":0.826827, + "std":0.012711 + }, + "oos_auprc":{ + "mean":0.478998, + "std":0.025869 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "margin", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":1.99402, + "std":0.962075 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.469677, + "std":0.078016 + }, + "validation_selective_risk":{ + "mean":0.013683, + "std":0.00052 + }, + "validation_risk_upper95":{ + "mean":0.019718, + "std":0.000189 + }, + "test":{ + "coverage":{ + "mean":0.400424, + "std":0.067239 + }, + "selective_risk":{ + "mean":0.033054, + "std":0.002248 + }, + "accuracy_8":{ + "mean":0.844364, + "std":0.008598 + }, + "oos_recall":{ + "mean":0.558, + "std":0.003606 + }, + "oos_recall_wilson95":[ + { + "mean":0.527056, + "std":0.003617 + }, + { + "mean":0.5885, + "std":0.003566 + } + ], + "oos_precision":{ + "mean":0.80276, + "std":0.016143 + }, + "oos_f1":{ + "mean":0.6583, + "std":0.003934 + }, + "oos_misroute_rate":{ + "mean":0.442, + "std":0.003606 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.04, + "std":0.006928 + }, + "llm_call_rate":{ + "mean":0.599576, + "std":0.067239 + }, + "llm_cost_usd_per_1k":{ + "mean":0.221344, + "std":0.024878 + }, + "error_recovery_rate":{ + "mean":0.6155, + "std":0.019232 + }, + "recoverable_caught":{ + "mean":0.963721, + "std":0.011422 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":983.666667, + "std":137.485757 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.824507, + "std":0.027333 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.445806, + "std":0.104405 + }, + "validation_selective_risk":{ + "mean":0.013606, + "std":0.00092 + }, + "validation_risk_upper95":{ + "mean":0.019871, + "std":0.000169 + }, + "test":{ + "coverage":{ + "mean":0.382121, + "std":0.085676 + }, + "selective_risk":{ + "mean":0.02967, + "std":0.001928 + }, + "accuracy_8":{ + "mean":0.844545, + "std":0.009683 + }, + "oos_recall":{ + "mean":0.562667, + "std":0.001528 + }, + "oos_recall_wilson95":[ + { + "mean":0.531739, + "std":0.001534 + }, + { + "mean":0.593115, + "std":0.00151 + } + ], + "oos_precision":{ + "mean":0.799566, + "std":0.018654 + }, + "oos_f1":{ + "mean":0.660446, + "std":0.005456 + }, + "oos_misroute_rate":{ + "mean":0.437333, + "std":0.001528 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.030667, + "std":0.006658 + }, + "llm_call_rate":{ + "mean":0.617879, + "std":0.085676 + }, + "llm_cost_usd_per_1k":{ + "mean":0.228096, + "std":0.031699 + }, + "error_recovery_rate":{ + "mean":0.619435, + "std":0.017446 + }, + "recoverable_caught":{ + "mean":0.969946, + "std":0.01199 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":990.0, + "std":138.632608 + } + } + }, + "entropy":{ + "tau":{ + "mean":-0.657687, + "std":0.054114 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.418495, + "std":0.083845 + }, + "validation_selective_risk":{ + "mean":0.013213, + "std":0.001105 + }, + "validation_risk_upper95":{ + "mean":0.01961, + "std":0.00049 + }, + "test":{ + "coverage":{ + "mean":0.360727, + "std":0.075645 + }, + "selective_risk":{ + "mean":0.027867, + "std":0.004748 + }, + "accuracy_8":{ + "mean":0.844364, + "std":0.009538 + }, + "oos_recall":{ + "mean":0.564667, + "std":0.001155 + }, + "oos_recall_wilson95":[ + { + "mean":0.533747, + "std":0.00116 + }, + { + "mean":0.595091, + "std":0.001141 + } + ], + "oos_precision":{ + "mean":0.798198, + "std":0.018568 + }, + "oos_f1":{ + "mean":0.661373, + "std":0.007019 + }, + "oos_misroute_rate":{ + "mean":0.435333, + "std":0.001155 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.024, + "std":0.006083 + }, + "llm_call_rate":{ + "mean":0.639273, + "std":0.075645 + }, + "llm_cost_usd_per_1k":{ + "mean":0.236014, + "std":0.027988 + }, + "error_recovery_rate":{ + "mean":0.621319, + "std":0.014182 + }, + "recoverable_caught":{ + "mean":0.972966, + "std":0.009181 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":992.666667, + "std":134.552344 + } + } + }, + "margin":{ + "tau":{ + "mean":2.680092, + "std":0.226599 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.465591, + "std":0.078714 + }, + "validation_selective_risk":{ + "mean":0.013571, + "std":0.000492 + }, + "validation_risk_upper95":{ + "mean":0.019617, + "std":1.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.396545, + "std":0.067297 + }, + "selective_risk":{ + "mean":0.032129, + "std":0.003441 + }, + "accuracy_8":{ + "mean":0.844242, + "std":0.008577 + }, + "oos_recall":{ + "mean":0.557333, + "std":0.002887 + }, + "oos_recall_wilson95":[ + { + "mean":0.526387, + "std":0.002896 + }, + { + "mean":0.587841, + "std":0.002855 + } + ], + "oos_precision":{ + "mean":0.803337, + "std":0.016102 + }, + "oos_f1":{ + "mean":0.658032, + "std":0.003785 + }, + "oos_misroute_rate":{ + "mean":0.442667, + "std":0.002887 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.040667, + "std":0.006429 + }, + "llm_call_rate":{ + "mean":0.603455, + "std":0.067297 + }, + "llm_cost_usd_per_1k":{ + "mean":0.222782, + "std":0.024896 + }, + "error_recovery_rate":{ + "mean":0.617605, + "std":0.021763 + }, + "recoverable_caught":{ + "mean":0.966951, + "std":0.013589 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":987.0, + "std":138.09055 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.870887, + "std":0.018133 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.447097, + "std":0.108353 + }, + "validation_selective_risk":{ + "mean":0.013581, + "std":0.000804 + }, + "validation_risk_upper95":{ + "mean":0.019842, + "std":8.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.381879, + "std":0.089017 + }, + "selective_risk":{ + "mean":0.029479, + "std":0.002003 + }, + "accuracy_8":{ + "mean":0.844606, + "std":0.010027 + }, + "oos_recall":{ + "mean":0.561, + "std":0.003 + }, + "oos_recall_wilson95":[ + { + "mean":0.530066, + "std":0.003011 + }, + { + "mean":0.591467, + "std":0.002966 + } + ], + "oos_precision":{ + "mean":0.798383, + "std":0.019112 + }, + "oos_f1":{ + "mean":0.658874, + "std":0.004504 + }, + "oos_misroute_rate":{ + "mean":0.439, + "std":0.003 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.032667, + "std":0.009452 + }, + "llm_call_rate":{ + "mean":0.618121, + "std":0.089017 + }, + "llm_cost_usd_per_1k":{ + "mean":0.22819, + "std":0.032927 + }, + "error_recovery_rate":{ + "mean":0.619435, + "std":0.017446 + }, + "recoverable_caught":{ + "mean":0.969946, + "std":0.01199 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":990.0, + "std":138.632608 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp", + "margin", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.234831, + "std":0.600639 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.665376, + "std":0.099866 + }, + "validation_selective_risk":{ + "mean":0.041791, + "std":0.000673 + }, + "validation_risk_upper95":{ + "mean":0.049709, + "std":0.000177 + }, + "test":{ + "coverage":{ + "mean":0.584364, + "std":0.090743 + }, + "selective_risk":{ + "mean":0.082761, + "std":0.007935 + }, + "accuracy_8":{ + "mean":0.842788, + "std":0.011069 + }, + "oos_recall":{ + "mean":0.520333, + "std":0.015011 + }, + "oos_recall_wilson95":[ + { + "mean":0.489359, + "std":0.01499 + }, + { + "mean":0.551152, + "std":0.014917 + } + ], + "oos_precision":{ + "mean":0.834127, + "std":0.023825 + }, + "oos_f1":{ + "mean":0.640625, + "std":0.009673 + }, + "oos_misroute_rate":{ + "mean":0.479667, + "std":0.015011 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.149, + "std":0.037323 + }, + "llm_call_rate":{ + "mean":0.415636, + "std":0.090743 + }, + "llm_cost_usd_per_1k":{ + "mean":0.15336, + "std":0.033575 + }, + "error_recovery_rate":{ + "mean":0.548183, + "std":0.033186 + }, + "recoverable_caught":{ + "mean":0.858313, + "std":0.045682 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":878.666667, + "std":156.090786 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.604164, + "std":0.054922 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.657419, + "std":0.102265 + }, + "validation_selective_risk":{ + "mean":0.041638, + "std":0.000671 + }, + "validation_risk_upper95":{ + "mean":0.049598, + "std":5.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.572727, + "std":0.097869 + }, + "selective_risk":{ + "mean":0.075821, + "std":0.004264 + }, + "accuracy_8":{ + "mean":0.845576, + "std":0.009546 + }, + "oos_recall":{ + "mean":0.531, + "std":0.01179 + }, + "oos_recall_wilson95":[ + { + "mean":0.500015, + "std":0.011793 + }, + { + "mean":0.561747, + "std":0.011696 + } + ], + "oos_precision":{ + "mean":0.836458, + "std":0.021316 + }, + "oos_f1":{ + "mean":0.64938, + "std":0.002511 + }, + "oos_misroute_rate":{ + "mean":0.469, + "std":0.01179 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.125, + "std":0.037643 + }, + "llm_call_rate":{ + "mean":0.427273, + "std":0.097869 + }, + "llm_cost_usd_per_1k":{ + "mean":0.157643, + "std":0.036192 + }, + "error_recovery_rate":{ + "mean":0.55856, + "std":0.038147 + }, + "recoverable_caught":{ + "mean":0.874312, + "std":0.048125 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":895.666667, + "std":163.02556 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.157111, + "std":0.100752 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.622258, + "std":0.104373 + }, + "validation_selective_risk":{ + "mean":0.041584, + "std":0.000546 + }, + "validation_risk_upper95":{ + "mean":0.049789, + "std":0.000161 + }, + "test":{ + "coverage":{ + "mean":0.537636, + "std":0.090903 + }, + "selective_risk":{ + "mean":0.067344, + "std":0.000914 + }, + "accuracy_8":{ + "mean":0.846121, + "std":0.009182 + }, + "oos_recall":{ + "mean":0.547, + "std":0.008544 + }, + "oos_recall_wilson95":[ + { + "mean":0.516029, + "std":0.00856 + }, + { + "mean":0.577612, + "std":0.008462 + } + ], + "oos_precision":{ + "mean":0.828055, + "std":0.019814 + }, + "oos_f1":{ + "mean":0.65864, + "std":0.000362 + }, + "oos_misroute_rate":{ + "mean":0.453, + "std":0.008544 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.085667, + "std":0.021939 + }, + "llm_call_rate":{ + "mean":0.462364, + "std":0.090903 + }, + "llm_cost_usd_per_1k":{ + "mean":0.170588, + "std":0.033623 + }, + "error_recovery_rate":{ + "mean":0.571266, + "std":0.035404 + }, + "recoverable_caught":{ + "mean":0.894174, + "std":0.040987 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":915.333333, + "std":157.944083 + } + } + }, + "margin":{ + "tau":{ + "mean":1.49653, + "std":0.154304 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.661505, + "std":0.09468 + }, + "validation_selective_risk":{ + "mean":0.041866, + "std":0.000755 + }, + "validation_risk_upper95":{ + "mean":0.04981, + "std":0.000182 + }, + "test":{ + "coverage":{ + "mean":0.578788, + "std":0.083165 + }, + "selective_risk":{ + "mean":0.08158, + "std":0.008892 + }, + "accuracy_8":{ + "mean":0.841818, + "std":0.009463 + }, + "oos_recall":{ + "mean":0.516, + "std":0.016523 + }, + "oos_recall_wilson95":[ + { + "mean":0.485035, + "std":0.016501 + }, + { + "mean":0.546843, + "std":0.016417 + } + ], + "oos_precision":{ + "mean":0.831184, + "std":0.019812 + }, + "oos_f1":{ + "mean":0.636421, + "std":0.007311 + }, + "oos_misroute_rate":{ + "mean":0.484, + "std":0.016523 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.152333, + "std":0.03894 + }, + "llm_call_rate":{ + "mean":0.421212, + "std":0.083165 + }, + "llm_cost_usd_per_1k":{ + "mean":0.155429, + "std":0.030763 + }, + "error_recovery_rate":{ + "mean":0.550303, + "std":0.030232 + }, + "recoverable_caught":{ + "mean":0.861711, + "std":0.042556 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":881.666667, + "std":151.862877 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.663863, + "std":0.054579 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.65129, + "std":0.101304 + }, + "validation_selective_risk":{ + "mean":0.041864, + "std":0.000687 + }, + "validation_risk_upper95":{ + "mean":0.049883, + "std":7.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.567152, + "std":0.097736 + }, + "selective_risk":{ + "mean":0.07484, + "std":0.004426 + }, + "accuracy_8":{ + "mean":0.845152, + "std":0.00891 + }, + "oos_recall":{ + "mean":0.530333, + "std":0.01365 + }, + "oos_recall_wilson95":[ + { + "mean":0.499351, + "std":0.013652 + }, + { + "mean":0.561084, + "std":0.013544 + } + ], + "oos_precision":{ + "mean":0.834122, + "std":0.020705 + }, + "oos_f1":{ + "mean":0.648146, + "std":0.003859 + }, + "oos_misroute_rate":{ + "mean":0.469667, + "std":0.01365 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.122667, + "std":0.036556 + }, + "llm_call_rate":{ + "mean":0.432848, + "std":0.097736 + }, + "llm_cost_usd_per_1k":{ + "mean":0.159712, + "std":0.036139 + }, + "error_recovery_rate":{ + "mean":0.559093, + "std":0.039986 + }, + "recoverable_caught":{ + "mean":0.875085, + "std":0.050187 + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603 + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371 + }, + "recovered_errors":{ + "mean":896.666667, + "std":165.122177 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.290061, + "std":0.033965, + "values":[ + 0.257273, + 0.325091, + 0.287818 + ] + }, + "accuracy_8":{ + "mean":0.709939, + "std":0.033965, + "values":[ + 0.742727, + 0.674909, + 0.712182 + ] + }, + "oos_recall":{ + "mean":0.11, + "std":0.04293, + "values":[ + 0.156, + 0.071, + 0.103 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.092385, + "std":0.039513, + "values":[ + 0.134831, + 0.056669, + 0.085655 + ] + }, + { + "mean":0.1306, + "std":0.04602, + "values":[ + 0.179802, + 0.088614, + 0.123384 + ] + } + ], + "oos_precision":{ + "mean":0.772455, + "std":0.064439, + "values":[ + 0.83871, + 0.71, + 0.768657 + ] + }, + "oos_f1":{ + "mean":0.191273, + "std":0.067505, + "values":[ + 0.263069, + 0.129091, + 0.181658 + ] + }, + "oos_misroute_rate":{ + "mean":0.89, + "std":0.04293, + "values":[ + 0.844, + 0.929, + 0.897 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.89, + "std":0.04293, + "values":[ + 0.844, + 0.929, + 0.897 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603, + "values":[ + 1415, + 1788, + 1583 + ] + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371, + "values":[ + 883, + 1144, + 1032 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.709939, + "std":0.033965, + "values":[ + 0.742727, + 0.674909, + 0.712182 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.895333, + "std":0.010897, + "values":[ + 0.903273, + 0.882909, + 0.899818 + ] + }, + "oos_recall":{ + "mean":0.590667, + "std":0.009452, + "values":[ + 0.594, + 0.58, + 0.598 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.559903, + "std":0.00952, + "values":[ + 0.563259, + 0.54916, + 0.567292 + ] + }, + { + "mean":0.620736, + "std":0.009311, + "values":[ + 0.624022, + 0.610228, + 0.627958 + ] + } + ], + "oos_precision":{ + "mean":0.910301, + "std":0.023483, + "values":[ + 0.929577, + 0.884146, + 0.917178 + ] + }, + "oos_f1":{ + "mean":0.716429, + "std":0.013816, + "values":[ + 0.724832, + 0.700483, + 0.723971 + ] + }, + "oos_misroute_rate":{ + "mean":0.409333, + "std":0.009452, + "values":[ + 0.406, + 0.42, + 0.402 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.290061, + "std":0.033965, + "values":[ + 0.257273, + 0.325091, + 0.287818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.106908, + "std":0.012565, + "values":[ + 0.094779, + 0.119867, + 0.106078 + ] + }, + "error_recovery_rate":{ + "mean":0.638592, + "std":0.01399, + "values":[ + 0.624028, + 0.639821, + 0.651927 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603, + "values":[ + 1415, + 1788, + 1583 + ] + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371, + "values":[ + 883, + 1144, + 1032 + ] + }, + "recovered_errors":{ + "mean":1019.666667, + "std":130.936371, + "values":[ + 883, + 1144, + 1032 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.040797, + "std":0.014402, + "values":[ + 0.029036, + 0.05686, + 0.036495 + ] + }, + "oos_auroc":{ + "mean":0.824351, + "std":0.024404, + "values":[ + 0.83946, + 0.796197, + 0.837397 + ] + }, + "oos_auprc":{ + "mean":0.146982, + "std":0.045039, + "values":[ + 0.150772, + 0.100167, + 0.190006 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.089409, + "std":0.021808, + "values":[ + 0.069229, + 0.112543, + 0.086456 + ] + }, + "oos_auroc":{ + "mean":0.830177, + "std":0.014527, + "values":[ + 0.842038, + 0.813974, + 0.83452 + ] + }, + "oos_auprc":{ + "mean":0.487184, + "std":0.030754, + "values":[ + 0.514182, + 0.453707, + 0.493662 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.044194, + "std":0.015, + "values":[ + 0.031667, + 0.060816, + 0.040097 + ] + }, + "oos_auroc":{ + "mean":0.838524, + "std":0.028866, + "values":[ + 0.854173, + 0.805213, + 0.856187 + ] + }, + "oos_auprc":{ + "mean":0.185507, + "std":0.063336, + "values":[ + 0.201363, + 0.115749, + 0.239408 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.091216, + "std":0.021719, + "values":[ + 0.071052, + 0.114212, + 0.088384 + ] + }, + "oos_auroc":{ + "mean":0.84594, + "std":0.016038, + "values":[ + 0.85988, + 0.828412, + 0.849527 + ] + }, + "oos_auprc":{ + "mean":0.543021, + "std":0.043922, + "values":[ + 0.586862, + 0.499019, + 0.543181 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.040659, + "std":0.01432, + "values":[ + 0.028531, + 0.056457, + 0.036989 + ] + }, + "oos_auroc":{ + "mean":0.788748, + "std":0.018542, + "values":[ + 0.80987, + 0.775153, + 0.78122 + ] + }, + "oos_auprc":{ + "mean":0.091813, + "std":0.007456, + "values":[ + 0.100419, + 0.087724, + 0.087297 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.092646, + "std":0.022292, + "values":[ + 0.071484, + 0.115919, + 0.090536 + ] + }, + "oos_auroc":{ + "mean":0.794558, + "std":0.01156, + "values":[ + 0.804918, + 0.78209, + 0.796667 + ] + }, + "oos_auprc":{ + "mean":0.391667, + "std":0.015806, + "values":[ + 0.405468, + 0.374423, + 0.39511 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.041067, + "std":0.014411, + "values":[ + 0.029022, + 0.057032, + 0.037146 + ] + }, + "oos_auroc":{ + "mean":0.818861, + "std":0.020977, + "values":[ + 0.835497, + 0.795297, + 0.82579 + ] + }, + "oos_auprc":{ + "mean":0.135761, + "std":0.032385, + "values":[ + 0.143951, + 0.100067, + 0.163265 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.089611, + "std":0.021915, + "values":[ + 0.069046, + 0.112663, + 0.087124 + ] + }, + "oos_auroc":{ + "mean":0.826827, + "std":0.012711, + "values":[ + 0.838408, + 0.813228, + 0.828846 + ] + }, + "oos_auprc":{ + "mean":0.478998, + "std":0.025869, + "values":[ + 0.50346, + 0.451921, + 0.481613 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "margin", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":1.99402, + "std":0.962075, + "values":[ + 2.537916, + 2.560953, + 0.88319 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.469677, + "std":0.078016, + "values":[ + 0.55, + 0.394194, + 0.464839 + ] + }, + "validation_selective_risk":{ + "mean":0.013683, + "std":0.00052, + "values":[ + 0.014076, + 0.013093, + 0.013879 + ] + }, + "validation_risk_upper95":{ + "mean":0.019718, + "std":0.000189, + "values":[ + 0.019598, + 0.019619, + 0.019936 + ] + }, + "test":{ + "coverage":{ + "mean":0.400424, + "std":0.067239, + "values":[ + 0.466, + 0.331636, + 0.403636 + ] + }, + "selective_risk":{ + "mean":0.033054, + "std":0.002248, + "values":[ + 0.031994, + 0.035636, + 0.031532 + ] + }, + "accuracy_8":{ + "mean":0.844364, + "std":0.008598, + "values":[ + 0.852364, + 0.835273, + 0.845455 + ] + }, + "oos_recall":{ + "mean":0.558, + "std":0.003606, + "values":[ + 0.554, + 0.559, + 0.561 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.527056, + "std":0.003617, + "values":[ + 0.523043, + 0.528059, + 0.530066 + ] + }, + { + "mean":0.5885, + "std":0.003566, + "values":[ + 0.584544, + 0.58949, + 0.591467 + ] + } + ], + "oos_precision":{ + "mean":0.80276, + "std":0.016143, + "values":[ + 0.819527, + 0.787324, + 0.801429 + ] + }, + "oos_f1":{ + "mean":0.6583, + "std":0.003934, + "values":[ + 0.661098, + 0.653801, + 0.66 + ] + }, + "oos_misroute_rate":{ + "mean":0.442, + "std":0.003606, + "values":[ + 0.446, + 0.441, + 0.439 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.04, + "std":0.006928, + "values":[ + 0.048, + 0.036, + 0.036 + ] + }, + "llm_call_rate":{ + "mean":0.599576, + "std":0.067239, + "values":[ + 0.534, + 0.668364, + 0.596364 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.221344, + "std":0.024878, + "values":[ + 0.197107, + 0.246817, + 0.220109 + ] + }, + "error_recovery_rate":{ + "mean":0.6155, + "std":0.019232, + "values":[ + 0.59364, + 0.623043, + 0.629817 + ] + }, + "recoverable_caught":{ + "mean":0.963721, + "std":0.011422, + "values":[ + 0.951302, + 0.973776, + 0.966085 + ] + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603, + "values":[ + 1415, + 1788, + 1583 + ] + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371, + "values":[ + 883, + 1144, + 1032 + ] + }, + "recovered_errors":{ + "mean":983.666667, + "std":137.485757, + "values":[ + 840, + 1114, + 997 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp", + "margin", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.234831, + "std":0.600639, + "values":[ + 0.54394, + 1.6329, + 1.527653 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.665376, + "std":0.099866, + "values":[ + 0.756129, + 0.558387, + 0.681613 + ] + }, + "validation_selective_risk":{ + "mean":0.041791, + "std":0.000673, + "values":[ + 0.042235, + 0.041017, + 0.04212 + ] + }, + "validation_risk_upper95":{ + "mean":0.049709, + "std":0.000177, + "values":[ + 0.049613, + 0.0496, + 0.049912 + ] + }, + "test":{ + "coverage":{ + "mean":0.584364, + "std":0.090743, + "values":[ + 0.668727, + 0.488364, + 0.596 + ] + }, + "selective_risk":{ + "mean":0.082761, + "std":0.007935, + "values":[ + 0.079391, + 0.077066, + 0.091824 + ] + }, + "accuracy_8":{ + "mean":0.842788, + "std":0.011069, + "values":[ + 0.855091, + 0.833636, + 0.839636 + ] + }, + "oos_recall":{ + "mean":0.520333, + "std":0.015011, + "values":[ + 0.521, + 0.535, + 0.505 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.489359, + "std":0.01499, + "values":[ + 0.490016, + 0.504011, + 0.474051 + ] + }, + { + "mean":0.551152, + "std":0.014917, + "values":[ + 0.551823, + 0.565721, + 0.53591 + ] + } + ], + "oos_precision":{ + "mean":0.834127, + "std":0.023825, + "values":[ + 0.856908, + 0.80938, + 0.836093 + ] + }, + "oos_f1":{ + "mean":0.640625, + "std":0.009673, + "values":[ + 0.64801, + 0.64419, + 0.629676 + ] + }, + "oos_misroute_rate":{ + "mean":0.479667, + "std":0.015011, + "values":[ + 0.479, + 0.465, + 0.495 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.149, + "std":0.037323, + "values":[ + 0.158, + 0.108, + 0.181 + ] + }, + "llm_call_rate":{ + "mean":0.415636, + "std":0.090743, + "values":[ + 0.331273, + 0.511636, + 0.404 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.15336, + "std":0.033575, + "values":[ + 0.122143, + 0.188879, + 0.149058 + ] + }, + "error_recovery_rate":{ + "mean":0.548183, + "std":0.033186, + "values":[ + 0.516608, + 0.582774, + 0.545167 + ] + }, + "recoverable_caught":{ + "mean":0.858313, + "std":0.045682, + "values":[ + 0.82786, + 0.910839, + 0.83624 + ] + }, + "small_errors":{ + "mean":1595.333333, + "std":186.805603, + "values":[ + 1415, + 1788, + 1583 + ] + }, + "recoverable_errors":{ + "mean":1019.666667, + "std":130.936371, + "values":[ + 883, + 1144, + 1032 + ] + }, + "recovered_errors":{ + "mean":878.666667, + "std":156.090786, + "values":[ + 731, + 1042, + 863 + ] + } + } + } + } + } + } + } + }, + "modernbert/k50":{ + "model":"modernbert", + "k":50, + "oos_train":null, + "kind":"encoder", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "ModernBERT-base-k50-seed42", + "ModernBERT-base-k50-seed43", + "ModernBERT-base-k50-seed44" + ], + "logits_sha256":[ + "86bcc65f899f519154881c6e1ccb10ba468e055b7a26049fc11727fc74347a4c", + "0f72b3ebe5b54dcee8b7857d339de335b31230f6dd06d4f43297730e19d955ee", + "c7427794ce3f205347c4a1bc703409bfece96251974823dcb075bfc2dd3d6da3" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":1.100153, + "std":0.031755, + "values":[ + 1.125663, + 1.110209, + 1.064588 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":{ + "ece_151":{ + "mean":0.036147, + "std":0.006214, + "values":[ + 0.029363, + 0.041565, + 0.037514 + ] + }, + "ece_8_summed":{ + "mean":0.039586, + "std":0.003979, + "values":[ + 0.035375, + 0.040099, + 0.043282 + ] + } + }, + "calibrated":{ + "ece_151":{ + "mean":0.028044, + "std":0.005936, + "values":[ + 0.021282, + 0.032395, + 0.030456 + ] + }, + "ece_8_summed":{ + "mean":0.031585, + "std":0.005319, + "values":[ + 0.026226, + 0.031665, + 0.036863 + ] + } + } + } + }, + "selected_aggregation":"argmax", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.978817, + "std":0.001624, + "values":[ + 0.977097, + 0.980323, + 0.979032 + ] + }, + "summed":{ + "mean":0.977742, + "std":0.001406, + "values":[ + 0.976129, + 0.978387, + 0.97871 + ] + } + }, + "signals_reported":[ + "msp", + "entropy", + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.978817, + "std":0.001624 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.096061, + "std":0.003772 + }, + "accuracy_8":{ + "mean":0.903939, + "std":0.003772 + }, + "oos_recall":{ + "mean":0.538667, + "std":0.023245 + }, + "oos_recall_wilson95":[ + { + "mean":0.507702, + "std":0.023284 + }, + { + "mean":0.569335, + "std":0.023029 + } + ], + "oos_precision":{ + "mean":0.97869, + "std":0.005135 + }, + "oos_f1":{ + "mean":0.694719, + "std":0.020376 + }, + "oos_misroute_rate":{ + "mean":0.461333, + "std":0.023245 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.461333, + "std":0.023245 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.903939, + "std":0.003772 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.942667, + "std":0.002184 + }, + "oos_recall":{ + "mean":0.715, + "std":0.011533 + }, + "oos_recall_wilson95":[ + { + "mean":0.686247, + "std":0.011825 + }, + { + "mean":0.742107, + "std":0.011151 + } + ], + "oos_precision":{ + "mean":0.983497, + "std":0.002329 + }, + "oos_f1":{ + "mean":0.827991, + "std":0.007813 + }, + "oos_misroute_rate":{ + "mean":0.285, + "std":0.011533 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.096061, + "std":0.003772 + }, + "llm_cost_usd_per_1k":{ + "mean":0.035438, + "std":0.001393 + }, + "error_recovery_rate":{ + "mean":0.402943, + "std":0.018545 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":213.0, + "std":15.620499 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.001978, + "std":0.000705 + }, + "oos_auroc":{ + "mean":0.890846, + "std":0.021822 + }, + "oos_auprc":{ + "mean":0.221927, + "std":0.016059 + } + }, + "test":{ + "aurc":{ + "mean":0.012497, + "std":0.000589 + }, + "oos_auroc":{ + "mean":0.920849, + "std":0.011725 + }, + "oos_auprc":{ + "mean":0.704655, + "std":0.018646 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.001931, + "std":0.000664 + }, + "oos_auroc":{ + "mean":0.898956, + "std":0.021927 + }, + "oos_auprc":{ + "mean":0.263933, + "std":0.005733 + } + }, + "test":{ + "aurc":{ + "mean":0.012358, + "std":0.00063 + }, + "oos_auroc":{ + "mean":0.928056, + "std":0.01188 + }, + "oos_auprc":{ + "mean":0.738685, + "std":0.021142 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.00214, + "std":0.000825 + }, + "oos_auroc":{ + "mean":0.866099, + "std":0.024075 + }, + "oos_auprc":{ + "mean":0.160217, + "std":0.011689 + } + }, + "test":{ + "aurc":{ + "mean":0.013573, + "std":0.000445 + }, + "oos_auroc":{ + "mean":0.900579, + "std":0.012252 + }, + "oos_auprc":{ + "mean":0.617099, + "std":0.013193 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.001955, + "std":0.000669 + }, + "oos_auroc":{ + "mean":0.894217, + "std":0.020713 + }, + "oos_auprc":{ + "mean":0.230632, + "std":0.015107 + } + }, + "test":{ + "aurc":{ + "mean":0.01241, + "std":0.000622 + }, + "oos_auroc":{ + "mean":0.923503, + "std":0.010961 + }, + "oos_auprc":{ + "mean":0.713682, + "std":0.016343 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.350617, + "std":0.073413 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.987204, + "std":0.005439 + }, + "validation_selective_risk":{ + "mean":0.015685, + "std":8.7e-05 + }, + "validation_risk_upper95":{ + "mean":0.01983, + "std":0.000109 + }, + "test":{ + "coverage":{ + "mean":0.961455, + "std":0.016545 + }, + "selective_risk":{ + "mean":0.072151, + "std":0.013156 + }, + "accuracy_8":{ + "mean":0.911515, + "std":0.005407 + }, + "oos_recall":{ + "mean":0.576, + "std":0.030512 + }, + "oos_recall_wilson95":[ + { + "mean":0.545176, + "std":0.030709 + }, + { + "mean":0.606242, + "std":0.030082 + } + ], + "oos_precision":{ + "mean":0.976213, + "std":0.00335 + }, + "oos_f1":{ + "mean":0.72422, + "std":0.024509 + }, + "oos_misroute_rate":{ + "mean":0.424, + "std":0.030512 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.328667, + "std":0.070316 + }, + "llm_call_rate":{ + "mean":0.038545, + "std":0.016545 + }, + "llm_cost_usd_per_1k":{ + "mean":0.014209, + "std":0.006101 + }, + "error_recovery_rate":{ + "mean":0.123, + "std":0.048153 + }, + "recoverable_caught":{ + "mean":0.306478, + "std":0.122333 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":64.333333, + "std":22.501852 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.350617, + "std":0.073413 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.987204, + "std":0.005439 + }, + "validation_selective_risk":{ + "mean":0.015685, + "std":8.7e-05 + }, + "validation_risk_upper95":{ + "mean":0.01983, + "std":0.000109 + }, + "test":{ + "coverage":{ + "mean":0.961455, + "std":0.016545 + }, + "selective_risk":{ + "mean":0.072151, + "std":0.013156 + }, + "accuracy_8":{ + "mean":0.911515, + "std":0.005407 + }, + "oos_recall":{ + "mean":0.576, + "std":0.030512 + }, + "oos_recall_wilson95":[ + { + "mean":0.545176, + "std":0.030709 + }, + { + "mean":0.606242, + "std":0.030082 + } + ], + "oos_precision":{ + "mean":0.976213, + "std":0.00335 + }, + "oos_f1":{ + "mean":0.72422, + "std":0.024509 + }, + "oos_misroute_rate":{ + "mean":0.424, + "std":0.030512 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.328667, + "std":0.070316 + }, + "llm_call_rate":{ + "mean":0.038545, + "std":0.016545 + }, + "llm_cost_usd_per_1k":{ + "mean":0.014209, + "std":0.006101 + }, + "error_recovery_rate":{ + "mean":0.123, + "std":0.048153 + }, + "recoverable_caught":{ + "mean":0.306478, + "std":0.122333 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":64.333333, + "std":22.501852 + } + } + }, + "entropy":{ + "tau":{ + "mean":-2.401472, + "std":0.30977 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.986129, + "std":0.00703 + }, + "validation_selective_risk":{ + "mean":0.015592, + "std":8e-05 + }, + "validation_risk_upper95":{ + "mean":0.01973, + "std":7.3e-05 + }, + "test":{ + "coverage":{ + "mean":0.954485, + "std":0.016195 + }, + "selective_risk":{ + "mean":0.069386, + "std":0.012655 + }, + "accuracy_8":{ + "mean":0.911212, + "std":0.004914 + }, + "oos_recall":{ + "mean":0.573667, + "std":0.026312 + }, + "oos_recall_wilson95":[ + { + "mean":0.54282, + "std":0.026465 + }, + { + "mean":0.60395, + "std":0.025958 + } + ], + "oos_precision":{ + "mean":0.975543, + "std":0.002907 + }, + "oos_f1":{ + "mean":0.722274, + "std":0.021575 + }, + "oos_misroute_rate":{ + "mean":0.426333, + "std":0.026312 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.312, + "std":0.067089 + }, + "llm_call_rate":{ + "mean":0.045515, + "std":0.016195 + }, + "llm_cost_usd_per_1k":{ + "mean":0.016776, + "std":0.005972 + }, + "error_recovery_rate":{ + "mean":0.137945, + "std":0.040347 + }, + "recoverable_caught":{ + "mean":0.343538, + "std":0.103917 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":72.333333, + "std":18.147543 + } + } + }, + "margin":{ + "tau":{ + "mean":0.458171, + "std":0.226048 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.981075, + "std":0.007643 + }, + "validation_selective_risk":{ + "mean":0.015673, + "std":6.8e-05 + }, + "validation_risk_upper95":{ + "mean":0.019831, + "std":5.8e-05 + }, + "test":{ + "coverage":{ + "mean":0.957394, + "std":0.020918 + }, + "selective_risk":{ + "mean":0.074322, + "std":0.012743 + }, + "accuracy_8":{ + "mean":0.909879, + "std":0.005007 + }, + "oos_recall":{ + "mean":0.572, + "std":0.032047 + }, + "oos_recall_wilson95":[ + { + "mean":0.541158, + "std":0.03225 + }, + { + "mean":0.60229, + "std":0.031598 + } + ], + "oos_precision":{ + "mean":0.976689, + "std":0.00063 + }, + "oos_f1":{ + "mean":0.721114, + "std":0.02502 + }, + "oos_misroute_rate":{ + "mean":0.428, + "std":0.032047 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.341, + "std":0.072746 + }, + "llm_call_rate":{ + "mean":0.042606, + "std":0.020918 + }, + "llm_cost_usd_per_1k":{ + "mean":0.01571, + "std":0.007702 + }, + "error_recovery_rate":{ + "mean":0.116091, + "std":0.053677 + }, + "recoverable_caught":{ + "mean":0.288409, + "std":0.134188 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":60.666667, + "std":25.57994 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.309316, + "std":0.06292 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.986989, + "std":0.005553 + }, + "validation_selective_risk":{ + "mean":0.015688, + "std":8.9e-05 + }, + "validation_risk_upper95":{ + "mean":0.019835, + "std":0.000112 + }, + "test":{ + "coverage":{ + "mean":0.959879, + "std":0.018195 + }, + "selective_risk":{ + "mean":0.071425, + "std":0.01387 + }, + "accuracy_8":{ + "mean":0.911636, + "std":0.005566 + }, + "oos_recall":{ + "mean":0.576667, + "std":0.030238 + }, + "oos_recall_wilson95":[ + { + "mean":0.545846, + "std":0.030433 + }, + { + "mean":0.606901, + "std":0.029812 + } + ], + "oos_precision":{ + "mean":0.976193, + "std":0.004116 + }, + "oos_f1":{ + "mean":0.724764, + "std":0.024711 + }, + "oos_misroute_rate":{ + "mean":0.423333, + "std":0.030238 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.325, + "std":0.073301 + }, + "llm_call_rate":{ + "mean":0.040121, + "std":0.018195 + }, + "llm_cost_usd_per_1k":{ + "mean":0.014789, + "std":0.00671 + }, + "error_recovery_rate":{ + "mean":0.125571, + "std":0.049667 + }, + "recoverable_caught":{ + "mean":0.313014, + "std":0.126386 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":65.666667, + "std":23.245071 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "entropy", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":-2.411483, + "std":2.10235 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.021183, + "std":0.001624 + }, + "validation_risk_upper95":{ + "mean":0.025871, + "std":0.00178 + }, + "test":{ + "coverage":{ + "mean":0.998848, + "std":0.001183 + }, + "selective_risk":{ + "mean":0.095259, + "std":0.004495 + }, + "accuracy_8":{ + "mean":0.904242, + "std":0.004265 + }, + "oos_recall":{ + "mean":0.540333, + "std":0.026083 + }, + "oos_recall_wilson95":[ + { + "mean":0.509377, + "std":0.026135 + }, + { + "mean":0.570981, + "std":0.025832 + } + ], + "oos_precision":{ + "mean":0.978125, + "std":0.005526 + }, + "oos_f1":{ + "mean":0.695925, + "std":0.022851 + }, + "oos_misroute_rate":{ + "mean":0.459667, + "std":0.026083 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.456667, + "std":0.028449 + }, + "llm_call_rate":{ + "mean":0.001152, + "std":0.001183 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000424, + "std":0.000436 + }, + "error_recovery_rate":{ + "mean":0.003294, + "std":0.005705 + }, + "recoverable_caught":{ + "mean":0.00821, + "std":0.01422 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":1.666667, + "std":2.886751 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.116744, + "std":0.016312 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.021183, + "std":0.001624 + }, + "validation_risk_upper95":{ + "mean":0.025871, + "std":0.00178 + }, + "test":{ + "coverage":{ + "mean":0.998788, + "std":0.000525 + }, + "selective_risk":{ + "mean":0.095084, + "std":0.004146 + }, + "accuracy_8":{ + "mean":0.904364, + "std":0.004166 + }, + "oos_recall":{ + "mean":0.541, + "std":0.025515 + }, + "oos_recall_wilson95":[ + { + "mean":0.510043, + "std":0.025567 + }, + { + "mean":0.571643, + "std":0.025267 + } + ], + "oos_precision":{ + "mean":0.977561, + "std":0.006178 + }, + "oos_f1":{ + "mean":0.696347, + "std":0.022455 + }, + "oos_misroute_rate":{ + "mean":0.459, + "std":0.025515 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.456, + "std":0.026287 + }, + "llm_call_rate":{ + "mean":0.001212, + "std":0.000525 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000446, + "std":0.000193 + }, + "error_recovery_rate":{ + "mean":0.005156, + "std":0.004204 + }, + "recoverable_caught":{ + "mean":0.012905, + "std":0.010511 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":2.666667, + "std":2.081666 + } + } + }, + "entropy":{ + "tau":{ + "mean":-3.651834, + "std":0.171912 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.021183, + "std":0.001624 + }, + "validation_risk_upper95":{ + "mean":0.025871, + "std":0.00178 + }, + "test":{ + "coverage":{ + "mean":0.99903, + "std":0.001238 + }, + "selective_risk":{ + "mean":0.095301, + "std":0.004546 + }, + "accuracy_8":{ + "mean":0.904364, + "std":0.00411 + }, + "oos_recall":{ + "mean":0.541, + "std":0.025357 + }, + "oos_recall_wilson95":[ + { + "mean":0.510043, + "std":0.02541 + }, + { + "mean":0.571644, + "std":0.025111 + } + ], + "oos_precision":{ + "mean":0.978764, + "std":0.005199 + }, + "oos_f1":{ + "mean":0.696646, + "std":0.022084 + }, + "oos_misroute_rate":{ + "mean":0.459, + "std":0.025357 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.456667, + "std":0.028449 + }, + "llm_call_rate":{ + "mean":0.00097, + "std":0.001238 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000357, + "std":0.000456 + }, + "error_recovery_rate":{ + "mean":0.004513, + "std":0.004996 + }, + "recoverable_caught":{ + "mean":0.011096, + "std":0.012495 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":2.333333, + "std":2.516611 + } + } + }, + "margin":{ + "tau":{ + "mean":0.003608, + "std":0.004438 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.021183, + "std":0.001624 + }, + "validation_risk_upper95":{ + "mean":0.025871, + "std":0.00178 + }, + "test":{ + "coverage":{ + "mean":0.999636, + "std":0.00063 + }, + "selective_risk":{ + "mean":0.095854, + "std":0.0035 + }, + "accuracy_8":{ + "mean":0.903939, + "std":0.003772 + }, + "oos_recall":{ + "mean":0.538667, + "std":0.023245 + }, + "oos_recall_wilson95":[ + { + "mean":0.507702, + "std":0.023284 + }, + { + "mean":0.569335, + "std":0.023029 + } + ], + "oos_precision":{ + "mean":0.97808, + "std":0.005449 + }, + "oos_f1":{ + "mean":0.694572, + "std":0.02054 + }, + "oos_misroute_rate":{ + "mean":0.461333, + "std":0.023245 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.460333, + "std":0.022143 + }, + "llm_call_rate":{ + "mean":0.000364, + "std":0.00063 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000134, + "std":0.000232 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.100762, + "std":0.010261 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.021183, + "std":0.001624 + }, + "validation_risk_upper95":{ + "mean":0.025871, + "std":0.00178 + }, + "test":{ + "coverage":{ + "mean":0.998848, + "std":0.00042 + }, + "selective_risk":{ + "mean":0.095139, + "std":0.004055 + }, + "accuracy_8":{ + "mean":0.904303, + "std":0.004067 + }, + "oos_recall":{ + "mean":0.540667, + "std":0.024947 + }, + "oos_recall_wilson95":[ + { + "mean":0.509708, + "std":0.024996 + }, + { + "mean":0.571314, + "std":0.024706 + } + ], + "oos_precision":{ + "mean":0.977552, + "std":0.006163 + }, + "oos_f1":{ + "mean":0.696077, + "std":0.021993 + }, + "oos_misroute_rate":{ + "mean":0.459333, + "std":0.024947 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.456333, + "std":0.025716 + }, + "llm_call_rate":{ + "mean":0.001152, + "std":0.00042 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000424, + "std":0.000154 + }, + "error_recovery_rate":{ + "mean":0.004498, + "std":0.003105 + }, + "recoverable_caught":{ + "mean":0.011263, + "std":0.007798 + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499 + }, + "recovered_errors":{ + "mean":2.333333, + "std":1.527525 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.977742, + "std":0.001406 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.102727, + "std":0.002278 + }, + "accuracy_8":{ + "mean":0.897273, + "std":0.002278 + }, + "oos_recall":{ + "mean":0.497, + "std":0.016643 + }, + "oos_recall_wilson95":[ + { + "mean":0.466092, + "std":0.016582 + }, + { + "mean":0.527931, + "std":0.016577 + } + ], + "oos_precision":{ + "mean":0.983519, + "std":0.00498 + }, + "oos_f1":{ + "mean":0.660211, + "std":0.014729 + }, + "oos_misroute_rate":{ + "mean":0.503, + "std":0.016643 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.503, + "std":0.016643 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.897273, + "std":0.002278 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.939212, + "std":0.001728 + }, + "oos_recall":{ + "mean":0.695333, + "std":0.009609 + }, + "oos_recall_wilson95":[ + { + "mean":0.666108, + "std":0.009822 + }, + { + "mean":0.723064, + "std":0.009323 + } + ], + "oos_precision":{ + "mean":0.984894, + "std":0.001587 + }, + "oos_f1":{ + "mean":0.815137, + "std":0.006622 + }, + "oos_misroute_rate":{ + "mean":0.304667, + "std":0.009609 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.102727, + "std":0.002278 + }, + "llm_cost_usd_per_1k":{ + "mean":0.037892, + "std":0.000844 + }, + "error_recovery_rate":{ + "mean":0.408201, + "std":0.014695 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":230.666667, + "std":10.969655 + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.001789, + "std":0.000561 + }, + "oos_auroc":{ + "mean":0.927162, + "std":0.015978 + }, + "oos_auprc":{ + "mean":0.335988, + "std":0.018371 + } + }, + "test":{ + "aurc":{ + "mean":0.012335, + "std":0.000448 + }, + "oos_auroc":{ + "mean":0.947289, + "std":0.009277 + }, + "oos_auprc":{ + "mean":0.793883, + "std":0.015694 + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.00179, + "std":0.000556 + }, + "oos_auroc":{ + "mean":0.929251, + "std":0.016415 + }, + "oos_auprc":{ + "mean":0.367993, + "std":0.013878 + } + }, + "test":{ + "aurc":{ + "mean":0.012358, + "std":0.000455 + }, + "oos_auroc":{ + "mean":0.949315, + "std":0.009368 + }, + "oos_auprc":{ + "mean":0.80932, + "std":0.019132 + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.00184, + "std":0.000554 + }, + "oos_auroc":{ + "mean":0.921091, + "std":0.015547 + }, + "oos_auprc":{ + "mean":0.291455, + "std":0.006122 + } + }, + "test":{ + "aurc":{ + "mean":0.012875, + "std":0.000402 + }, + "oos_auroc":{ + "mean":0.941793, + "std":0.009097 + }, + "oos_auprc":{ + "mean":0.759398, + "std":0.018709 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.001787, + "std":0.000539 + }, + "oos_auroc":{ + "mean":0.928946, + "std":0.01557 + }, + "oos_auprc":{ + "mean":0.349502, + "std":0.024961 + } + }, + "test":{ + "aurc":{ + "mean":0.012438, + "std":0.000441 + }, + "oos_auroc":{ + "mean":0.948736, + "std":0.008806 + }, + "oos_auprc":{ + "mean":0.801503, + "std":0.015014 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.512989, + "std":0.056757 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.988602, + "std":0.00411 + }, + "validation_selective_risk":{ + "mean":0.015663, + "std":6.5e-05 + }, + "validation_risk_upper95":{ + "mean":0.019802, + "std":8.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.951879, + "std":0.014161 + }, + "selective_risk":{ + "mean":0.071891, + "std":0.01202 + }, + "accuracy_8":{ + "mean":0.909091, + "std":0.004902 + }, + "oos_recall":{ + "mean":0.559, + "std":0.028583 + }, + "oos_recall_wilson95":[ + { + "mean":0.528093, + "std":0.028713 + }, + { + "mean":0.589455, + "std":0.028234 + } + ], + "oos_precision":{ + "mean":0.980046, + "std":0.004549 + }, + "oos_f1":{ + "mean":0.711675, + "std":0.023735 + }, + "oos_misroute_rate":{ + "mean":0.441, + "std":0.028583 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.33, + "std":0.062642 + }, + "llm_call_rate":{ + "mean":0.048121, + "std":0.014161 + }, + "llm_cost_usd_per_1k":{ + "mean":0.017722, + "std":0.005218 + }, + "error_recovery_rate":{ + "mean":0.165828, + "std":0.049456 + }, + "recoverable_caught":{ + "mean":0.405842, + "std":0.117873 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":93.333333, + "std":25.813433 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.512989, + "std":0.056757 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.988602, + "std":0.00411 + }, + "validation_selective_risk":{ + "mean":0.015663, + "std":6.5e-05 + }, + "validation_risk_upper95":{ + "mean":0.019802, + "std":8.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.951879, + "std":0.014161 + }, + "selective_risk":{ + "mean":0.071891, + "std":0.01202 + }, + "accuracy_8":{ + "mean":0.909091, + "std":0.004902 + }, + "oos_recall":{ + "mean":0.559, + "std":0.028583 + }, + "oos_recall_wilson95":[ + { + "mean":0.528093, + "std":0.028713 + }, + { + "mean":0.589455, + "std":0.028234 + } + ], + "oos_precision":{ + "mean":0.980046, + "std":0.004549 + }, + "oos_f1":{ + "mean":0.711675, + "std":0.023735 + }, + "oos_misroute_rate":{ + "mean":0.441, + "std":0.028583 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.33, + "std":0.062642 + }, + "llm_call_rate":{ + "mean":0.048121, + "std":0.014161 + }, + "llm_cost_usd_per_1k":{ + "mean":0.017722, + "std":0.005218 + }, + "error_recovery_rate":{ + "mean":0.165828, + "std":0.049456 + }, + "recoverable_caught":{ + "mean":0.405842, + "std":0.117873 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":93.333333, + "std":25.813433 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.321855, + "std":0.076498 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.988172, + "std":0.004033 + }, + "validation_selective_risk":{ + "mean":0.015669, + "std":6.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.019811, + "std":8.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.949273, + "std":0.008868 + }, + "selective_risk":{ + "mean":0.069917, + "std":0.009255 + }, + "accuracy_8":{ + "mean":0.910182, + "std":0.003963 + }, + "oos_recall":{ + "mean":0.565333, + "std":0.024846 + }, + "oos_recall_wilson95":[ + { + "mean":0.534443, + "std":0.024976 + }, + { + "mean":0.595724, + "std":0.024527 + } + ], + "oos_precision":{ + "mean":0.979152, + "std":0.003509 + }, + "oos_f1":{ + "mean":0.716615, + "std":0.020469 + }, + "oos_misroute_rate":{ + "mean":0.434667, + "std":0.024846 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.314, + "std":0.050478 + }, + "llm_call_rate":{ + "mean":0.050727, + "std":0.008868 + }, + "llm_cost_usd_per_1k":{ + "mean":0.018682, + "std":0.003269 + }, + "error_recovery_rate":{ + "mean":0.177352, + "std":0.032787 + }, + "recoverable_caught":{ + "mean":0.433961, + "std":0.074804 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":100.0, + "std":16.703293 + } + } + }, + "margin":{ + "tau":{ + "mean":0.636361, + "std":0.279525 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.986989, + "std":0.005225 + }, + "validation_selective_risk":{ + "mean":0.015688, + "std":8.3e-05 + }, + "validation_risk_upper95":{ + "mean":0.019835, + "std":0.000105 + }, + "test":{ + "coverage":{ + "mean":0.952182, + "std":0.021821 + }, + "selective_risk":{ + "mean":0.074169, + "std":0.015588 + }, + "accuracy_8":{ + "mean":0.90697, + "std":0.004403 + }, + "oos_recall":{ + "mean":0.547333, + "std":0.030039 + }, + "oos_recall_wilson95":[ + { + "mean":0.516397, + "std":0.030133 + }, + { + "mean":0.577907, + "std":0.029715 + } + ], + "oos_precision":{ + "mean":0.980376, + "std":0.002625 + }, + "oos_f1":{ + "mean":0.702134, + "std":0.023946 + }, + "oos_misroute_rate":{ + "mean":0.452667, + "std":0.030039 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.346333, + "std":0.08529 + }, + "llm_call_rate":{ + "mean":0.047818, + "std":0.021821 + }, + "llm_cost_usd_per_1k":{ + "mean":0.017616, + "std":0.008043 + }, + "error_recovery_rate":{ + "mean":0.150777, + "std":0.065007 + }, + "recoverable_caught":{ + "mean":0.369625, + "std":0.15774 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":84.666667, + "std":34.588052 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.48039, + "std":0.037704 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.988065, + "std":0.003924 + }, + "validation_selective_risk":{ + "mean":0.015671, + "std":6.2e-05 + }, + "validation_risk_upper95":{ + "mean":0.019813, + "std":7.9e-05 + }, + "test":{ + "coverage":{ + "mean":0.948606, + "std":0.013043 + }, + "selective_risk":{ + "mean":0.071009, + "std":0.010977 + }, + "accuracy_8":{ + "mean":0.909212, + "std":0.004572 + }, + "oos_recall":{ + "mean":0.559, + "std":0.026963 + }, + "oos_recall_wilson95":[ + { + "mean":0.528089, + "std":0.027083 + }, + { + "mean":0.589459, + "std":0.026637 + } + ], + "oos_precision":{ + "mean":0.980643, + "std":0.003457 + }, + "oos_f1":{ + "mean":0.711854, + "std":0.022255 + }, + "oos_misroute_rate":{ + "mean":0.441, + "std":0.026963 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.323333, + "std":0.058398 + }, + "llm_call_rate":{ + "mean":0.051394, + "std":0.013043 + }, + "llm_cost_usd_per_1k":{ + "mean":0.018925, + "std":0.004806 + }, + "error_recovery_rate":{ + "mean":0.173454, + "std":0.04433 + }, + "recoverable_caught":{ + "mean":0.424809, + "std":0.106142 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":97.666667, + "std":22.810816 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp", + "msp_t", + "msp" + ], + "hybrid":{ + "tau":{ + "mean":0.242781, + "std":0.034221 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001406 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001537 + }, + "test":{ + "coverage":{ + "mean":0.997939, + "std":0.00164 + }, + "selective_risk":{ + "mean":0.101357, + "std":0.003505 + }, + "accuracy_8":{ + "mean":0.898182, + "std":0.003107 + }, + "oos_recall":{ + "mean":0.502, + "std":0.021166 + }, + "oos_recall_wilson95":[ + { + "mean":0.47108, + "std":0.021104 + }, + { + "mean":0.532905, + "std":0.021066 + } + ], + "oos_precision":{ + "mean":0.98368, + "std":0.004932 + }, + "oos_f1":{ + "mean":0.664581, + "std":0.018536 + }, + "oos_misroute_rate":{ + "mean":0.498, + "std":0.021166 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.494333, + "std":0.024194 + }, + "llm_call_rate":{ + "mean":0.002061, + "std":0.00164 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000756, + "std":0.000601 + }, + "error_recovery_rate":{ + "mean":0.01018, + "std":0.01036 + }, + "recoverable_caught":{ + "mean":0.024999, + "std":0.025113 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":5.666667, + "std":5.686241 + } + } + }, + "by_signal":{ + "msp":{ + "tau":{ + "mean":0.242522, + "std":0.034306 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001406 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001537 + }, + "test":{ + "coverage":{ + "mean":0.998242, + "std":0.001689 + }, + "selective_risk":{ + "mean":0.101326, + "std":0.003499 + }, + "accuracy_8":{ + "mean":0.898182, + "std":0.003107 + }, + "oos_recall":{ + "mean":0.502, + "std":0.021166 + }, + "oos_recall_wilson95":[ + { + "mean":0.47108, + "std":0.021104 + }, + { + "mean":0.532905, + "std":0.021066 + } + ], + "oos_precision":{ + "mean":0.98368, + "std":0.004932 + }, + "oos_f1":{ + "mean":0.664581, + "std":0.018536 + }, + "oos_misroute_rate":{ + "mean":0.498, + "std":0.021166 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.494333, + "std":0.024194 + }, + "llm_call_rate":{ + "mean":0.001758, + "std":0.001689 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000645, + "std":0.000619 + }, + "error_recovery_rate":{ + "mean":0.01018, + "std":0.01036 + }, + "recoverable_caught":{ + "mean":0.024999, + "std":0.025113 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":5.666667, + "std":5.686241 + } + } + }, + "entropy":{ + "tau":{ + "mean":-1.879365, + "std":0.065926 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001406 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001537 + }, + "test":{ + "coverage":{ + "mean":0.997091, + "std":0.00317 + }, + "selective_risk":{ + "mean":0.10083, + "std":0.004152 + }, + "accuracy_8":{ + "mean":0.898424, + "std":0.003379 + }, + "oos_recall":{ + "mean":0.503333, + "std":0.02318 + }, + "oos_recall_wilson95":[ + { + "mean":0.472412, + "std":0.02312 + }, + { + "mean":0.534229, + "std":0.023064 + } + ], + "oos_precision":{ + "mean":0.983705, + "std":0.005016 + }, + "oos_f1":{ + "mean":0.665726, + "std":0.020347 + }, + "oos_misroute_rate":{ + "mean":0.496667, + "std":0.02318 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.491, + "std":0.029445 + }, + "llm_call_rate":{ + "mean":0.002909, + "std":0.00317 + }, + "llm_cost_usd_per_1k":{ + "mean":0.001069, + "std":0.001166 + }, + "error_recovery_rate":{ + "mean":0.014988, + "std":0.018416 + }, + "recoverable_caught":{ + "mean":0.036487, + "std":0.044734 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":8.333333, + "std":10.115994 + } + } + }, + "margin":{ + "tau":{ + "mean":0.012661, + "std":0.015284 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001406 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001537 + }, + "test":{ + "coverage":{ + "mean":0.998606, + "std":0.001821 + }, + "selective_risk":{ + "mean":0.101835, + "std":0.003533 + }, + "accuracy_8":{ + "mean":0.897818, + "std":0.003037 + }, + "oos_recall":{ + "mean":0.500333, + "std":0.021572 + }, + "oos_recall_wilson95":[ + { + "mean":0.46942, + "std":0.021505 + }, + { + "mean":0.531244, + "std":0.021473 + } + ], + "oos_precision":{ + "mean":0.983619, + "std":0.00499 + }, + "oos_f1":{ + "mean":0.663099, + "std":0.018976 + }, + "oos_misroute_rate":{ + "mean":0.499667, + "std":0.021572 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.497667, + "std":0.024214 + }, + "llm_call_rate":{ + "mean":0.001394, + "std":0.001821 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000512, + "std":0.00067 + }, + "error_recovery_rate":{ + "mean":0.006024, + "std":0.008947 + }, + "recoverable_caught":{ + "mean":0.014717, + "std":0.021707 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":3.333333, + "std":4.932883 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.228757, + "std":0.031056 + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.022258, + "std":0.001406 + }, + "validation_risk_upper95":{ + "mean":0.027049, + "std":0.001537 + }, + "test":{ + "coverage":{ + "mean":0.997636, + "std":0.00246 + }, + "selective_risk":{ + "mean":0.101385, + "std":0.003783 + }, + "accuracy_8":{ + "mean":0.898182, + "std":0.003288 + }, + "oos_recall":{ + "mean":0.502, + "std":0.022113 + }, + "oos_recall_wilson95":[ + { + "mean":0.471082, + "std":0.022048 + }, + { + "mean":0.532903, + "std":0.022009 + } + ], + "oos_precision":{ + "mean":0.983682, + "std":0.004925 + }, + "oos_f1":{ + "mean":0.664564, + "std":0.019349 + }, + "oos_misroute_rate":{ + "mean":0.498, + "std":0.022113 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.494333, + "std":0.026102 + }, + "llm_call_rate":{ + "mean":0.002364, + "std":0.00246 + }, + "llm_cost_usd_per_1k":{ + "mean":0.000867, + "std":0.000903 + }, + "error_recovery_rate":{ + "mean":0.011414, + "std":0.014099 + }, + "recoverable_caught":{ + "mean":0.028032, + "std":0.034159 + }, + "small_errors":{ + "mean":565.0, + "std":12.529964 + }, + "recoverable_errors":{ + "mean":230.666667, + "std":10.969655 + }, + "recovered_errors":{ + "mean":6.333333, + "std":7.767453 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.096061, + "std":0.003772, + "values":[ + 0.092, + 0.096727, + 0.099455 + ] + }, + "accuracy_8":{ + "mean":0.903939, + "std":0.003772, + "values":[ + 0.908, + 0.903273, + 0.900545 + ] + }, + "oos_recall":{ + "mean":0.538667, + "std":0.023245, + "values":[ + 0.565, + 0.53, + 0.521 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.507702, + "std":0.023284, + "values":[ + 0.534082, + 0.49901, + 0.490016 + ] + }, + { + "mean":0.569335, + "std":0.023029, + "values":[ + 0.595421, + 0.560761, + 0.551823 + ] + } + ], + "oos_precision":{ + "mean":0.97869, + "std":0.005135, + "values":[ + 0.984321, + 0.974265, + 0.977486 + ] + }, + "oos_f1":{ + "mean":0.694719, + "std":0.020376, + "values":[ + 0.717916, + 0.686528, + 0.679713 + ] + }, + "oos_misroute_rate":{ + "mean":0.461333, + "std":0.023245, + "values":[ + 0.435, + 0.47, + 0.479 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.461333, + "std":0.023245, + "values":[ + 0.435, + 0.47, + 0.479 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477, + "values":[ + 506, + 532, + 547 + ] + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499, + "values":[ + 203, + 205, + 231 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.903939, + "std":0.003772, + "values":[ + 0.908, + 0.903273, + 0.900545 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.942667, + "std":0.002184, + "values":[ + 0.944909, + 0.940545, + 0.942545 + ] + }, + "oos_recall":{ + "mean":0.715, + "std":0.011533, + "values":[ + 0.726, + 0.703, + 0.716 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.686247, + "std":0.011825, + "values":[ + 0.697531, + 0.673945, + 0.687265 + ] + }, + { + "mean":0.742107, + "std":0.011151, + "values":[ + 0.75274, + 0.730501, + 0.743081 + ] + } + ], + "oos_precision":{ + "mean":0.983497, + "std":0.002329, + "values":[ + 0.985075, + 0.984594, + 0.980822 + ] + }, + "oos_f1":{ + "mean":0.827991, + "std":0.007813, + "values":[ + 0.835924, + 0.820303, + 0.827746 + ] + }, + "oos_misroute_rate":{ + "mean":0.285, + "std":0.011533, + "values":[ + 0.274, + 0.297, + 0.284 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.096061, + "std":0.003772, + "values":[ + 0.092, + 0.096727, + 0.099455 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.035438, + "std":0.001393, + "values":[ + 0.033941, + 0.035679, + 0.036695 + ] + }, + "error_recovery_rate":{ + "mean":0.402943, + "std":0.018545, + "values":[ + 0.401186, + 0.385338, + 0.422303 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477, + "values":[ + 506, + 532, + 547 + ] + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499, + "values":[ + 203, + 205, + 231 + ] + }, + "recovered_errors":{ + "mean":213.0, + "std":15.620499, + "values":[ + 203, + 205, + 231 + ] + } + }, + "signals":{ + "msp":{ + "validation":{ + "aurc":{ + "mean":0.001978, + "std":0.000705, + "values":[ + 0.001641, + 0.002788, + 0.001506 + ] + }, + "oos_auroc":{ + "mean":0.890846, + "std":0.021822, + "values":[ + 0.86848, + 0.891977, + 0.91208 + ] + }, + "oos_auprc":{ + "mean":0.221927, + "std":0.016059, + "values":[ + 0.231666, + 0.203392, + 0.230723 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.012497, + "std":0.000589, + "values":[ + 0.011816, + 0.012828, + 0.012846 + ] + }, + "oos_auroc":{ + "mean":0.920849, + "std":0.011725, + "values":[ + 0.9075, + 0.925569, + 0.929479 + ] + }, + "oos_auprc":{ + "mean":0.704655, + "std":0.018646, + "values":[ + 0.687304, + 0.70229, + 0.724371 + ] + } + } + }, + "entropy":{ + "validation":{ + "aurc":{ + "mean":0.001931, + "std":0.000664, + "values":[ + 0.001564, + 0.002698, + 0.001531 + ] + }, + "oos_auroc":{ + "mean":0.898956, + "std":0.021927, + "values":[ + 0.876053, + 0.901057, + 0.919757 + ] + }, + "oos_auprc":{ + "mean":0.263933, + "std":0.005733, + "values":[ + 0.260194, + 0.261072, + 0.270532 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.012358, + "std":0.00063, + "values":[ + 0.011645, + 0.012592, + 0.012838 + ] + }, + "oos_auroc":{ + "mean":0.928056, + "std":0.01188, + "values":[ + 0.914499, + 0.933016, + 0.936652 + ] + }, + "oos_auprc":{ + "mean":0.738685, + "std":0.021142, + "values":[ + 0.717093, + 0.739613, + 0.759348 + ] + } + } + }, + "margin":{ + "validation":{ + "aurc":{ + "mean":0.00214, + "std":0.000825, + "values":[ + 0.001888, + 0.003062, + 0.001471 + ] + }, + "oos_auroc":{ + "mean":0.866099, + "std":0.024075, + "values":[ + 0.84311, + 0.864057, + 0.89113 + ] + }, + "oos_auprc":{ + "mean":0.160217, + "std":0.011689, + "values":[ + 0.148851, + 0.159595, + 0.172205 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.013573, + "std":0.000445, + "values":[ + 0.013117, + 0.014006, + 0.013595 + ] + }, + "oos_auroc":{ + "mean":0.900579, + "std":0.012252, + "values":[ + 0.886495, + 0.906456, + 0.908785 + ] + }, + "oos_auprc":{ + "mean":0.617099, + "std":0.013193, + "values":[ + 0.604753, + 0.615543, + 0.631001 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.001955, + "std":0.000669, + "values":[ + 0.00161, + 0.002726, + 0.001529 + ] + }, + "oos_auroc":{ + "mean":0.894217, + "std":0.020713, + "values":[ + 0.872643, + 0.89606, + 0.913947 + ] + }, + "oos_auprc":{ + "mean":0.230632, + "std":0.015107, + "values":[ + 0.241731, + 0.213428, + 0.236737 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.01241, + "std":0.000622, + "values":[ + 0.011693, + 0.012728, + 0.01281 + ] + }, + "oos_auroc":{ + "mean":0.923503, + "std":0.010961, + "values":[ + 0.910956, + 0.928344, + 0.93121 + ] + }, + "oos_auprc":{ + "mean":0.713682, + "std":0.016343, + "values":[ + 0.697912, + 0.712591, + 0.730543 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp", + "hybrid":{ + "tau":{ + "mean":0.350617, + "std":0.073413, + "values":[ + 0.433208, + 0.325855, + 0.292786 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.987204, + "std":0.005439, + "values":[ + 0.980968, + 0.989677, + 0.990968 + ] + }, + "validation_selective_risk":{ + "mean":0.015685, + "std":8.7e-05, + "values":[ + 0.015784, + 0.015645, + 0.015625 + ] + }, + "validation_risk_upper95":{ + "mean":0.01983, + "std":0.000109, + "values":[ + 0.019956, + 0.01978, + 0.019755 + ] + }, + "test":{ + "coverage":{ + "mean":0.961455, + "std":0.016545, + "values":[ + 0.942545, + 0.968545, + 0.973273 + ] + }, + "selective_risk":{ + "mean":0.072151, + "std":0.013156, + "values":[ + 0.057099, + 0.077905, + 0.08145 + ] + }, + "accuracy_8":{ + "mean":0.911515, + "std":0.005407, + "values":[ + 0.917273, + 0.910727, + 0.906545 + ] + }, + "oos_recall":{ + "mean":0.576, + "std":0.030512, + "values":[ + 0.61, + 0.567, + 0.551 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.545176, + "std":0.030709, + "values":[ + 0.579403, + 0.536091, + 0.520035 + ] + }, + { + "mean":0.606242, + "std":0.030082, + "values":[ + 0.639755, + 0.597397, + 0.581575 + ] + } + ], + "oos_precision":{ + "mean":0.976213, + "std":0.00335, + "values":[ + 0.979133, + 0.972556, + 0.97695 + ] + }, + "oos_f1":{ + "mean":0.72422, + "std":0.024509, + "values":[ + 0.751694, + 0.716361, + 0.704604 + ] + }, + "oos_misroute_rate":{ + "mean":0.424, + "std":0.030512, + "values":[ + 0.39, + 0.433, + 0.449 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.328667, + "std":0.070316, + "values":[ + 0.248, + 0.361, + 0.377 + ] + }, + "llm_call_rate":{ + "mean":0.038545, + "std":0.016545, + "values":[ + 0.057455, + 0.031455, + 0.026727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.014209, + "std":0.006101, + "values":[ + 0.021184, + 0.011583, + 0.009861 + ] + }, + "error_recovery_rate":{ + "mean":0.123, + "std":0.048153, + "values":[ + 0.177866, + 0.103383, + 0.087751 + ] + }, + "recoverable_caught":{ + "mean":0.306478, + "std":0.122333, + "values":[ + 0.44335, + 0.268293, + 0.207792 + ] + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477, + "values":[ + 506, + 532, + 547 + ] + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499, + "values":[ + 203, + 205, + 231 + ] + }, + "recovered_errors":{ + "mean":64.333333, + "std":22.501852, + "values":[ + 90, + 55, + 48 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "entropy", + "entropy", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":-2.411483, + "std":2.10235, + "values":[ + -3.457853, + -3.785324, + 0.008728 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "validation_selective_risk":{ + "mean":0.021183, + "std":0.001624, + "values":[ + 0.022903, + 0.019677, + 0.020968 + ] + }, + "validation_risk_upper95":{ + "mean":0.025871, + "std":0.00178, + "values":[ + 0.027756, + 0.024219, + 0.025637 + ] + }, + "test":{ + "coverage":{ + "mean":0.998848, + "std":0.001183, + "values":[ + 0.997636, + 1.0, + 0.998909 + ] + }, + "selective_risk":{ + "mean":0.095259, + "std":0.004495, + "values":[ + 0.090213, + 0.096727, + 0.098835 + ] + }, + "accuracy_8":{ + "mean":0.904242, + "std":0.004265, + "values":[ + 0.908909, + 0.903273, + 0.900545 + ] + }, + "oos_recall":{ + "mean":0.540333, + "std":0.026083, + "values":[ + 0.57, + 0.53, + 0.521 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.509377, + "std":0.026135, + "values":[ + 0.539105, + 0.49901, + 0.490016 + ] + }, + { + "mean":0.570981, + "std":0.025832, + "values":[ + 0.60036, + 0.560761, + 0.551823 + ] + } + ], + "oos_precision":{ + "mean":0.978125, + "std":0.005526, + "values":[ + 0.984456, + 0.974265, + 0.975655 + ] + }, + "oos_f1":{ + "mean":0.695925, + "std":0.022851, + "values":[ + 0.721976, + 0.686528, + 0.67927 + ] + }, + "oos_misroute_rate":{ + "mean":0.459667, + "std":0.026083, + "values":[ + 0.43, + 0.47, + 0.479 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.456667, + "std":0.028449, + "values":[ + 0.424, + 0.47, + 0.476 + ] + }, + "llm_call_rate":{ + "mean":0.001152, + "std":0.001183, + "values":[ + 0.002364, + 0.0, + 0.001091 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.000424, + "std":0.000436, + "values":[ + 0.000871, + 0.0, + 0.000402 + ] + }, + "error_recovery_rate":{ + "mean":0.003294, + "std":0.005705, + "values":[ + 0.009881, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.00821, + "std":0.01422, + "values":[ + 0.024631, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":528.333333, + "std":20.744477, + "values":[ + 506, + 532, + 547 + ] + }, + "recoverable_errors":{ + "mean":213.0, + "std":15.620499, + "values":[ + 203, + 205, + 231 + ] + }, + "recovered_errors":{ + "mean":1.666667, + "std":2.886751, + "values":[ + 5, + 0, + 0 + ] + } + } + } + } + } + } + } + }, + "tfidf-centroid/k1":{ + "model":"tfidf-centroid", + "k":1, + "oos_train":null, + "kind":"tfidf", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "tfidf-centroid-k1-seed42", + "tfidf-centroid-k1-seed43", + "tfidf-centroid-k1-seed44" + ], + "logits_sha256":[ + "94258de1803c08b0ba3fd2066c15b4814dfb701f499959b33319623e70d6089d", + "d2d86e9f3d9cb80ca61f3323aa17dcb00ce80afbddf52beddf89da9ccb841380", + "fe1f9f4b8c84c652cec9717dd20fd94591d7134748764883360c9fe27ef5dbc5" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":7.203522, + "std":0.110872, + "values":[ + 7.212682, + 7.088354, + 7.30953 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":null, + "calibrated":{ + "ece_151":{ + "mean":0.081398, + "std":0.006929, + "values":[ + 0.080544, + 0.088715, + 0.074936 + ] + }, + "ece_8_summed":{ + "mean":0.050224, + "std":0.015033, + "values":[ + 0.044354, + 0.039011, + 0.067305 + ] + } + } + } + }, + "selected_aggregation":"summed", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.588172, + "std":0.004356, + "values":[ + 0.588065, + 0.592581, + 0.583871 + ] + }, + "summed":{ + "mean":0.602366, + "std":0.013657, + "values":[ + 0.595806, + 0.618065, + 0.593226 + ] + } + }, + "signals_reported":[ + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.588172, + "std":0.004356 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.512121, + "std":0.013706 + }, + "accuracy_8":{ + "mean":0.487879, + "std":0.013706 + }, + "oos_recall":{ + "mean":0.012, + "std":0.005196 + }, + "oos_recall_wilson95":[ + { + "mean":0.006967, + "std":0.003853 + }, + { + "mean":0.020769, + "std":0.0065 + } + ], + "oos_precision":{ + "mean":0.712162, + "std":0.209337 + }, + "oos_f1":{ + "mean":0.023441, + "std":0.009764 + }, + "oos_misroute_rate":{ + "mean":0.988, + "std":0.005196 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.988, + "std":0.005196 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":2816.666667, + "std":75.381253 + }, + "recoverable_errors":{ + "mean":2118.0, + "std":70.14984 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.487879, + "std":0.013706 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.87297, + "std":0.001961 + }, + "oos_recall":{ + "mean":0.572333, + "std":0.002082 + }, + "oos_recall_wilson95":[ + { + "mean":0.54145, + "std":0.002092 + }, + { + "mean":0.602663, + "std":0.002055 + } + ], + "oos_precision":{ + "mean":0.850524, + "std":0.011141 + }, + "oos_f1":{ + "mean":0.684211, + "std":0.003658 + }, + "oos_misroute_rate":{ + "mean":0.427667, + "std":0.002082 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.512121, + "std":0.013706 + }, + "llm_cost_usd_per_1k":{ + "mean":0.18905, + "std":0.005107 + }, + "error_recovery_rate":{ + "mean":0.75187, + "std":0.005603 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2816.666667, + "std":75.381253 + }, + "recoverable_errors":{ + "mean":2118.0, + "std":70.14984 + }, + "recovered_errors":{ + "mean":2118.0, + "std":70.14984 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.257203, + "std":0.010604 + }, + "oos_auroc":{ + "mean":0.630799, + "std":0.03416 + }, + "oos_auprc":{ + "mean":0.052661, + "std":0.009196 + } + }, + "test":{ + "aurc":{ + "mean":0.350108, + "std":0.006691 + }, + "oos_auroc":{ + "mean":0.599883, + "std":0.007753 + }, + "oos_auprc":{ + "mean":0.224888, + "std":0.005803 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.256867, + "std":0.010918 + }, + "oos_auroc":{ + "mean":0.645771, + "std":0.028722 + }, + "oos_auprc":{ + "mean":0.058212, + "std":0.011744 + } + }, + "test":{ + "aurc":{ + "mean":0.346264, + "std":0.006256 + }, + "oos_auroc":{ + "mean":0.623509, + "std":0.009535 + }, + "oos_auprc":{ + "mean":0.244495, + "std":0.00708 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.75187, + "std":0.005603 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2816.666667, + "std":75.381253 + }, + "recoverable_errors":{ + "mean":2118.0, + "std":70.14984 + }, + "recovered_errors":{ + "mean":2118.0, + "std":70.14984 + } + } + }, + "by_signal":{ + "margin":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.75187, + "std":0.005603 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2816.666667, + "std":75.381253 + }, + "recoverable_errors":{ + "mean":2118.0, + "std":70.14984 + }, + "recovered_errors":{ + "mean":2118.0, + "std":70.14984 + } + } + }, + "msp_t":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.75187, + "std":0.005603 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2816.666667, + "std":75.381253 + }, + "recoverable_errors":{ + "mean":2118.0, + "std":70.14984 + }, + "recovered_errors":{ + "mean":2118.0, + "std":70.14984 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp_t", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":2.189096, + "std":2.13581 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.056989, + "std":0.031269 + }, + "validation_selective_risk":{ + "mean":0.017249, + "std":0.015 + }, + "validation_risk_upper95":{ + "mean":0.04557, + "std":0.006788 + }, + "test":{ + "coverage":{ + "mean":0.046788, + "std":0.023617 + }, + "selective_risk":{ + "mean":0.089765, + "std":0.033692 + }, + "accuracy_8":{ + "mean":0.823455, + "std":0.003288 + }, + "oos_recall":{ + "mean":0.563333, + "std":0.001528 + }, + "oos_recall_wilson95":[ + { + "mean":0.532408, + "std":0.001534 + }, + { + "mean":0.593774, + "std":0.00151 + } + ], + "oos_precision":{ + "mean":0.761065, + "std":0.01192 + }, + "oos_f1":{ + "mean":0.647403, + "std":0.003319 + }, + "oos_misroute_rate":{ + "mean":0.436667, + "std":0.001528 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.009667, + "std":0.004933 + }, + "llm_call_rate":{ + "mean":0.953212, + "std":0.023617 + }, + "llm_cost_usd_per_1k":{ + "mean":0.352247, + "std":0.008702 + }, + "error_recovery_rate":{ + "mean":0.746905, + "std":0.006563 + }, + "recoverable_caught":{ + "mean":0.993392, + "std":0.002449 + }, + "small_errors":{ + "mean":2816.666667, + "std":75.381253 + }, + "recoverable_errors":{ + "mean":2118.0, + "std":70.14984 + }, + "recovered_errors":{ + "mean":2104.0, + "std":69.763888 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":5.650929, + "std":1.042755 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.053763, + "std":0.030941 + }, + "validation_selective_risk":{ + "mean":0.016612, + "std":0.014572 + }, + "validation_risk_upper95":{ + "mean":0.046086, + "std":0.005809 + }, + "test":{ + "coverage":{ + "mean":0.045394, + "std":0.023379 + }, + "selective_risk":{ + "mean":0.058391, + "std":0.028289 + }, + "accuracy_8":{ + "mean":0.823758, + "std":0.002699 + }, + "oos_recall":{ + "mean":0.564, + "std":0.003606 + }, + "oos_recall_wilson95":[ + { + "mean":0.533078, + "std":0.003621 + }, + { + "mean":0.594432, + "std":0.003563 + } + ], + "oos_precision":{ + "mean":0.760939, + "std":0.010763 + }, + "oos_f1":{ + "mean":0.647789, + "std":0.00156 + }, + "oos_misroute_rate":{ + "mean":0.436, + "std":0.003606 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.008667, + "std":0.007506 + }, + "llm_call_rate":{ + "mean":0.954606, + "std":0.023379 + }, + "llm_cost_usd_per_1k":{ + "mean":0.352764, + "std":0.008615 + }, + "error_recovery_rate":{ + "mean":0.748004, + "std":0.005669 + }, + "recoverable_caught":{ + "mean":0.994863, + "std":0.00396 + }, + "small_errors":{ + "mean":2816.666667, + "std":75.381253 + }, + "recoverable_errors":{ + "mean":2118.0, + "std":70.14984 + }, + "recovered_errors":{ + "mean":2107.0, + "std":64.815122 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.959628, + "std":0.02309 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.042903, + "std":0.021862 + }, + "validation_selective_risk":{ + "mean":0.013461, + "std":0.012432 + }, + "validation_risk_upper95":{ + "mean":0.044684, + "std":0.006165 + }, + "test":{ + "coverage":{ + "mean":0.034667, + "std":0.018065 + }, + "selective_risk":{ + "mean":0.08497, + "std":0.035962 + }, + "accuracy_8":{ + "mean":0.822788, + "std":0.003588 + }, + "oos_recall":{ + "mean":0.563667, + "std":0.001528 + }, + "oos_recall_wilson95":[ + { + "mean":0.532743, + "std":0.001534 + }, + { + "mean":0.594103, + "std":0.00151 + } + ], + "oos_precision":{ + "mean":0.757775, + "std":0.012518 + }, + "oos_f1":{ + "mean":0.646426, + "std":0.003537 + }, + "oos_misroute_rate":{ + "mean":0.436333, + "std":0.001528 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.007333, + "std":0.004163 + }, + "llm_call_rate":{ + "mean":0.965333, + "std":0.018065 + }, + "llm_cost_usd_per_1k":{ + "mean":0.356713, + "std":0.00666 + }, + "error_recovery_rate":{ + "mean":0.748191, + "std":0.005998 + }, + "recoverable_caught":{ + "mean":0.995104, + "std":0.00088 + }, + "small_errors":{ + "mean":2816.666667, + "std":75.381253 + }, + "recoverable_errors":{ + "mean":2118.0, + "std":70.14984 + }, + "recovered_errors":{ + "mean":2107.666667, + "std":71.451615 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.602366, + "std":0.013657 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.502, + "std":0.011789 + }, + "accuracy_8":{ + "mean":0.498, + "std":0.011789 + }, + "oos_recall":{ + "mean":0.001333, + "std":0.001155 + }, + "oos_recall_wilson95":[ + { + "mean":0.000366, + "std":0.000317 + }, + { + "mean":0.006118, + "std":0.001984 + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.002661, + "std":0.002305 + }, + "oos_misroute_rate":{ + "mean":0.998667, + "std":0.001155 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.998667, + "std":0.001155 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826 + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.498, + "std":0.011789 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.869152, + "std":0.003456 + }, + "oos_recall":{ + "mean":0.568667, + "std":0.001155 + }, + "oos_recall_wilson95":[ + { + "mean":0.537765, + "std":0.00116 + }, + { + "mean":0.599043, + "std":0.00114 + } + ], + "oos_precision":{ + "mean":0.845467, + "std":0.010443 + }, + "oos_f1":{ + "mean":0.679964, + "std":0.004045 + }, + "oos_misroute_rate":{ + "mean":0.431333, + "std":0.001155 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.502, + "std":0.011789 + }, + "llm_cost_usd_per_1k":{ + "mean":0.185303, + "std":0.004398 + }, + "error_recovery_rate":{ + "mean":0.739287, + "std":0.00752 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826 + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971 + }, + "recovered_errors":{ + "mean":2041.333333, + "std":61.32971 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.215991, + "std":0.008655 + }, + "oos_auroc":{ + "mean":0.645843, + "std":0.036771 + }, + "oos_auprc":{ + "mean":0.056653, + "std":0.010635 + } + }, + "test":{ + "aurc":{ + "mean":0.306797, + "std":0.000705 + }, + "oos_auroc":{ + "mean":0.631819, + "std":0.002821 + }, + "oos_auprc":{ + "mean":0.243673, + "std":0.002406 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.219267, + "std":0.008543 + }, + "oos_auroc":{ + "mean":0.647014, + "std":0.030274 + }, + "oos_auprc":{ + "mean":0.064036, + "std":0.024908 + } + }, + "test":{ + "aurc":{ + "mean":0.310728, + "std":0.001868 + }, + "oos_auroc":{ + "mean":0.633958, + "std":0.01085 + }, + "oos_auprc":{ + "mean":0.248729, + "std":0.009254 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.739287, + "std":0.00752 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826 + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971 + }, + "recovered_errors":{ + "mean":2041.333333, + "std":61.32971 + } + } + }, + "by_signal":{ + "margin":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.739287, + "std":0.00752 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826 + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971 + }, + "recovered_errors":{ + "mean":2041.333333, + "std":61.32971 + } + } + }, + "msp_t":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0 + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0 + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0 + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.739287, + "std":0.00752 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826 + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971 + }, + "recovered_errors":{ + "mean":2041.333333, + "std":61.32971 + } + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp_t", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":2.004105, + "std":1.803545 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.052151, + "std":0.025623 + }, + "validation_selective_risk":{ + "mean":0.014448, + "std":0.012566 + }, + "validation_risk_upper95":{ + "mean":0.042589, + "std":0.005131 + }, + "test":{ + "coverage":{ + "mean":0.042727, + "std":0.018584 + }, + "selective_risk":{ + "mean":0.078947, + "std":0.045322 + }, + "accuracy_8":{ + "mean":0.823152, + "std":0.003432 + }, + "oos_recall":{ + "mean":0.564, + "std":0.001732 + }, + "oos_recall_wilson95":[ + { + "mean":0.533078, + "std":0.001739 + }, + { + "mean":0.594432, + "std":0.001712 + } + ], + "oos_precision":{ + "mean":0.758894, + "std":0.012019 + }, + "oos_f1":{ + "mean":0.647054, + "std":0.0033 + }, + "oos_misroute_rate":{ + "mean":0.436, + "std":0.001732 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.008, + "std":0.003606 + }, + "llm_call_rate":{ + "mean":0.957273, + "std":0.018584 + }, + "llm_cost_usd_per_1k":{ + "mean":0.353738, + "std":0.00685 + }, + "error_recovery_rate":{ + "mean":0.735536, + "std":0.00712 + }, + "recoverable_caught":{ + "mean":0.994931, + "std":0.00129 + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826 + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971 + }, + "recovered_errors":{ + "mean":2031.0, + "std":61.830413 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":4.37263, + "std":0.614103 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.051505, + "std":0.025831 + }, + "validation_selective_risk":{ + "mean":0.014554, + "std":0.012673 + }, + "validation_risk_upper95":{ + "mean":0.043318, + "std":0.004475 + }, + "test":{ + "coverage":{ + "mean":0.042667, + "std":0.020102 + }, + "selective_risk":{ + "mean":0.065288, + "std":0.020505 + }, + "accuracy_8":{ + "mean":0.823394, + "std":0.002956 + }, + "oos_recall":{ + "mean":0.564333, + "std":0.002082 + }, + "oos_recall_wilson95":[ + { + "mean":0.533412, + "std":0.00209 + }, + { + "mean":0.594762, + "std":0.002057 + } + ], + "oos_precision":{ + "mean":0.759005, + "std":0.011874 + }, + "oos_f1":{ + "mean":0.647311, + "std":0.002939 + }, + "oos_misroute_rate":{ + "mean":0.435667, + "std":0.002082 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.007667, + "std":0.005033 + }, + "llm_call_rate":{ + "mean":0.957333, + "std":0.020102 + }, + "llm_cost_usd_per_1k":{ + "mean":0.35376, + "std":0.007411 + }, + "error_recovery_rate":{ + "mean":0.736152, + "std":0.007239 + }, + "recoverable_caught":{ + "mean":0.995763, + "std":0.000431 + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826 + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971 + }, + "recovered_errors":{ + "mean":2032.666667, + "std":60.177515 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.966742, + "std":0.01863 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.044946, + "std":0.02408 + }, + "validation_selective_risk":{ + "mean":0.012819, + "std":0.011671 + }, + "validation_risk_upper95":{ + "mean":0.042993, + "std":0.005443 + }, + "test":{ + "coverage":{ + "mean":0.036424, + "std":0.01928 + }, + "selective_risk":{ + "mean":0.079285, + "std":0.044997 + }, + "accuracy_8":{ + "mean":0.822788, + "std":0.003611 + }, + "oos_recall":{ + "mean":0.564, + "std":0.001732 + }, + "oos_recall_wilson95":[ + { + "mean":0.533078, + "std":0.001739 + }, + { + "mean":0.594432, + "std":0.001712 + } + ], + "oos_precision":{ + "mean":0.757551, + "std":0.012652 + }, + "oos_f1":{ + "mean":0.646561, + "std":0.003458 + }, + "oos_misroute_rate":{ + "mean":0.436, + "std":0.001732 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.007333, + "std":0.004041 + }, + "llm_call_rate":{ + "mean":0.963576, + "std":0.01928 + }, + "llm_cost_usd_per_1k":{ + "mean":0.356063, + "std":0.007108 + }, + "error_recovery_rate":{ + "mean":0.735778, + "std":0.006751 + }, + "recoverable_caught":{ + "mean":0.995262, + "std":0.001754 + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826 + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971 + }, + "recovered_errors":{ + "mean":2031.666667, + "std":61.37046 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.502, + "std":0.011789, + "values":[ + 0.514727, + 0.499818, + 0.491455 + ] + }, + "accuracy_8":{ + "mean":0.498, + "std":0.011789, + "values":[ + 0.485273, + 0.500182, + 0.508545 + ] + }, + "oos_recall":{ + "mean":0.001333, + "std":0.001155, + "values":[ + 0.002, + 0.002, + 0.0 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.000366, + "std":0.000317, + "values":[ + 0.000549, + 0.000549, + 0.0 + ] + }, + { + "mean":0.006118, + "std":0.001984, + "values":[ + 0.007263, + 0.007263, + 0.003827 + ] + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735, + "values":[ + 1.0, + 1.0, + 0.0 + ] + }, + "oos_f1":{ + "mean":0.002661, + "std":0.002305, + "values":[ + 0.003992, + 0.003992, + 0.0 + ] + }, + "oos_misroute_rate":{ + "mean":0.998667, + "std":0.001155, + "values":[ + 0.998, + 0.998, + 1.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.998667, + "std":0.001155, + "values":[ + 0.998, + 0.998, + 1.0 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826, + "values":[ + 2831, + 2749, + 2703 + ] + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971, + "values":[ + 2112, + 2010, + 2002 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.498, + "std":0.011789, + "values":[ + 0.485273, + 0.500182, + 0.508545 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.869152, + "std":0.003456, + "values":[ + 0.869273, + 0.865636, + 0.872545 + ] + }, + "oos_recall":{ + "mean":0.568667, + "std":0.001155, + "values":[ + 0.568, + 0.57, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537765, + "std":0.00116, + "values":[ + 0.537095, + 0.539105, + 0.537095 + ] + }, + { + "mean":0.599043, + "std":0.00114, + "values":[ + 0.598384, + 0.60036, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.845467, + "std":0.010443, + "values":[ + 0.847761, + 0.854573, + 0.834068 + ] + }, + "oos_f1":{ + "mean":0.679964, + "std":0.004045, + "values":[ + 0.68024, + 0.683863, + 0.675788 + ] + }, + "oos_misroute_rate":{ + "mean":0.431333, + "std":0.001155, + "values":[ + 0.432, + 0.43, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.502, + "std":0.011789, + "values":[ + 0.514727, + 0.499818, + 0.491455 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.185303, + "std":0.004398, + "values":[ + 0.190043, + 0.184515, + 0.181352 + ] + }, + "error_recovery_rate":{ + "mean":0.739287, + "std":0.00752, + "values":[ + 0.746026, + 0.731175, + 0.740659 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826, + "values":[ + 2831, + 2749, + 2703 + ] + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971, + "values":[ + 2112, + 2010, + 2002 + ] + }, + "recovered_errors":{ + "mean":2041.333333, + "std":61.32971, + "values":[ + 2112, + 2010, + 2002 + ] + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.215991, + "std":0.008655, + "values":[ + 0.207092, + 0.216502, + 0.224379 + ] + }, + "oos_auroc":{ + "mean":0.645843, + "std":0.036771, + "values":[ + 0.642158, + 0.684318, + 0.611053 + ] + }, + "oos_auprc":{ + "mean":0.056653, + "std":0.010635, + "values":[ + 0.050524, + 0.068933, + 0.050501 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.306797, + "std":0.000705, + "values":[ + 0.307375, + 0.307004, + 0.306011 + ] + }, + "oos_auroc":{ + "mean":0.631819, + "std":0.002821, + "values":[ + 0.633552, + 0.633341, + 0.628564 + ] + }, + "oos_auprc":{ + "mean":0.243673, + "std":0.002406, + "values":[ + 0.24603, + 0.24122, + 0.243768 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.219267, + "std":0.008543, + "values":[ + 0.209507, + 0.222905, + 0.225389 + ] + }, + "oos_auroc":{ + "mean":0.647014, + "std":0.030274, + "values":[ + 0.645295, + 0.678112, + 0.617637 + ] + }, + "oos_auprc":{ + "mean":0.064036, + "std":0.024908, + "values":[ + 0.049071, + 0.092789, + 0.050248 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.310728, + "std":0.001868, + "values":[ + 0.312294, + 0.311229, + 0.30866 + ] + }, + "oos_auroc":{ + "mean":0.633958, + "std":0.01085, + "values":[ + 0.638001, + 0.642205, + 0.621667 + ] + }, + "oos_auprc":{ + "mean":0.248729, + "std":0.009254, + "values":[ + 0.247514, + 0.258531, + 0.240143 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":null, + "feasible":false, + "validation_coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "validation_selective_risk":null, + "validation_risk_upper95":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "selective_risk":null, + "accuracy_8":{ + "mean":0.820727, + "std":0.0, + "values":[ + 0.820727, + 0.820727, + 0.820727 + ] + }, + "oos_recall":{ + "mean":0.568, + "std":0.0, + "values":[ + 0.568, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0, + "values":[ + 0.537095, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.598384, + "std":0.0, + "values":[ + 0.598384, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.748353, + "std":0.0, + "values":[ + 0.748353, + 0.748353, + 0.748353 + ] + }, + "oos_f1":{ + "mean":0.645821, + "std":0.0, + "values":[ + 0.645821, + 0.645821, + 0.645821 + ] + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0, + "values":[ + 0.432, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.369492, + "std":0.0, + "values":[ + 0.369492, + 0.369492, + 0.369492 + ] + }, + "error_recovery_rate":{ + "mean":0.739287, + "std":0.00752, + "values":[ + 0.746026, + 0.731175, + 0.740659 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826, + "values":[ + 2831, + 2749, + 2703 + ] + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971, + "values":[ + 2112, + 2010, + 2002 + ] + }, + "recovered_errors":{ + "mean":2041.333333, + "std":61.32971, + "values":[ + 2112, + 2010, + 2002 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp_t", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":2.004105, + "std":1.803545, + "values":[ + 0.945491, + 4.086562, + 0.980262 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.052151, + "std":0.025623, + "values":[ + 0.070645, + 0.062903, + 0.022903 + ] + }, + "validation_selective_risk":{ + "mean":0.014448, + "std":0.012566, + "values":[ + 0.022831, + 0.020513, + 0.0 + ] + }, + "validation_risk_upper95":{ + "mean":0.042589, + "std":0.005131, + "values":[ + 0.046152, + 0.044907, + 0.036707 + ] + }, + "test":{ + "coverage":{ + "mean":0.042727, + "std":0.018584, + "values":[ + 0.058364, + 0.047636, + 0.022182 + ] + }, + "selective_risk":{ + "mean":0.078947, + "std":0.045322, + "values":[ + 0.056075, + 0.049618, + 0.131148 + ] + }, + "accuracy_8":{ + "mean":0.823152, + "std":0.003432, + "values":[ + 0.826909, + 0.822364, + 0.820182 + ] + }, + "oos_recall":{ + "mean":0.564, + "std":0.001732, + "values":[ + 0.562, + 0.565, + 0.565 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.533078, + "std":0.001739, + "values":[ + 0.53107, + 0.534082, + 0.534082 + ] + }, + { + "mean":0.594432, + "std":0.001712, + "values":[ + 0.592456, + 0.595421, + 0.595421 + ] + } + ], + "oos_precision":{ + "mean":0.758894, + "std":0.012019, + "values":[ + 0.771978, + 0.756359, + 0.748344 + ] + }, + "oos_f1":{ + "mean":0.647054, + "std":0.0033, + "values":[ + 0.650463, + 0.646823, + 0.643875 + ] + }, + "oos_misroute_rate":{ + "mean":0.436, + "std":0.001732, + "values":[ + 0.438, + 0.435, + 0.435 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.008, + "std":0.003606, + "values":[ + 0.012, + 0.007, + 0.005 + ] + }, + "llm_call_rate":{ + "mean":0.957273, + "std":0.018584, + "values":[ + 0.941636, + 0.952364, + 0.977818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.353738, + "std":0.00685, + "values":[ + 0.347974, + 0.351929, + 0.361311 + ] + }, + "error_recovery_rate":{ + "mean":0.735536, + "std":0.00712, + "values":[ + 0.742494, + 0.728265, + 0.735849 + ] + }, + "recoverable_caught":{ + "mean":0.994931, + "std":0.00129, + "values":[ + 0.995265, + 0.99602, + 0.993506 + ] + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826, + "values":[ + 2831, + 2749, + 2703 + ] + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971, + "values":[ + 2112, + 2010, + 2002 + ] + }, + "recovered_errors":{ + "mean":2031.0, + "std":61.830413, + "values":[ + 2102, + 2002, + 1989 + ] + } + } + } + } + } + } + } + }, + "tfidf-centroid/k10":{ + "model":"tfidf-centroid", + "k":10, + "oos_train":null, + "kind":"tfidf", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "tfidf-centroid-k10-seed42", + "tfidf-centroid-k10-seed43", + "tfidf-centroid-k10-seed44" + ], + "logits_sha256":[ + "efa61924b5913a37d54b9d4a3d13cd5be52fa35b56491dbfa9709fc8755b1aef", + "aa4fc1ec5e6cdcac08cd404d778130f36725e6151a54f9ab2be795d38ed523e2", + "cfb9163965d84af33aef86d28145073b143ebb604bd9901d37b18d54d842db22" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":3.784708, + "std":0.134826, + "values":[ + 3.903436, + 3.638132, + 3.812555 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":null, + "calibrated":{ + "ece_151":{ + "mean":0.069222, + "std":0.004983, + "values":[ + 0.074916, + 0.067099, + 0.065653 + ] + }, + "ece_8_summed":{ + "mean":0.043898, + "std":0.005378, + "values":[ + 0.041014, + 0.040579, + 0.050103 + ] + } + } + } + }, + "selected_aggregation":[ + "summed", + "argmax", + "argmax" + ], + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.853763, + "std":0.004701, + "values":[ + 0.849355, + 0.85871, + 0.853226 + ] + }, + "summed":{ + "mean":0.852903, + "std":0.00434, + "values":[ + 0.851613, + 0.857742, + 0.849355 + ] + } + }, + "signals_reported":[ + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.853763, + "std":0.004701 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.268788, + "std":0.004543 + }, + "accuracy_8":{ + "mean":0.731212, + "std":0.004543 + }, + "oos_recall":{ + "mean":0.020667, + "std":0.015885 + }, + "oos_recall_wilson95":[ + { + "mean":0.013896, + "std":0.01279 + }, + { + "mean":0.031106, + "std":0.018859 + } + ], + "oos_precision":{ + "mean":0.897564, + "std":0.107453 + }, + "oos_f1":{ + "mean":0.040065, + "std":0.030084 + }, + "oos_misroute_rate":{ + "mean":0.979333, + "std":0.015885 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.979333, + "std":0.015885 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1478.333333, + "std":24.986663 + }, + "recoverable_errors":{ + "mean":938.666667, + "std":18.009257 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.731212, + "std":0.004543 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.901879, + "std":0.001546 + }, + "oos_recall":{ + "mean":0.574, + "std":0.005292 + }, + "oos_recall_wilson95":[ + { + "mean":0.543126, + "std":0.005321 + }, + { + "mean":0.604307, + "std":0.005222 + } + ], + "oos_precision":{ + "mean":0.92381, + "std":0.0051 + }, + "oos_f1":{ + "mean":0.708054, + "std":0.005409 + }, + "oos_misroute_rate":{ + "mean":0.426, + "std":0.005292 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.268788, + "std":0.004543 + }, + "llm_cost_usd_per_1k":{ + "mean":0.098969, + "std":0.001658 + }, + "error_recovery_rate":{ + "mean":0.634937, + "std":0.002967 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1478.333333, + "std":24.986663 + }, + "recoverable_errors":{ + "mean":938.666667, + "std":18.009257 + }, + "recovered_errors":{ + "mean":938.666667, + "std":18.009257 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.046309, + "std":0.006464 + }, + "oos_auroc":{ + "mean":0.768016, + "std":0.027404 + }, + "oos_auprc":{ + "mean":0.078791, + "std":0.010593 + } + }, + "test":{ + "aurc":{ + "mean":0.101805, + "std":0.007428 + }, + "oos_auroc":{ + "mean":0.772759, + "std":0.014894 + }, + "oos_auprc":{ + "mean":0.362142, + "std":0.019219 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.043015, + "std":0.006257 + }, + "oos_auroc":{ + "mean":0.812323, + "std":0.021691 + }, + "oos_auprc":{ + "mean":0.12387, + "std":0.013965 + } + }, + "test":{ + "aurc":{ + "mean":0.093849, + "std":0.007596 + }, + "oos_auroc":{ + "mean":0.811775, + "std":0.012594 + }, + "oos_auprc":{ + "mean":0.455825, + "std":0.009635 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":6.012044, + "std":1.941326 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.274839, + "std":0.139491 + }, + "validation_selective_risk":{ + "mean":0.011201, + "std":0.002778 + }, + "validation_risk_upper95":{ + "mean":0.019803, + "std":9.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.23103, + "std":0.119544 + }, + "selective_risk":{ + "mean":0.026701, + "std":0.012962 + }, + "accuracy_8":{ + "mean":0.837091, + "std":0.01001 + }, + "oos_recall":{ + "mean":0.553667, + "std":0.012662 + }, + "oos_recall_wilson95":[ + { + "mean":0.522715, + "std":0.012695 + }, + { + "mean":0.584208, + "std":0.012532 + } + ], + "oos_precision":{ + "mean":0.788562, + "std":0.032387 + }, + "oos_f1":{ + "mean":0.650134, + "std":0.002121 + }, + "oos_misroute_rate":{ + "mean":0.446333, + "std":0.012662 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.026333, + "std":0.022811 + }, + "llm_call_rate":{ + "mean":0.76897, + "std":0.119544 + }, + "llm_cost_usd_per_1k":{ + "mean":0.284127, + "std":0.044217 + }, + "error_recovery_rate":{ + "mean":0.621168, + "std":0.016026 + }, + "recoverable_caught":{ + "mean":0.978253, + "std":0.020842 + }, + "small_errors":{ + "mean":1478.333333, + "std":24.986663 + }, + "recoverable_errors":{ + "mean":938.666667, + "std":18.009257 + }, + "recovered_errors":{ + "mean":918.333333, + "std":30.435725 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":6.012044, + "std":1.941326 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.274839, + "std":0.139491 + }, + "validation_selective_risk":{ + "mean":0.011201, + "std":0.002778 + }, + "validation_risk_upper95":{ + "mean":0.019803, + "std":9.2e-05 + }, + "test":{ + "coverage":{ + "mean":0.23103, + "std":0.119544 + }, + "selective_risk":{ + "mean":0.026701, + "std":0.012962 + }, + "accuracy_8":{ + "mean":0.837091, + "std":0.01001 + }, + "oos_recall":{ + "mean":0.553667, + "std":0.012662 + }, + "oos_recall_wilson95":[ + { + "mean":0.522715, + "std":0.012695 + }, + { + "mean":0.584208, + "std":0.012532 + } + ], + "oos_precision":{ + "mean":0.788562, + "std":0.032387 + }, + "oos_f1":{ + "mean":0.650134, + "std":0.002121 + }, + "oos_misroute_rate":{ + "mean":0.446333, + "std":0.012662 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.026333, + "std":0.022811 + }, + "llm_call_rate":{ + "mean":0.76897, + "std":0.119544 + }, + "llm_cost_usd_per_1k":{ + "mean":0.284127, + "std":0.044217 + }, + "error_recovery_rate":{ + "mean":0.621168, + "std":0.016026 + }, + "recoverable_caught":{ + "mean":0.978253, + "std":0.020842 + }, + "small_errors":{ + "mean":1478.333333, + "std":24.986663 + }, + "recoverable_errors":{ + "mean":938.666667, + "std":18.009257 + }, + "recovered_errors":{ + "mean":918.333333, + "std":30.435725 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.983013, + "std":0.019564 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.244839, + "std":0.131722 + }, + "validation_selective_risk":{ + "mean":0.01024, + "std":0.00343 + }, + "validation_risk_upper95":{ + "mean":0.019314, + "std":0.000564 + }, + "test":{ + "coverage":{ + "mean":0.206242, + "std":0.109969 + }, + "selective_risk":{ + "mean":0.025518, + "std":0.008834 + }, + "accuracy_8":{ + "mean":0.834, + "std":0.007677 + }, + "oos_recall":{ + "mean":0.556667, + "std":0.009713 + }, + "oos_recall_wilson95":[ + { + "mean":0.525721, + "std":0.009742 + }, + { + "mean":0.587178, + "std":0.009609 + } + ], + "oos_precision":{ + "mean":0.776958, + "std":0.020067 + }, + "oos_f1":{ + "mean":0.648422, + "std":0.000841 + }, + "oos_misroute_rate":{ + "mean":0.443333, + "std":0.009713 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.021333, + "std":0.016623 + }, + "llm_call_rate":{ + "mean":0.793758, + "std":0.109969 + }, + "llm_cost_usd_per_1k":{ + "mean":0.293294, + "std":0.040677 + }, + "error_recovery_rate":{ + "mean":0.623656, + "std":0.01265 + }, + "recoverable_caught":{ + "mean":0.982188, + "std":0.01554 + }, + "small_errors":{ + "mean":1478.333333, + "std":24.986663 + }, + "recoverable_errors":{ + "mean":938.666667, + "std":18.009257 + }, + "recovered_errors":{ + "mean":922.0, + "std":26.057628 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.710511, + "std":0.07044 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.613763, + "std":0.059537 + }, + "validation_selective_risk":{ + "mean":0.041668, + "std":0.000426 + }, + "validation_risk_upper95":{ + "mean":0.049898, + "std":4e-05 + }, + "test":{ + "coverage":{ + "mean":0.541333, + "std":0.054546 + }, + "selective_risk":{ + "mean":0.082792, + "std":0.004487 + }, + "accuracy_8":{ + "mean":0.846667, + "std":0.002729 + }, + "oos_recall":{ + "mean":0.477667, + "std":0.012897 + }, + "oos_recall_wilson95":[ + { + "mean":0.446859, + "std":0.012816 + }, + { + "mean":0.508645, + "std":0.012879 + } + ], + "oos_precision":{ + "mean":0.832246, + "std":0.017664 + }, + "oos_f1":{ + "mean":0.606753, + "std":0.006031 + }, + "oos_misroute_rate":{ + "mean":0.522333, + "std":0.012897 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.164667, + "std":0.019088 + }, + "llm_call_rate":{ + "mean":0.458667, + "std":0.054546 + }, + "llm_cost_usd_per_1k":{ + "mean":0.169387, + "std":0.020157 + }, + "error_recovery_rate":{ + "mean":0.533307, + "std":0.018225 + }, + "recoverable_caught":{ + "mean":0.839879, + "std":0.025614 + }, + "small_errors":{ + "mean":1478.333333, + "std":24.986663 + }, + "recoverable_errors":{ + "mean":938.666667, + "std":18.009257 + }, + "recovered_errors":{ + "mean":788.666667, + "std":39.068316 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":2.510897, + "std":0.470602 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.575699, + "std":0.056137 + }, + "validation_selective_risk":{ + "mean":0.041244, + "std":0.000515 + }, + "validation_risk_upper95":{ + "mean":0.049727, + "std":0.000127 + }, + "test":{ + "coverage":{ + "mean":0.511879, + "std":0.051603 + }, + "selective_risk":{ + "mean":0.0838, + "std":0.007073 + }, + "accuracy_8":{ + "mean":0.846, + "std":0.003455 + }, + "oos_recall":{ + "mean":0.478333, + "std":0.006429 + }, + "oos_recall_wilson95":[ + { + "mean":0.447516, + "std":0.006386 + }, + { + "mean":0.509317, + "std":0.006423 + } + ], + "oos_precision":{ + "mean":0.843211, + "std":0.006834 + }, + "oos_f1":{ + "mean":0.610361, + "std":0.003985 + }, + "oos_misroute_rate":{ + "mean":0.521667, + "std":0.006429 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.162, + "std":0.028213 + }, + "llm_call_rate":{ + "mean":0.488121, + "std":0.051603 + }, + "llm_cost_usd_per_1k":{ + "mean":0.180283, + "std":0.019078 + }, + "error_recovery_rate":{ + "mean":0.537871, + "std":0.012412 + }, + "recoverable_caught":{ + "mean":0.847145, + "std":0.020458 + }, + "small_errors":{ + "mean":1478.333333, + "std":24.986663 + }, + "recoverable_errors":{ + "mean":938.666667, + "std":18.009257 + }, + "recovered_errors":{ + "mean":795.333333, + "std":30.6159 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.710511, + "std":0.07044 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.613763, + "std":0.059537 + }, + "validation_selective_risk":{ + "mean":0.041668, + "std":0.000426 + }, + "validation_risk_upper95":{ + "mean":0.049898, + "std":4e-05 + }, + "test":{ + "coverage":{ + "mean":0.541333, + "std":0.054546 + }, + "selective_risk":{ + "mean":0.082792, + "std":0.004487 + }, + "accuracy_8":{ + "mean":0.846667, + "std":0.002729 + }, + "oos_recall":{ + "mean":0.477667, + "std":0.012897 + }, + "oos_recall_wilson95":[ + { + "mean":0.446859, + "std":0.012816 + }, + { + "mean":0.508645, + "std":0.012879 + } + ], + "oos_precision":{ + "mean":0.832246, + "std":0.017664 + }, + "oos_f1":{ + "mean":0.606753, + "std":0.006031 + }, + "oos_misroute_rate":{ + "mean":0.522333, + "std":0.012897 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.164667, + "std":0.019088 + }, + "llm_call_rate":{ + "mean":0.458667, + "std":0.054546 + }, + "llm_cost_usd_per_1k":{ + "mean":0.169387, + "std":0.020157 + }, + "error_recovery_rate":{ + "mean":0.533307, + "std":0.018225 + }, + "recoverable_caught":{ + "mean":0.839879, + "std":0.025614 + }, + "small_errors":{ + "mean":1478.333333, + "std":24.986663 + }, + "recoverable_errors":{ + "mean":938.666667, + "std":18.009257 + }, + "recovered_errors":{ + "mean":788.666667, + "std":39.068316 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.852903, + "std":0.00434 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.271394, + "std":0.003413 + }, + "accuracy_8":{ + "mean":0.728606, + "std":0.003413 + }, + "oos_recall":{ + "mean":0.007333, + "std":0.010116 + }, + "oos_recall_wilson95":[ + { + "mean":0.004307, + "std":0.006835 + }, + { + "mean":0.01413, + "std":0.013322 + } + ], + "oos_precision":{ + "mean":1.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.014427, + "std":0.019826 + }, + "oos_misroute_rate":{ + "mean":0.992667, + "std":0.010116 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.992667, + "std":0.010116 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1492.666667, + "std":18.770544 + }, + "recoverable_errors":{ + "mean":945.333333, + "std":17.097758 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.728606, + "std":0.003413 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.900485, + "std":0.001458 + }, + "oos_recall":{ + "mean":0.568667, + "std":0.001155 + }, + "oos_recall_wilson95":[ + { + "mean":0.537765, + "std":0.00116 + }, + { + "mean":0.599043, + "std":0.00114 + } + ], + "oos_precision":{ + "mean":0.922675, + "std":0.00539 + }, + "oos_f1":{ + "mean":0.703653, + "std":0.0024 + }, + "oos_misroute_rate":{ + "mean":0.431333, + "std":0.001155 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.271394, + "std":0.003413 + }, + "llm_cost_usd_per_1k":{ + "mean":0.099935, + "std":0.001237 + }, + "error_recovery_rate":{ + "mean":0.633299, + "std":0.005433 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1492.666667, + "std":18.770544 + }, + "recoverable_errors":{ + "mean":945.333333, + "std":17.097758 + }, + "recovered_errors":{ + "mean":945.333333, + "std":17.097758 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.033811, + "std":0.005273 + }, + "oos_auroc":{ + "mean":0.81495, + "std":0.02131 + }, + "oos_auprc":{ + "mean":0.118094, + "std":0.005629 + } + }, + "test":{ + "aurc":{ + "mean":0.085665, + "std":0.006678 + }, + "oos_auroc":{ + "mean":0.813214, + "std":0.014957 + }, + "oos_auprc":{ + "mean":0.431868, + "std":0.013475 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.035841, + "std":0.005079 + }, + "oos_auroc":{ + "mean":0.823931, + "std":0.018456 + }, + "oos_auprc":{ + "mean":0.135789, + "std":0.013696 + } + }, + "test":{ + "aurc":{ + "mean":0.086845, + "std":0.006887 + }, + "oos_auroc":{ + "mean":0.819339, + "std":0.015304 + }, + "oos_auprc":{ + "mean":0.46118, + "std":0.012629 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":4.582558, + "std":0.746145 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.437849, + "std":0.089007 + }, + "validation_selective_risk":{ + "mean":0.013393, + "std":0.00084 + }, + "validation_risk_upper95":{ + "mean":0.019661, + "std":0.000194 + }, + "test":{ + "coverage":{ + "mean":0.377576, + "std":0.078611 + }, + "selective_risk":{ + "mean":0.038153, + "std":0.011645 + }, + "accuracy_8":{ + "mean":0.844788, + "std":0.004987 + }, + "oos_recall":{ + "mean":0.538, + "std":0.01562 + }, + "oos_recall_wilson95":[ + { + "mean":0.507023, + "std":0.015626 + }, + { + "mean":0.568686, + "std":0.015496 + } + ], + "oos_precision":{ + "mean":0.794362, + "std":0.022871 + }, + "oos_f1":{ + "mean":0.641181, + "std":0.004685 + }, + "oos_misroute_rate":{ + "mean":0.462, + "std":0.01562 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.061667, + "std":0.03166 + }, + "llm_call_rate":{ + "mean":0.622424, + "std":0.078611 + }, + "llm_cost_usd_per_1k":{ + "mean":0.22994, + "std":0.029058 + }, + "error_recovery_rate":{ + "mean":0.606042, + "std":0.018276 + }, + "recoverable_caught":{ + "mean":0.956847, + "std":0.021073 + }, + "small_errors":{ + "mean":1492.666667, + "std":18.770544 + }, + "recoverable_errors":{ + "mean":945.333333, + "std":17.097758 + }, + "recovered_errors":{ + "mean":904.666667, + "std":31.722757 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":4.582558, + "std":0.746145 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.437849, + "std":0.089007 + }, + "validation_selective_risk":{ + "mean":0.013393, + "std":0.00084 + }, + "validation_risk_upper95":{ + "mean":0.019661, + "std":0.000194 + }, + "test":{ + "coverage":{ + "mean":0.377576, + "std":0.078611 + }, + "selective_risk":{ + "mean":0.038153, + "std":0.011645 + }, + "accuracy_8":{ + "mean":0.844788, + "std":0.004987 + }, + "oos_recall":{ + "mean":0.538, + "std":0.01562 + }, + "oos_recall_wilson95":[ + { + "mean":0.507023, + "std":0.015626 + }, + { + "mean":0.568686, + "std":0.015496 + } + ], + "oos_precision":{ + "mean":0.794362, + "std":0.022871 + }, + "oos_f1":{ + "mean":0.641181, + "std":0.004685 + }, + "oos_misroute_rate":{ + "mean":0.462, + "std":0.01562 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.061667, + "std":0.03166 + }, + "llm_call_rate":{ + "mean":0.622424, + "std":0.078611 + }, + "llm_cost_usd_per_1k":{ + "mean":0.22994, + "std":0.029058 + }, + "error_recovery_rate":{ + "mean":0.606042, + "std":0.018276 + }, + "recoverable_caught":{ + "mean":0.956847, + "std":0.021073 + }, + "small_errors":{ + "mean":1492.666667, + "std":18.770544 + }, + "recoverable_errors":{ + "mean":945.333333, + "std":17.097758 + }, + "recovered_errors":{ + "mean":904.666667, + "std":31.722757 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.984052, + "std":0.010198 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.380645, + "std":0.07787 + }, + "validation_selective_risk":{ + "mean":0.012055, + "std":0.001296 + }, + "validation_risk_upper95":{ + "mean":0.018597, + "std":0.001278 + }, + "test":{ + "coverage":{ + "mean":0.324667, + "std":0.06759 + }, + "selective_risk":{ + "mean":0.03093, + "std":0.008085 + }, + "accuracy_8":{ + "mean":0.842, + "std":0.004276 + }, + "oos_recall":{ + "mean":0.546667, + "std":0.01097 + }, + "oos_recall_wilson95":[ + { + "mean":0.515697, + "std":0.010987 + }, + { + "mean":0.57728, + "std":0.010869 + } + ], + "oos_precision":{ + "mean":0.780943, + "std":0.01495 + }, + "oos_f1":{ + "mean":0.642985, + "std":0.004246 + }, + "oos_misroute_rate":{ + "mean":0.453333, + "std":0.01097 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.042, + "std":0.019698 + }, + "llm_call_rate":{ + "mean":0.675333, + "std":0.06759 + }, + "llm_cost_usd_per_1k":{ + "mean":0.249512, + "std":0.024985 + }, + "error_recovery_rate":{ + "mean":0.613626, + "std":0.015123 + }, + "recoverable_caught":{ + "mean":0.968848, + "std":0.015734 + }, + "small_errors":{ + "mean":1492.666667, + "std":18.770544 + }, + "recoverable_errors":{ + "mean":945.333333, + "std":17.097758 + }, + "recovered_errors":{ + "mean":916.0, + "std":28.513155 + } + } + } + } + }, + "0.05":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":1.865655, + "std":0.29784 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.723441, + "std":0.041079 + }, + "validation_selective_risk":{ + "mean":0.042209, + "std":6e-05 + }, + "validation_risk_upper95":{ + "mean":0.049769, + "std":0.000166 + }, + "test":{ + "coverage":{ + "mean":0.645333, + "std":0.033825 + }, + "selective_risk":{ + "mean":0.095942, + "std":0.001363 + }, + "accuracy_8":{ + "mean":0.845152, + "std":0.003093 + }, + "oos_recall":{ + "mean":0.440667, + "std":0.01159 + }, + "oos_recall_wilson95":[ + { + "mean":0.410186, + "std":0.011465 + }, + { + "mean":0.471601, + "std":0.011627 + } + ], + "oos_precision":{ + "mean":0.85241, + "std":0.006456 + }, + "oos_f1":{ + "mean":0.580912, + "std":0.009799 + }, + "oos_misroute_rate":{ + "mean":0.559333, + "std":0.01159 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.246667, + "std":0.015177 + }, + "llm_call_rate":{ + "mean":0.354667, + "std":0.033825 + }, + "llm_cost_usd_per_1k":{ + "mean":0.130937, + "std":0.012521 + }, + "error_recovery_rate":{ + "mean":0.504142, + "std":0.012727 + }, + "recoverable_caught":{ + "mean":0.796016, + "std":0.015924 + }, + "small_errors":{ + "mean":1492.666667, + "std":18.770544 + }, + "recoverable_errors":{ + "mean":945.333333, + "std":17.097758 + }, + "recovered_errors":{ + "mean":752.666667, + "std":28.219379 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":1.865655, + "std":0.29784 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.723441, + "std":0.041079 + }, + "validation_selective_risk":{ + "mean":0.042209, + "std":6e-05 + }, + "validation_risk_upper95":{ + "mean":0.049769, + "std":0.000166 + }, + "test":{ + "coverage":{ + "mean":0.645333, + "std":0.033825 + }, + "selective_risk":{ + "mean":0.095942, + "std":0.001363 + }, + "accuracy_8":{ + "mean":0.845152, + "std":0.003093 + }, + "oos_recall":{ + "mean":0.440667, + "std":0.01159 + }, + "oos_recall_wilson95":[ + { + "mean":0.410186, + "std":0.011465 + }, + { + "mean":0.471601, + "std":0.011627 + } + ], + "oos_precision":{ + "mean":0.85241, + "std":0.006456 + }, + "oos_f1":{ + "mean":0.580912, + "std":0.009799 + }, + "oos_misroute_rate":{ + "mean":0.559333, + "std":0.01159 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.246667, + "std":0.015177 + }, + "llm_call_rate":{ + "mean":0.354667, + "std":0.033825 + }, + "llm_cost_usd_per_1k":{ + "mean":0.130937, + "std":0.012521 + }, + "error_recovery_rate":{ + "mean":0.504142, + "std":0.012727 + }, + "recoverable_caught":{ + "mean":0.796016, + "std":0.015924 + }, + "small_errors":{ + "mean":1492.666667, + "std":18.770544 + }, + "recoverable_errors":{ + "mean":945.333333, + "std":17.097758 + }, + "recovered_errors":{ + "mean":752.666667, + "std":28.219379 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.754894, + "std":0.044303 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.702581, + "std":0.038896 + }, + "validation_selective_risk":{ + "mean":0.042237, + "std":9.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.049919, + "std":0.000132 + }, + "test":{ + "coverage":{ + "mean":0.624303, + "std":0.028391 + }, + "selective_risk":{ + "mean":0.091663, + "std":0.00339 + }, + "accuracy_8":{ + "mean":0.845939, + "std":0.00447 + }, + "oos_recall":{ + "mean":0.454667, + "std":0.015275 + }, + "oos_recall_wilson95":[ + { + "mean":0.424046, + "std":0.015127 + }, + { + "mean":0.485634, + "std":0.015307 + } + ], + "oos_precision":{ + "mean":0.846136, + "std":0.01197 + }, + "oos_f1":{ + "mean":0.591418, + "std":0.014557 + }, + "oos_misroute_rate":{ + "mean":0.545333, + "std":0.015275 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.220333, + "std":0.01701 + }, + "llm_call_rate":{ + "mean":0.375697, + "std":0.028391 + }, + "llm_cost_usd_per_1k":{ + "mean":0.138701, + "std":0.010507 + }, + "error_recovery_rate":{ + "mean":0.513315, + "std":0.01522 + }, + "recoverable_caught":{ + "mean":0.810449, + "std":0.017432 + }, + "small_errors":{ + "mean":1492.666667, + "std":18.770544 + }, + "recoverable_errors":{ + "mean":945.333333, + "std":17.097758 + }, + "recovered_errors":{ + "mean":766.333333, + "std":30.105371 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.269576, + "std":0.00564, + "values":[ + 0.275091, + 0.263818, + 0.269818 + ] + }, + "accuracy_8":{ + "mean":0.730424, + "std":0.00564, + "values":[ + 0.724909, + 0.736182, + 0.730182 + ] + }, + "oos_recall":{ + "mean":0.017667, + "std":0.01914, + "values":[ + 0.002, + 0.039, + 0.012 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.012028, + "std":0.014746, + "values":[ + 0.000549, + 0.028659, + 0.006878 + ] + }, + { + "mean":0.026997, + "std":0.023415, + "values":[ + 0.007263, + 0.05287, + 0.020857 + ] + } + ], + "oos_precision":{ + "mean":0.968992, + "std":0.053707, + "values":[ + 1.0, + 0.906977, + 1.0 + ] + }, + "oos_f1":{ + "mean":0.034164, + "std":0.036534, + "values":[ + 0.003992, + 0.074784, + 0.023715 + ] + }, + "oos_misroute_rate":{ + "mean":0.982333, + "std":0.01914, + "values":[ + 0.998, + 0.961, + 0.988 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.982333, + "std":0.01914, + "values":[ + 0.998, + 0.961, + 0.988 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1482.666667, + "std":31.021498, + "values":[ + 1513, + 1451, + 1484 + ] + }, + "recoverable_errors":{ + "mean":941.333333, + "std":22.188586, + "values":[ + 965, + 921, + 938 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.730424, + "std":0.00564, + "values":[ + 0.724909, + 0.736182, + 0.730182 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.901576, + "std":0.001794, + "values":[ + 0.900364, + 0.903636, + 0.900727 + ] + }, + "oos_recall":{ + "mean":0.573333, + "std":0.00611, + "values":[ + 0.568, + 0.58, + 0.572 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.542457, + "std":0.006143, + "values":[ + 0.537095, + 0.54916, + 0.541115 + ] + }, + { + "mean":0.603649, + "std":0.00603, + "values":[ + 0.598384, + 0.610228, + 0.602334 + ] + } + ], + "oos_precision":{ + "mean":0.923726, + "std":0.005139, + "values":[ + 0.922078, + 0.929487, + 0.919614 + ] + }, + "oos_f1":{ + "mean":0.707519, + "std":0.005975, + "values":[ + 0.70297, + 0.714286, + 0.705302 + ] + }, + "oos_misroute_rate":{ + "mean":0.426667, + "std":0.00611, + "values":[ + 0.432, + 0.42, + 0.428 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.269576, + "std":0.00564, + "values":[ + 0.275091, + 0.263818, + 0.269818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.099261, + "std":0.002065, + "values":[ + 0.101283, + 0.097156, + 0.099344 + ] + }, + "error_recovery_rate":{ + "mean":0.634872, + "std":0.002868, + "values":[ + 0.637806, + 0.634735, + 0.632075 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1482.666667, + "std":31.021498, + "values":[ + 1513, + 1451, + 1484 + ] + }, + "recoverable_errors":{ + "mean":941.333333, + "std":22.188586, + "values":[ + 965, + 921, + 938 + ] + }, + "recovered_errors":{ + "mean":941.333333, + "std":22.188586, + "values":[ + 965, + 921, + 938 + ] + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.041645, + "std":0.004581, + "values":[ + 0.038468, + 0.039572, + 0.046896 + ] + }, + "oos_auroc":{ + "mean":0.785206, + "std":0.024152, + "values":[ + 0.799145, + 0.799155, + 0.757318 + ] + }, + "oos_auprc":{ + "mean":0.093986, + "std":0.023992, + "values":[ + 0.119347, + 0.090962, + 0.07165 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.095843, + "std":0.007186, + "values":[ + 0.090079, + 0.093556, + 0.103894 + ] + }, + "oos_auroc":{ + "mean":0.787346, + "std":0.019551, + "values":[ + 0.805597, + 0.789725, + 0.766714 + ] + }, + "oos_auprc":{ + "mean":0.387032, + "std":0.037313, + "values":[ + 0.425622, + 0.384333, + 0.351142 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.040051, + "std":0.002936, + "values":[ + 0.040521, + 0.036909, + 0.042723 + ] + }, + "oos_auroc":{ + "mean":0.817459, + "std":0.016965, + "values":[ + 0.810742, + 0.836755, + 0.804882 + ] + }, + "oos_auprc":{ + "mean":0.132582, + "std":0.015839, + "values":[ + 0.143431, + 0.139909, + 0.114406 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.090876, + "std":0.005096, + "values":[ + 0.091459, + 0.085513, + 0.095655 + ] + }, + "oos_auroc":{ + "mean":0.814764, + "std":0.011028, + "values":[ + 0.81363, + 0.826315, + 0.804347 + ] + }, + "oos_auprc":{ + "mean":0.457466, + "std":0.008725, + "values":[ + 0.455714, + 0.466934, + 0.449749 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":5.07988, + "std":0.571603, + "values":[ + 5.378551, + 5.440277, + 4.420813 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.346882, + "std":0.041262, + "values":[ + 0.336774, + 0.311613, + 0.392258 + ] + }, + "validation_selective_risk":{ + "mean":0.012677, + "std":0.000416, + "values":[ + 0.012452, + 0.012422, + 0.013158 + ] + }, + "validation_risk_upper95":{ + "mean":0.019666, + "std":0.000158, + "values":[ + 0.019489, + 0.019794, + 0.019715 + ] + }, + "test":{ + "coverage":{ + "mean":0.294606, + "std":0.039107, + "values":[ + 0.291273, + 0.257273, + 0.335273 + ] + }, + "selective_risk":{ + "mean":0.032929, + "std":0.009464, + "values":[ + 0.034956, + 0.022615, + 0.041215 + ] + }, + "accuracy_8":{ + "mean":0.841455, + "std":0.003483, + "values":[ + 0.839091, + 0.839818, + 0.845455 + ] + }, + "oos_recall":{ + "mean":0.547333, + "std":0.008083, + "values":[ + 0.546, + 0.556, + 0.54 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.516362, + "std":0.0081, + "values":[ + 0.515024, + 0.525049, + 0.509015 + ] + }, + { + "mean":0.577942, + "std":0.008004, + "values":[ + 0.576624, + 0.586523, + 0.570679 + ] + } + ], + "oos_precision":{ + "mean":0.793634, + "std":0.026714, + "values":[ + 0.776671, + 0.779804, + 0.824427 + ] + }, + "oos_f1":{ + "mean":0.647648, + "std":0.005821, + "values":[ + 0.641221, + 0.649154, + 0.652568 + ] + }, + "oos_misroute_rate":{ + "mean":0.452667, + "std":0.008083, + "values":[ + 0.454, + 0.444, + 0.46 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.037667, + "std":0.01464, + "values":[ + 0.04, + 0.022, + 0.051 + ] + }, + "llm_call_rate":{ + "mean":0.705394, + "std":0.039107, + "values":[ + 0.708727, + 0.742727, + 0.664727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.260612, + "std":0.014459, + "values":[ + 0.261851, + 0.274413, + 0.245574 + ] + }, + "error_recovery_rate":{ + "mean":0.615162, + "std":0.010479, + "values":[ + 0.617317, + 0.624397, + 0.603774 + ] + }, + "recoverable_caught":{ + "mean":0.968938, + "std":0.014274, + "values":[ + 0.967876, + 0.983713, + 0.955224 + ] + }, + "small_errors":{ + "mean":1482.666667, + "std":31.021498, + "values":[ + 1513, + 1451, + 1484 + ] + }, + "recoverable_errors":{ + "mean":941.333333, + "std":22.188586, + "values":[ + 965, + 921, + 938 + ] + }, + "recovered_errors":{ + "mean":912.0, + "std":19.697716, + "values":[ + 934, + 906, + 896 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "margin", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":1.08125, + "std":0.695219, + "values":[ + 1.882091, + 0.632676, + 0.728981 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.663333, + "std":0.04692, + "values":[ + 0.703871, + 0.674194, + 0.611935 + ] + }, + "validation_selective_risk":{ + "mean":0.041971, + "std":0.000284, + "values":[ + 0.042163, + 0.042105, + 0.041645 + ] + }, + "validation_risk_upper95":{ + "mean":0.049877, + "std":6e-05, + "values":[ + 0.049825, + 0.049943, + 0.049865 + ] + }, + "test":{ + "coverage":{ + "mean":0.588061, + "std":0.043546, + "values":[ + 0.627091, + 0.596, + 0.541091 + ] + }, + "selective_risk":{ + "mean":0.088409, + "std":0.005336, + "values":[ + 0.09452, + 0.086028, + 0.084677 + ] + }, + "accuracy_8":{ + "mean":0.846727, + "std":0.002678, + "values":[ + 0.845273, + 0.849818, + 0.845091 + ] + }, + "oos_recall":{ + "mean":0.465, + "std":0.010149, + "values":[ + 0.454, + 0.474, + 0.467 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.434283, + "std":0.010064, + "values":[ + 0.423376, + 0.44321, + 0.436263 + ] + }, + { + "mean":0.495985, + "std":0.010156, + "values":[ + 0.484976, + 0.504989, + 0.49799 + ] + } + ], + "oos_precision":{ + "mean":0.845015, + "std":0.004587, + "values":[ + 0.850187, + 0.843416, + 0.841441 + ] + }, + "oos_f1":{ + "mean":0.599825, + "std":0.007532, + "values":[ + 0.591917, + 0.606914, + 0.600643 + ] + }, + "oos_misroute_rate":{ + "mean":0.535, + "std":0.010149, + "values":[ + 0.546, + 0.526, + 0.533 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.194667, + "std":0.033382, + "values":[ + 0.233, + 0.179, + 0.172 + ] + }, + "llm_call_rate":{ + "mean":0.411939, + "std":0.043546, + "values":[ + 0.372909, + 0.404, + 0.458909 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.152115, + "std":0.016133, + "values":[ + 0.137662, + 0.149164, + 0.169519 + ] + }, + "error_recovery_rate":{ + "mean":0.521586, + "std":0.004085, + "values":[ + 0.518837, + 0.519642, + 0.52628 + ] + }, + "recoverable_caught":{ + "mean":0.82159, + "std":0.009903, + "values":[ + 0.813472, + 0.818675, + 0.832623 + ] + }, + "small_errors":{ + "mean":1482.666667, + "std":31.021498, + "values":[ + 1513, + 1451, + 1484 + ] + }, + "recoverable_errors":{ + "mean":941.333333, + "std":22.188586, + "values":[ + 965, + 921, + 938 + ] + }, + "recovered_errors":{ + "mean":773.333333, + "std":16.862186, + "values":[ + 785, + 754, + 781 + ] + } + } + } + } + } + } + } + }, + "tfidf-centroid/k100":{ + "model":"tfidf-centroid", + "k":100, + "oos_train":null, + "kind":"tfidf", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "tfidf-centroid-k100-seed42", + "tfidf-centroid-k100-seed43", + "tfidf-centroid-k100-seed44" + ], + "logits_sha256":[ + "0ae4c761890317d76efc8d3b08a3920eab94633e32f710f0059cb7b96f82cb75", + "01e56ce77413fd0e6dcb11398fdd0eb8024997983daf7bac5fea2a410d1bbe33", + "ae69c656b6f00892ace610769c997b03dcfe588cb8568b4cd4e94bba9fee14bf" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":2.915677, + "std":0.0, + "values":[ + 2.915677, + 2.915677, + 2.915677 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":null, + "calibrated":{ + "ece_151":{ + "mean":0.073697, + "std":0.0, + "values":[ + 0.073697, + 0.073697, + 0.073697 + ] + }, + "ece_8_summed":{ + "mean":0.067706, + "std":0.0, + "values":[ + 0.067706, + 0.067706, + 0.067706 + ] + } + } + } + }, + "selected_aggregation":"argmax", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.892903, + "std":0.0, + "values":[ + 0.892903, + 0.892903, + 0.892903 + ] + }, + "summed":{ + "mean":0.890968, + "std":0.0, + "values":[ + 0.890968, + 0.890968, + 0.890968 + ] + } + }, + "signals_reported":[ + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.892903, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.223818, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.776182, + "std":0.0 + }, + "oos_recall":{ + "mean":0.096, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.079257, + "std":0.0 + }, + { + "mean":0.115836, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.932039, + "std":0.0 + }, + "oos_f1":{ + "mean":0.174071, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.904, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.904, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1231.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.776182, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.910545, + "std":0.0 + }, + "oos_recall":{ + "mean":0.588, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.557213, + "std":0.0 + }, + { + "mean":0.618113, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.945338, + "std":0.0 + }, + "oos_f1":{ + "mean":0.725031, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.412, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.223818, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.082427, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.600325, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1231.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":739.0, + "std":0.0 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.026609, + "std":0.0 + }, + "oos_auroc":{ + "mean":0.790862, + "std":0.0 + }, + "oos_auprc":{ + "mean":0.098241, + "std":0.0 + } + }, + "test":{ + "aurc":{ + "mean":0.069831, + "std":0.0 + }, + "oos_auroc":{ + "mean":0.830446, + "std":0.0 + }, + "oos_auprc":{ + "mean":0.467543, + "std":0.0 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.02568, + "std":0.0 + }, + "oos_auroc":{ + "mean":0.837072, + "std":0.0 + }, + "oos_auprc":{ + "mean":0.147488, + "std":0.0 + } + }, + "test":{ + "aurc":{ + "mean":0.064978, + "std":0.0 + }, + "oos_auroc":{ + "mean":0.862373, + "std":0.0 + }, + "oos_auprc":{ + "mean":0.559526, + "std":0.0 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.964532, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.509677, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.013924, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.01967, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.452, + "std":0.0 + }, + "selective_risk":{ + "mean":0.029364, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.859636, + "std":0.0 + }, + "oos_recall":{ + "mean":0.54, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.509015, + "std":0.0 + }, + { + "mean":0.570679, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.841121, + "std":0.0 + }, + "oos_f1":{ + "mean":0.657734, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.46, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.052, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.548, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.202369, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.568643, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.947226, + "std":0.0 + }, + "small_errors":{ + "mean":1231.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":700.0, + "std":0.0 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":4.545894, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.509677, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.013924, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.01967, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.444, + "std":0.0 + }, + "selective_risk":{ + "mean":0.032351, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.857636, + "std":0.0 + }, + "oos_recall":{ + "mean":0.537, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.506012, + "std":0.0 + }, + { + "mean":0.567705, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.832558, + "std":0.0 + }, + "oos_f1":{ + "mean":0.652888, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.463, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.06, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.556, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.205307, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.567831, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.945873, + "std":0.0 + }, + "small_errors":{ + "mean":1231.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":699.0, + "std":0.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.964532, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.509677, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.013924, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.01967, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.452, + "std":0.0 + }, + "selective_risk":{ + "mean":0.029364, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.859636, + "std":0.0 + }, + "oos_recall":{ + "mean":0.54, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.509015, + "std":0.0 + }, + { + "mean":0.570679, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.841121, + "std":0.0 + }, + "oos_f1":{ + "mean":0.657734, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.46, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.052, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.548, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.202369, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.568643, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.947226, + "std":0.0 + }, + "small_errors":{ + "mean":1231.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":700.0, + "std":0.0 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.619517, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.78, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.042597, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.049879, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.707273, + "std":0.0 + }, + "selective_risk":{ + "mean":0.08509, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.856545, + "std":0.0 + }, + "oos_recall":{ + "mean":0.437, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.406556, + "std":0.0 + }, + { + "mean":0.467926, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.875752, + "std":0.0 + }, + "oos_f1":{ + "mean":0.583055, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.563, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.242, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.292727, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.107993, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.444354, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.740189, + "std":0.0 + }, + "small_errors":{ + "mean":1231.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":547.0, + "std":0.0 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":1.68983, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.764194, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.042634, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.049999, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.694727, + "std":0.0 + }, + "selective_risk":{ + "mean":0.094478, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.853091, + "std":0.0 + }, + "oos_recall":{ + "mean":0.418, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.3878, + "std":0.0 + }, + { + "mean":0.448828, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.87631, + "std":0.0 + }, + "oos_f1":{ + "mean":0.566012, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.582, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.277, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.305273, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.112642, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.42892, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.714479, + "std":0.0 + }, + "small_errors":{ + "mean":1231.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":528.0, + "std":0.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.619517, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.78, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.042597, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.049879, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.707273, + "std":0.0 + }, + "selective_risk":{ + "mean":0.08509, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.856545, + "std":0.0 + }, + "oos_recall":{ + "mean":0.437, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.406556, + "std":0.0 + }, + { + "mean":0.467926, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.875752, + "std":0.0 + }, + "oos_f1":{ + "mean":0.583055, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.563, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.242, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.292727, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.107993, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.444354, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.740189, + "std":0.0 + }, + "small_errors":{ + "mean":1231.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":547.0, + "std":0.0 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.890968, + "std":0.0 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.237273, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.762727, + "std":0.0 + }, + "oos_recall":{ + "mean":0.025, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.01699, + "std":0.0 + }, + { + "mean":0.036646, + "std":0.0 + } + ], + "oos_precision":{ + "mean":1.0, + "std":0.0 + }, + "oos_f1":{ + "mean":0.04878, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.975, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.975, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1305.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":794.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.762727, + "std":0.0 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.907091, + "std":0.0 + }, + "oos_recall":{ + "mean":0.572, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.541115, + "std":0.0 + }, + { + "mean":0.602334, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.940789, + "std":0.0 + }, + "oos_f1":{ + "mean":0.711443, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.428, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.237273, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.087348, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.608429, + "std":0.0 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1305.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":794.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":794.0, + "std":0.0 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.019238, + "std":0.0 + }, + "oos_auroc":{ + "mean":0.859057, + "std":0.0 + }, + "oos_auprc":{ + "mean":0.178981, + "std":0.0 + } + }, + "test":{ + "aurc":{ + "mean":0.059351, + "std":0.0 + }, + "oos_auroc":{ + "mean":0.866122, + "std":0.0 + }, + "oos_auprc":{ + "mean":0.528085, + "std":0.0 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.020598, + "std":0.0 + }, + "oos_auroc":{ + "mean":0.865442, + "std":0.0 + }, + "oos_auprc":{ + "mean":0.214247, + "std":0.0 + } + }, + "test":{ + "aurc":{ + "mean":0.059806, + "std":0.0 + }, + "oos_auroc":{ + "mean":0.873458, + "std":0.0 + }, + "oos_auprc":{ + "mean":0.567753, + "std":0.0 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":3.836273, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.665161, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.014549, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.019566, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.588364, + "std":0.0 + }, + "selective_risk":{ + "mean":0.037392, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.866364, + "std":0.0 + }, + "oos_recall":{ + "mean":0.519, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.488019, + "std":0.0 + }, + { + "mean":0.549836, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.859272, + "std":0.0 + }, + "oos_f1":{ + "mean":0.647132, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.481, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.096, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.411636, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.151949, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.560153, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.920655, + "std":0.0 + }, + "small_errors":{ + "mean":1305.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":794.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":731.0, + "std":0.0 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":3.836273, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.665161, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.014549, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.019566, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.588364, + "std":0.0 + }, + "selective_risk":{ + "mean":0.037392, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.866364, + "std":0.0 + }, + "oos_recall":{ + "mean":0.519, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.488019, + "std":0.0 + }, + { + "mean":0.549836, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.859272, + "std":0.0 + }, + "oos_f1":{ + "mean":0.647132, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.481, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.096, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.411636, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.151949, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.560153, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.920655, + "std":0.0 + }, + "small_errors":{ + "mean":1305.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":794.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":731.0, + "std":0.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.967093, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.635806, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.014713, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.019886, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.562545, + "std":0.0 + }, + "selective_risk":{ + "mean":0.034906, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.864909, + "std":0.0 + }, + "oos_recall":{ + "mean":0.526, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.495011, + "std":0.0 + }, + { + "mean":0.55679, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.852512, + "std":0.0 + }, + "oos_f1":{ + "mean":0.650588, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.474, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.08, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.437455, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.161504, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.563985, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.926952, + "std":0.0 + }, + "small_errors":{ + "mean":1305.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":794.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":736.0, + "std":0.0 + } + } + } + } + }, + "0.05":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":1.535342, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.839032, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.042676, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.049684, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.772182, + "std":0.0 + }, + "selective_risk":{ + "mean":0.098658, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.853273, + "std":0.0 + }, + "oos_recall":{ + "mean":0.391, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.361227, + "std":0.0 + }, + { + "mean":0.421607, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.896789, + "std":0.0 + }, + "oos_f1":{ + "mean":0.544568, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.609, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.333, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.227818, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.084003, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.431418, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.709068, + "std":0.0 + }, + "small_errors":{ + "mean":1305.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":794.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":563.0, + "std":0.0 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":1.535342, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.839032, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.042676, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.049684, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.772182, + "std":0.0 + }, + "selective_risk":{ + "mean":0.098658, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.853273, + "std":0.0 + }, + "oos_recall":{ + "mean":0.391, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.361227, + "std":0.0 + }, + { + "mean":0.421607, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.896789, + "std":0.0 + }, + "oos_f1":{ + "mean":0.544568, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.609, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.333, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.227818, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.084003, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.431418, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.709068, + "std":0.0 + }, + "small_errors":{ + "mean":1305.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":794.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":563.0, + "std":0.0 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.688303, + "std":0.0 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.835484, + "std":0.0 + }, + "validation_selective_risk":{ + "mean":0.042857, + "std":0.0 + }, + "validation_risk_upper95":{ + "mean":0.049894, + "std":0.0 + }, + "test":{ + "coverage":{ + "mean":0.761091, + "std":0.0 + }, + "selective_risk":{ + "mean":0.095796, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.854545, + "std":0.0 + }, + "oos_recall":{ + "mean":0.413, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.382872, + "std":0.0 + }, + { + "mean":0.443794, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.892009, + "std":0.0 + }, + "oos_f1":{ + "mean":0.564593, + "std":0.0 + }, + "oos_misroute_rate":{ + "mean":0.587, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.306, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.238909, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.088094, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.441379, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.725441, + "std":0.0 + }, + "small_errors":{ + "mean":1305.0, + "std":0.0 + }, + "recoverable_errors":{ + "mean":794.0, + "std":0.0 + }, + "recovered_errors":{ + "mean":576.0, + "std":0.0 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.223818, + "std":0.0, + "values":[ + 0.223818, + 0.223818, + 0.223818 + ] + }, + "accuracy_8":{ + "mean":0.776182, + "std":0.0, + "values":[ + 0.776182, + 0.776182, + 0.776182 + ] + }, + "oos_recall":{ + "mean":0.096, + "std":0.0, + "values":[ + 0.096, + 0.096, + 0.096 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.079257, + "std":0.0, + "values":[ + 0.079257, + 0.079257, + 0.079257 + ] + }, + { + "mean":0.115836, + "std":0.0, + "values":[ + 0.115836, + 0.115836, + 0.115836 + ] + } + ], + "oos_precision":{ + "mean":0.932039, + "std":0.0, + "values":[ + 0.932039, + 0.932039, + 0.932039 + ] + }, + "oos_f1":{ + "mean":0.174071, + "std":0.0, + "values":[ + 0.174071, + 0.174071, + 0.174071 + ] + }, + "oos_misroute_rate":{ + "mean":0.904, + "std":0.0, + "values":[ + 0.904, + 0.904, + 0.904 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.904, + "std":0.0, + "values":[ + 0.904, + 0.904, + 0.904 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1231.0, + "std":0.0, + "values":[ + 1231, + 1231, + 1231 + ] + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0, + "values":[ + 739, + 739, + 739 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.776182, + "std":0.0, + "values":[ + 0.776182, + 0.776182, + 0.776182 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.910545, + "std":0.0, + "values":[ + 0.910545, + 0.910545, + 0.910545 + ] + }, + "oos_recall":{ + "mean":0.588, + "std":0.0, + "values":[ + 0.588, + 0.588, + 0.588 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.557213, + "std":0.0, + "values":[ + 0.557213, + 0.557213, + 0.557213 + ] + }, + { + "mean":0.618113, + "std":0.0, + "values":[ + 0.618113, + 0.618113, + 0.618113 + ] + } + ], + "oos_precision":{ + "mean":0.945338, + "std":0.0, + "values":[ + 0.945338, + 0.945338, + 0.945338 + ] + }, + "oos_f1":{ + "mean":0.725031, + "std":0.0, + "values":[ + 0.725031, + 0.725031, + 0.725031 + ] + }, + "oos_misroute_rate":{ + "mean":0.412, + "std":0.0, + "values":[ + 0.412, + 0.412, + 0.412 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.223818, + "std":0.0, + "values":[ + 0.223818, + 0.223818, + 0.223818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.082427, + "std":0.0, + "values":[ + 0.082427, + 0.082427, + 0.082427 + ] + }, + "error_recovery_rate":{ + "mean":0.600325, + "std":0.0, + "values":[ + 0.600325, + 0.600325, + 0.600325 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1231.0, + "std":0.0, + "values":[ + 1231, + 1231, + 1231 + ] + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0, + "values":[ + 739, + 739, + 739 + ] + }, + "recovered_errors":{ + "mean":739.0, + "std":0.0, + "values":[ + 739, + 739, + 739 + ] + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.026609, + "std":0.0, + "values":[ + 0.026609, + 0.026609, + 0.026609 + ] + }, + "oos_auroc":{ + "mean":0.790862, + "std":0.0, + "values":[ + 0.790862, + 0.790862, + 0.790862 + ] + }, + "oos_auprc":{ + "mean":0.098241, + "std":0.0, + "values":[ + 0.098241, + 0.098241, + 0.098241 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.069831, + "std":0.0, + "values":[ + 0.069831, + 0.069831, + 0.069831 + ] + }, + "oos_auroc":{ + "mean":0.830446, + "std":0.0, + "values":[ + 0.830446, + 0.830446, + 0.830446 + ] + }, + "oos_auprc":{ + "mean":0.467543, + "std":0.0, + "values":[ + 0.467543, + 0.467543, + 0.467543 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.02568, + "std":0.0, + "values":[ + 0.02568, + 0.02568, + 0.02568 + ] + }, + "oos_auroc":{ + "mean":0.837072, + "std":0.0, + "values":[ + 0.837072, + 0.837072, + 0.837072 + ] + }, + "oos_auprc":{ + "mean":0.147488, + "std":0.0, + "values":[ + 0.147488, + 0.147488, + 0.147488 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.064978, + "std":0.0, + "values":[ + 0.064978, + 0.064978, + 0.064978 + ] + }, + "oos_auroc":{ + "mean":0.862373, + "std":0.0, + "values":[ + 0.862373, + 0.862373, + 0.862373 + ] + }, + "oos_auprc":{ + "mean":0.559526, + "std":0.0, + "values":[ + 0.559526, + 0.559526, + 0.559526 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.964532, + "std":0.0, + "values":[ + 0.964532, + 0.964532, + 0.964532 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.509677, + "std":0.0, + "values":[ + 0.509677, + 0.509677, + 0.509677 + ] + }, + "validation_selective_risk":{ + "mean":0.013924, + "std":0.0, + "values":[ + 0.013924, + 0.013924, + 0.013924 + ] + }, + "validation_risk_upper95":{ + "mean":0.01967, + "std":0.0, + "values":[ + 0.01967, + 0.01967, + 0.01967 + ] + }, + "test":{ + "coverage":{ + "mean":0.452, + "std":0.0, + "values":[ + 0.452, + 0.452, + 0.452 + ] + }, + "selective_risk":{ + "mean":0.029364, + "std":0.0, + "values":[ + 0.029364, + 0.029364, + 0.029364 + ] + }, + "accuracy_8":{ + "mean":0.859636, + "std":0.0, + "values":[ + 0.859636, + 0.859636, + 0.859636 + ] + }, + "oos_recall":{ + "mean":0.54, + "std":0.0, + "values":[ + 0.54, + 0.54, + 0.54 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.509015, + "std":0.0, + "values":[ + 0.509015, + 0.509015, + 0.509015 + ] + }, + { + "mean":0.570679, + "std":0.0, + "values":[ + 0.570679, + 0.570679, + 0.570679 + ] + } + ], + "oos_precision":{ + "mean":0.841121, + "std":0.0, + "values":[ + 0.841121, + 0.841121, + 0.841121 + ] + }, + "oos_f1":{ + "mean":0.657734, + "std":0.0, + "values":[ + 0.657734, + 0.657734, + 0.657734 + ] + }, + "oos_misroute_rate":{ + "mean":0.46, + "std":0.0, + "values":[ + 0.46, + 0.46, + 0.46 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.052, + "std":0.0, + "values":[ + 0.052, + 0.052, + 0.052 + ] + }, + "llm_call_rate":{ + "mean":0.548, + "std":0.0, + "values":[ + 0.548, + 0.548, + 0.548 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.202369, + "std":0.0, + "values":[ + 0.202369, + 0.202369, + 0.202369 + ] + }, + "error_recovery_rate":{ + "mean":0.568643, + "std":0.0, + "values":[ + 0.568643, + 0.568643, + 0.568643 + ] + }, + "recoverable_caught":{ + "mean":0.947226, + "std":0.0, + "values":[ + 0.947226, + 0.947226, + 0.947226 + ] + }, + "small_errors":{ + "mean":1231.0, + "std":0.0, + "values":[ + 1231, + 1231, + 1231 + ] + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0, + "values":[ + 739, + 739, + 739 + ] + }, + "recovered_errors":{ + "mean":700.0, + "std":0.0, + "values":[ + 700, + 700, + 700 + ] + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.619517, + "std":0.0, + "values":[ + 0.619517, + 0.619517, + 0.619517 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.78, + "std":0.0, + "values":[ + 0.78, + 0.78, + 0.78 + ] + }, + "validation_selective_risk":{ + "mean":0.042597, + "std":0.0, + "values":[ + 0.042597, + 0.042597, + 0.042597 + ] + }, + "validation_risk_upper95":{ + "mean":0.049879, + "std":0.0, + "values":[ + 0.049879, + 0.049879, + 0.049879 + ] + }, + "test":{ + "coverage":{ + "mean":0.707273, + "std":0.0, + "values":[ + 0.707273, + 0.707273, + 0.707273 + ] + }, + "selective_risk":{ + "mean":0.08509, + "std":0.0, + "values":[ + 0.08509, + 0.08509, + 0.08509 + ] + }, + "accuracy_8":{ + "mean":0.856545, + "std":0.0, + "values":[ + 0.856545, + 0.856545, + 0.856545 + ] + }, + "oos_recall":{ + "mean":0.437, + "std":0.0, + "values":[ + 0.437, + 0.437, + 0.437 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.406556, + "std":0.0, + "values":[ + 0.406556, + 0.406556, + 0.406556 + ] + }, + { + "mean":0.467926, + "std":0.0, + "values":[ + 0.467926, + 0.467926, + 0.467926 + ] + } + ], + "oos_precision":{ + "mean":0.875752, + "std":0.0, + "values":[ + 0.875752, + 0.875752, + 0.875752 + ] + }, + "oos_f1":{ + "mean":0.583055, + "std":0.0, + "values":[ + 0.583055, + 0.583055, + 0.583055 + ] + }, + "oos_misroute_rate":{ + "mean":0.563, + "std":0.0, + "values":[ + 0.563, + 0.563, + 0.563 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.242, + "std":0.0, + "values":[ + 0.242, + 0.242, + 0.242 + ] + }, + "llm_call_rate":{ + "mean":0.292727, + "std":0.0, + "values":[ + 0.292727, + 0.292727, + 0.292727 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.107993, + "std":0.0, + "values":[ + 0.107993, + 0.107993, + 0.107993 + ] + }, + "error_recovery_rate":{ + "mean":0.444354, + "std":0.0, + "values":[ + 0.444354, + 0.444354, + 0.444354 + ] + }, + "recoverable_caught":{ + "mean":0.740189, + "std":0.0, + "values":[ + 0.740189, + 0.740189, + 0.740189 + ] + }, + "small_errors":{ + "mean":1231.0, + "std":0.0, + "values":[ + 1231, + 1231, + 1231 + ] + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0, + "values":[ + 739, + 739, + 739 + ] + }, + "recovered_errors":{ + "mean":547.0, + "std":0.0, + "values":[ + 547, + 547, + 547 + ] + } + } + } + } + } + } + } + }, + "tfidf-centroid/k25":{ + "model":"tfidf-centroid", + "k":25, + "oos_train":null, + "kind":"tfidf", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "tfidf-centroid-k25-seed42", + "tfidf-centroid-k25-seed43", + "tfidf-centroid-k25-seed44" + ], + "logits_sha256":[ + "be4fdf7bd9f789ab99de9042ffbc8b0d41d6cb152e08a7c079fd3bf623909a47", + "c09e6879bf292df29c7fa24208687bdc43a2513f62033313cc100d5d6a1f0fa4", + "e7aa6e711a2a6b5322a78def95a33738c6bf0f80f6006710d203af2c9c3fdcd5" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":3.326153, + "std":0.055271, + "values":[ + 3.384514, + 3.274604, + 3.319341 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":null, + "calibrated":{ + "ece_151":{ + "mean":0.071672, + "std":0.006115, + "values":[ + 0.075937, + 0.064666, + 0.074413 + ] + }, + "ece_8_summed":{ + "mean":0.061539, + "std":0.004408, + "values":[ + 0.058827, + 0.059164, + 0.066625 + ] + } + } + } + }, + "selected_aggregation":[ + "summed", + "argmax", + "argmax" + ], + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.879677, + "std":0.007517, + "values":[ + 0.87129, + 0.885806, + 0.881935 + ] + }, + "summed":{ + "mean":0.877634, + "std":0.005497, + "values":[ + 0.871935, + 0.882903, + 0.878065 + ] + } + }, + "signals_reported":[ + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.879677, + "std":0.007517 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.244182, + "std":0.008458 + }, + "accuracy_8":{ + "mean":0.755818, + "std":0.008458 + }, + "oos_recall":{ + "mean":0.048333, + "std":0.035019 + }, + "oos_recall_wilson95":[ + { + "mean":0.037414, + "std":0.030045 + }, + { + "mean":0.062709, + "std":0.039737 + } + ], + "oos_precision":{ + "mean":0.935556, + "std":0.003849 + }, + "oos_f1":{ + "mean":0.090413, + "std":0.063276 + }, + "oos_misroute_rate":{ + "mean":0.951667, + "std":0.035019 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.951667, + "std":0.035019 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1343.0, + "std":46.518813 + }, + "recoverable_errors":{ + "mean":827.0, + "std":40.447497 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.755818, + "std":0.008458 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.906182, + "std":0.001924 + }, + "oos_recall":{ + "mean":0.579, + "std":0.010817 + }, + "oos_recall_wilson95":[ + { + "mean":0.548159, + "std":0.010885 + }, + { + "mean":0.609237, + "std":0.010666 + } + ], + "oos_precision":{ + "mean":0.937889, + "std":0.005367 + }, + "oos_f1":{ + "mean":0.715959, + "std":0.009212 + }, + "oos_misroute_rate":{ + "mean":0.421, + "std":0.010817 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.244182, + "std":0.008458 + }, + "llm_cost_usd_per_1k":{ + "mean":0.089903, + "std":0.003113 + }, + "error_recovery_rate":{ + "mean":0.6156, + "std":0.009951 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1343.0, + "std":46.518813 + }, + "recoverable_errors":{ + "mean":827.0, + "std":40.447497 + }, + "recovered_errors":{ + "mean":827.0, + "std":40.447497 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.031423, + "std":0.002325 + }, + "oos_auroc":{ + "mean":0.782024, + "std":0.006656 + }, + "oos_auprc":{ + "mean":0.090052, + "std":0.008144 + } + }, + "test":{ + "aurc":{ + "mean":0.081794, + "std":0.004101 + }, + "oos_auroc":{ + "mean":0.803111, + "std":0.006473 + }, + "oos_auprc":{ + "mean":0.40631, + "std":0.001925 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.029062, + "std":0.001627 + }, + "oos_auroc":{ + "mean":0.827768, + "std":0.008132 + }, + "oos_auprc":{ + "mean":0.135344, + "std":0.008028 + } + }, + "test":{ + "aurc":{ + "mean":0.074244, + "std":0.003327 + }, + "oos_auroc":{ + "mean":0.843013, + "std":0.006663 + }, + "oos_auprc":{ + "mean":0.5105, + "std":0.014289 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "msp_t", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":1.887236, + "std":1.619518 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.492043, + "std":0.027744 + }, + "validation_selective_risk":{ + "mean":0.013978, + "std":0.000212 + }, + "validation_risk_upper95":{ + "mean":0.019858, + "std":6.4e-05 + }, + "test":{ + "coverage":{ + "mean":0.427152, + "std":0.026977 + }, + "selective_risk":{ + "mean":0.038118, + "std":0.004536 + }, + "accuracy_8":{ + "mean":0.854909, + "std":0.004639 + }, + "oos_recall":{ + "mean":0.532333, + "std":0.006658 + }, + "oos_recall_wilson95":[ + { + "mean":0.501345, + "std":0.006658 + }, + { + "mean":0.563074, + "std":0.006607 + } + ], + "oos_precision":{ + "mean":0.830373, + "std":0.015865 + }, + "oos_f1":{ + "mean":0.648658, + "std":0.000278 + }, + "oos_misroute_rate":{ + "mean":0.467667, + "std":0.006658 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.065667, + "std":0.014224 + }, + "llm_call_rate":{ + "mean":0.572848, + "std":0.026977 + }, + "llm_cost_usd_per_1k":{ + "mean":0.211576, + "std":0.009981 + }, + "error_recovery_rate":{ + "mean":0.577269, + "std":0.01309 + }, + "recoverable_caught":{ + "mean":0.937686, + "std":0.009298 + }, + "small_errors":{ + "mean":1343.0, + "std":46.518813 + }, + "recoverable_errors":{ + "mean":827.0, + "std":40.447497 + }, + "recovered_errors":{ + "mean":775.666667, + "std":44.377171 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":4.243757, + "std":0.431944 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.47828, + "std":0.039993 + }, + "validation_selective_risk":{ + "mean":0.013926, + "std":0.000235 + }, + "validation_risk_upper95":{ + "mean":0.019903, + "std":7.5e-05 + }, + "test":{ + "coverage":{ + "mean":0.417212, + "std":0.036147 + }, + "selective_risk":{ + "mean":0.038724, + "std":0.004106 + }, + "accuracy_8":{ + "mean":0.853152, + "std":0.006099 + }, + "oos_recall":{ + "mean":0.531, + "std":0.006 + }, + "oos_recall_wilson95":[ + { + "mean":0.500011, + "std":0.006 + }, + { + "mean":0.561752, + "std":0.005954 + } + ], + "oos_precision":{ + "mean":0.825394, + "std":0.019981 + }, + "oos_f1":{ + "mean":0.646133, + "std":0.003851 + }, + "oos_misroute_rate":{ + "mean":0.469, + "std":0.006 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.068333, + "std":0.01193 + }, + "llm_call_rate":{ + "mean":0.582788, + "std":0.036147 + }, + "llm_cost_usd_per_1k":{ + "mean":0.215231, + "std":0.013354 + }, + "error_recovery_rate":{ + "mean":0.5778, + "std":0.011248 + }, + "recoverable_caught":{ + "mean":0.938596, + "std":0.010209 + }, + "small_errors":{ + "mean":1343.0, + "std":46.518813 + }, + "recoverable_errors":{ + "mean":827.0, + "std":40.447497 + }, + "recovered_errors":{ + "mean":776.333333, + "std":42.122836 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.952647, + "std":0.003589 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.477957, + "std":0.008634 + }, + "validation_selective_risk":{ + "mean":0.013723, + "std":0.000269 + }, + "validation_risk_upper95":{ + "mean":0.019655, + "std":0.000315 + }, + "test":{ + "coverage":{ + "mean":0.412606, + "std":0.005675 + }, + "selective_risk":{ + "mean":0.034955, + "std":0.001168 + }, + "accuracy_8":{ + "mean":0.854, + "std":0.003102 + }, + "oos_recall":{ + "mean":0.536, + "std":0.002 + }, + "oos_recall_wilson95":[ + { + "mean":0.505011, + "std":0.002001 + }, + { + "mean":0.566713, + "std":0.001983 + } + ], + "oos_precision":{ + "mean":0.824233, + "std":0.006123 + }, + "oos_f1":{ + "mean":0.649567, + "std":0.001502 + }, + "oos_misroute_rate":{ + "mean":0.464, + "std":0.002 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.056667, + "std":0.002082 + }, + "llm_call_rate":{ + "mean":0.587394, + "std":0.005675 + }, + "llm_cost_usd_per_1k":{ + "mean":0.216961, + "std":0.002078 + }, + "error_recovery_rate":{ + "mean":0.580856, + "std":0.01051 + }, + "recoverable_caught":{ + "mean":0.943541, + "std":0.002365 + }, + "small_errors":{ + "mean":1343.0, + "std":46.518813 + }, + "recoverable_errors":{ + "mean":827.0, + "std":40.447497 + }, + "recovered_errors":{ + "mean":780.333333, + "std":39.004273 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.619049, + "std":0.05162 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.738387, + "std":0.028959 + }, + "validation_selective_risk":{ + "mean":0.04237, + "std":0.000263 + }, + "validation_risk_upper95":{ + "mean":0.049856, + "std":0.000163 + }, + "test":{ + "coverage":{ + "mean":0.668909, + "std":0.025793 + }, + "selective_risk":{ + "mean":0.087451, + "std":0.006418 + }, + "accuracy_8":{ + "mean":0.852545, + "std":0.000182 + }, + "oos_recall":{ + "mean":0.441, + "std":0.005 + }, + "oos_recall_wilson95":[ + { + "mean":0.410511, + "std":0.004944 + }, + { + "mean":0.47194, + "std":0.005017 + } + ], + "oos_precision":{ + "mean":0.883567, + "std":0.018552 + }, + "oos_f1":{ + "mean":0.588259, + "std":0.00137 + }, + "oos_misroute_rate":{ + "mean":0.559, + "std":0.005 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.228, + "std":0.022271 + }, + "llm_call_rate":{ + "mean":0.331091, + "std":0.025793 + }, + "llm_cost_usd_per_1k":{ + "mean":0.122217, + "std":0.009521 + }, + "error_recovery_rate":{ + "mean":0.472827, + "std":0.02128 + }, + "recoverable_caught":{ + "mean":0.767903, + "std":0.02511 + }, + "small_errors":{ + "mean":1343.0, + "std":46.518813 + }, + "recoverable_errors":{ + "mean":827.0, + "std":40.447497 + }, + "recovered_errors":{ + "mean":635.666667, + "std":50.846173 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":1.842639, + "std":0.294142 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.709462, + "std":0.039345 + }, + "validation_selective_risk":{ + "mean":0.042128, + "std":0.000194 + }, + "validation_risk_upper95":{ + "mean":0.049761, + "std":0.000126 + }, + "test":{ + "coverage":{ + "mean":0.643455, + "std":0.033203 + }, + "selective_risk":{ + "mean":0.095534, + "std":0.007447 + }, + "accuracy_8":{ + "mean":0.847152, + "std":0.000458 + }, + "oos_recall":{ + "mean":0.42, + "std":0.005292 + }, + "oos_recall_wilson95":[ + { + "mean":0.389773, + "std":0.005218 + }, + { + "mean":0.450839, + "std":0.005325 + } + ], + "oos_precision":{ + "mean":0.867541, + "std":0.017956 + }, + "oos_f1":{ + "mean":0.565905, + "std":0.002793 + }, + "oos_misroute_rate":{ + "mean":0.58, + "std":0.005292 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.252667, + "std":0.027062 + }, + "llm_call_rate":{ + "mean":0.356545, + "std":0.033203 + }, + "llm_cost_usd_per_1k":{ + "mean":0.131639, + "std":0.012272 + }, + "error_recovery_rate":{ + "mean":0.460818, + "std":0.025887 + }, + "recoverable_caught":{ + "mean":0.748395, + "std":0.034977 + }, + "small_errors":{ + "mean":1343.0, + "std":46.518813 + }, + "recoverable_errors":{ + "mean":827.0, + "std":40.447497 + }, + "recovered_errors":{ + "mean":619.666667, + "std":56.047599 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.619049, + "std":0.05162 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.738387, + "std":0.028959 + }, + "validation_selective_risk":{ + "mean":0.04237, + "std":0.000263 + }, + "validation_risk_upper95":{ + "mean":0.049856, + "std":0.000163 + }, + "test":{ + "coverage":{ + "mean":0.668909, + "std":0.025793 + }, + "selective_risk":{ + "mean":0.087451, + "std":0.006418 + }, + "accuracy_8":{ + "mean":0.852545, + "std":0.000182 + }, + "oos_recall":{ + "mean":0.441, + "std":0.005 + }, + "oos_recall_wilson95":[ + { + "mean":0.410511, + "std":0.004944 + }, + { + "mean":0.47194, + "std":0.005017 + } + ], + "oos_precision":{ + "mean":0.883567, + "std":0.018552 + }, + "oos_f1":{ + "mean":0.588259, + "std":0.00137 + }, + "oos_misroute_rate":{ + "mean":0.559, + "std":0.005 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.228, + "std":0.022271 + }, + "llm_call_rate":{ + "mean":0.331091, + "std":0.025793 + }, + "llm_cost_usd_per_1k":{ + "mean":0.122217, + "std":0.009521 + }, + "error_recovery_rate":{ + "mean":0.472827, + "std":0.02128 + }, + "recoverable_caught":{ + "mean":0.767903, + "std":0.02511 + }, + "small_errors":{ + "mean":1343.0, + "std":46.518813 + }, + "recoverable_errors":{ + "mean":827.0, + "std":40.447497 + }, + "recovered_errors":{ + "mean":635.666667, + "std":50.846173 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.877634, + "std":0.005497 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.251273, + "std":0.005178 + }, + "accuracy_8":{ + "mean":0.748727, + "std":0.005178 + }, + "oos_recall":{ + "mean":0.012667, + "std":0.015011 + }, + "oos_recall_wilson95":[ + { + "mean":0.008069, + "std":0.01128 + }, + { + "mean":0.020994, + "std":0.018628 + } + ], + "oos_precision":{ + "mean":0.989247, + "std":0.018624 + }, + "oos_f1":{ + "mean":0.024711, + "std":0.028999 + }, + "oos_misroute_rate":{ + "mean":0.987333, + "std":0.015011 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.987333, + "std":0.015011 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1382.0, + "std":28.478062 + }, + "recoverable_errors":{ + "mean":855.666667, + "std":26.501572 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.748727, + "std":0.005178 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.904303, + "std":0.000933 + }, + "oos_recall":{ + "mean":0.571333, + "std":0.004041 + }, + "oos_recall_wilson95":[ + { + "mean":0.540445, + "std":0.004062 + }, + { + "mean":0.601675, + "std":0.003989 + } + ], + "oos_precision":{ + "mean":0.936613, + "std":0.000771 + }, + "oos_f1":{ + "mean":0.709725, + "std":0.003115 + }, + "oos_misroute_rate":{ + "mean":0.428667, + "std":0.004041 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.251273, + "std":0.005178 + }, + "llm_cost_usd_per_1k":{ + "mean":0.092497, + "std":0.001899 + }, + "error_recovery_rate":{ + "mean":0.619067, + "std":0.006929 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1382.0, + "std":28.478062 + }, + "recoverable_errors":{ + "mean":855.666667, + "std":26.501572 + }, + "recovered_errors":{ + "mean":855.666667, + "std":26.501572 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.022558, + "std":0.001388 + }, + "oos_auroc":{ + "mean":0.845375, + "std":0.00971 + }, + "oos_auprc":{ + "mean":0.150085, + "std":0.012688 + } + }, + "test":{ + "aurc":{ + "mean":0.067827, + "std":0.002784 + }, + "oos_auroc":{ + "mean":0.844761, + "std":0.010459 + }, + "oos_auprc":{ + "mean":0.481106, + "std":0.025123 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.023849, + "std":0.001578 + }, + "oos_auroc":{ + "mean":0.852648, + "std":0.005677 + }, + "oos_auprc":{ + "mean":0.167545, + "std":0.005504 + } + }, + "test":{ + "aurc":{ + "mean":0.068314, + "std":0.002721 + }, + "oos_auroc":{ + "mean":0.852393, + "std":0.009085 + }, + "oos_auprc":{ + "mean":0.511681, + "std":0.029532 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":3.594459, + "std":0.353858 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.631828, + "std":0.036845 + }, + "validation_selective_risk":{ + "mean":0.014628, + "std":0.000225 + }, + "validation_risk_upper95":{ + "mean":0.019811, + "std":0.000129 + }, + "test":{ + "coverage":{ + "mean":0.553273, + "std":0.031665 + }, + "selective_risk":{ + "mean":0.043691, + "std":0.003285 + }, + "accuracy_8":{ + "mean":0.859879, + "std":0.005277 + }, + "oos_recall":{ + "mean":0.515, + "std":0.004583 + }, + "oos_recall_wilson95":[ + { + "mean":0.484026, + "std":0.004574 + }, + { + "mean":0.545859, + "std":0.004556 + } + ], + "oos_precision":{ + "mean":0.841287, + "std":0.014472 + }, + "oos_f1":{ + "mean":0.63883, + "std":0.001744 + }, + "oos_misroute_rate":{ + "mean":0.485, + "std":0.004583 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.104, + "std":0.015716 + }, + "llm_call_rate":{ + "mean":0.446727, + "std":0.031665 + }, + "llm_cost_usd_per_1k":{ + "mean":0.164925, + "std":0.011714 + }, + "error_recovery_rate":{ + "mean":0.566947, + "std":0.008143 + }, + "recoverable_caught":{ + "mean":0.915789, + "std":0.003395 + }, + "small_errors":{ + "mean":1382.0, + "std":28.478062 + }, + "recoverable_errors":{ + "mean":855.666667, + "std":26.501572 + }, + "recovered_errors":{ + "mean":783.666667, + "std":27.02468 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":3.594459, + "std":0.353858 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.631828, + "std":0.036845 + }, + "validation_selective_risk":{ + "mean":0.014628, + "std":0.000225 + }, + "validation_risk_upper95":{ + "mean":0.019811, + "std":0.000129 + }, + "test":{ + "coverage":{ + "mean":0.553273, + "std":0.031665 + }, + "selective_risk":{ + "mean":0.043691, + "std":0.003285 + }, + "accuracy_8":{ + "mean":0.859879, + "std":0.005277 + }, + "oos_recall":{ + "mean":0.515, + "std":0.004583 + }, + "oos_recall_wilson95":[ + { + "mean":0.484026, + "std":0.004574 + }, + { + "mean":0.545859, + "std":0.004556 + } + ], + "oos_precision":{ + "mean":0.841287, + "std":0.014472 + }, + "oos_f1":{ + "mean":0.63883, + "std":0.001744 + }, + "oos_misroute_rate":{ + "mean":0.485, + "std":0.004583 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.104, + "std":0.015716 + }, + "llm_call_rate":{ + "mean":0.446727, + "std":0.031665 + }, + "llm_cost_usd_per_1k":{ + "mean":0.164925, + "std":0.011714 + }, + "error_recovery_rate":{ + "mean":0.566947, + "std":0.008143 + }, + "recoverable_caught":{ + "mean":0.915789, + "std":0.003395 + }, + "small_errors":{ + "mean":1382.0, + "std":28.478062 + }, + "recoverable_errors":{ + "mean":855.666667, + "std":26.501572 + }, + "recovered_errors":{ + "mean":783.666667, + "std":27.02468 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.960293, + "std":0.005392 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.588387, + "std":0.00952 + }, + "validation_selective_risk":{ + "mean":0.014436, + "std":8.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.019802, + "std":5.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.510727, + "std":0.014404 + }, + "selective_risk":{ + "mean":0.038527, + "std":0.002233 + }, + "accuracy_8":{ + "mean":0.858909, + "std":0.003731 + }, + "oos_recall":{ + "mean":0.527667, + "std":0.001528 + }, + "oos_recall_wilson95":[ + { + "mean":0.496677, + "std":0.001527 + }, + { + "mean":0.558444, + "std":0.001517 + } + ], + "oos_precision":{ + "mean":0.831888, + "std":0.007085 + }, + "oos_f1":{ + "mean":0.645731, + "std":0.002189 + }, + "oos_misroute_rate":{ + "mean":0.472333, + "std":0.001528 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.08, + "std":0.007 + }, + "llm_call_rate":{ + "mean":0.489273, + "std":0.014404 + }, + "llm_cost_usd_per_1k":{ + "mean":0.180667, + "std":0.005335 + }, + "error_recovery_rate":{ + "mean":0.576139, + "std":0.007032 + }, + "recoverable_caught":{ + "mean":0.930651, + "std":0.001696 + }, + "small_errors":{ + "mean":1382.0, + "std":28.478062 + }, + "recoverable_errors":{ + "mean":855.666667, + "std":26.501572 + }, + "recovered_errors":{ + "mean":796.333333, + "std":25.026652 + } + } + } + } + }, + "0.05":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":1.587472, + "std":0.16733 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.814731, + "std":0.020669 + }, + "validation_selective_risk":{ + "mean":0.042627, + "std":0.000149 + }, + "validation_risk_upper95":{ + "mean":0.049744, + "std":9.4e-05 + }, + "test":{ + "coverage":{ + "mean":0.741273, + "std":0.02049 + }, + "selective_risk":{ + "mean":0.103085, + "std":0.005266 + }, + "accuracy_8":{ + "mean":0.847212, + "std":0.000584 + }, + "oos_recall":{ + "mean":0.390667, + "std":0.005508 + }, + "oos_recall_wilson95":[ + { + "mean":0.360901, + "std":0.00541 + }, + { + "mean":0.421269, + "std":0.005563 + } + ], + "oos_precision":{ + "mean":0.88277, + "std":0.020376 + }, + "oos_f1":{ + "mean":0.541597, + "std":0.007557 + }, + "oos_misroute_rate":{ + "mean":0.609333, + "std":0.005508 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.325333, + "std":0.017214 + }, + "llm_call_rate":{ + "mean":0.258727, + "std":0.02049 + }, + "llm_cost_usd_per_1k":{ + "mean":0.095465, + "std":0.00759 + }, + "error_recovery_rate":{ + "mean":0.447455, + "std":0.01572 + }, + "recoverable_caught":{ + "mean":0.722716, + "std":0.020659 + }, + "small_errors":{ + "mean":1382.0, + "std":28.478062 + }, + "recoverable_errors":{ + "mean":855.666667, + "std":26.501572 + }, + "recovered_errors":{ + "mean":618.666667, + "std":33.857545 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":1.587472, + "std":0.16733 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.814731, + "std":0.020669 + }, + "validation_selective_risk":{ + "mean":0.042627, + "std":0.000149 + }, + "validation_risk_upper95":{ + "mean":0.049744, + "std":9.4e-05 + }, + "test":{ + "coverage":{ + "mean":0.741273, + "std":0.02049 + }, + "selective_risk":{ + "mean":0.103085, + "std":0.005266 + }, + "accuracy_8":{ + "mean":0.847212, + "std":0.000584 + }, + "oos_recall":{ + "mean":0.390667, + "std":0.005508 + }, + "oos_recall_wilson95":[ + { + "mean":0.360901, + "std":0.00541 + }, + { + "mean":0.421269, + "std":0.005563 + } + ], + "oos_precision":{ + "mean":0.88277, + "std":0.020376 + }, + "oos_f1":{ + "mean":0.541597, + "std":0.007557 + }, + "oos_misroute_rate":{ + "mean":0.609333, + "std":0.005508 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.325333, + "std":0.017214 + }, + "llm_call_rate":{ + "mean":0.258727, + "std":0.02049 + }, + "llm_cost_usd_per_1k":{ + "mean":0.095465, + "std":0.00759 + }, + "error_recovery_rate":{ + "mean":0.447455, + "std":0.01572 + }, + "recoverable_caught":{ + "mean":0.722716, + "std":0.020659 + }, + "small_errors":{ + "mean":1382.0, + "std":28.478062 + }, + "recoverable_errors":{ + "mean":855.666667, + "std":26.501572 + }, + "recovered_errors":{ + "mean":618.666667, + "std":33.857545 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.710584, + "std":0.049135 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.796882, + "std":0.023666 + }, + "validation_selective_risk":{ + "mean":0.042639, + "std":3.4e-05 + }, + "validation_risk_upper95":{ + "mean":0.049842, + "std":7.7e-05 + }, + "test":{ + "coverage":{ + "mean":0.719636, + "std":0.024843 + }, + "selective_risk":{ + "mean":0.094635, + "std":0.004803 + }, + "accuracy_8":{ + "mean":0.851091, + "std":0.001135 + }, + "oos_recall":{ + "mean":0.424333, + "std":0.003512 + }, + "oos_recall_wilson95":[ + { + "mean":0.394047, + "std":0.003465 + }, + { + "mean":0.455198, + "std":0.003531 + } + ], + "oos_precision":{ + "mean":0.883654, + "std":0.019828 + }, + "oos_f1":{ + "mean":0.573313, + "std":0.006457 + }, + "oos_misroute_rate":{ + "mean":0.575667, + "std":0.003512 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.278667, + "std":0.015044 + }, + "llm_call_rate":{ + "mean":0.280364, + "std":0.024843 + }, + "llm_cost_usd_per_1k":{ + "mean":0.103453, + "std":0.009203 + }, + "error_recovery_rate":{ + "mean":0.468915, + "std":0.015806 + }, + "recoverable_caught":{ + "mean":0.757374, + "std":0.020005 + }, + "small_errors":{ + "mean":1382.0, + "std":28.478062 + }, + "recoverable_errors":{ + "mean":855.666667, + "std":26.501572 + }, + "recovered_errors":{ + "mean":648.333333, + "std":34.77547 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.247394, + "std":0.009406, + "values":[ + 0.252364, + 0.236545, + 0.253273 + ] + }, + "accuracy_8":{ + "mean":0.752606, + "std":0.009406, + "values":[ + 0.747636, + 0.763455, + 0.746727 + ] + }, + "oos_recall":{ + "mean":0.034, + "std":0.043589, + "values":[ + 0.004, + 0.084, + 0.014 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.026091, + "std":0.036762, + "values":[ + 0.001557, + 0.068359, + 0.008357 + ] + }, + { + "mean":0.045476, + "std":0.050098, + "values":[ + 0.01024, + 0.102825, + 0.023362 + ] + } + ], + "oos_precision":{ + "mean":0.955556, + "std":0.03849, + "values":[ + 1.0, + 0.933333, + 0.933333 + ] + }, + "oos_f1":{ + "mean":0.063228, + "std":0.079331, + "values":[ + 0.007968, + 0.154128, + 0.027586 + ] + }, + "oos_misroute_rate":{ + "mean":0.966, + "std":0.043589, + "values":[ + 0.996, + 0.916, + 0.986 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.966, + "std":0.043589, + "values":[ + 0.996, + 0.916, + 0.986 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1360.666667, + "std":51.73329, + "values":[ + 1388, + 1301, + 1393 + ] + }, + "recoverable_errors":{ + "mean":842.0, + "std":39.887341, + "values":[ + 856, + 797, + 873 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.752606, + "std":0.009406, + "values":[ + 0.747636, + 0.763455, + 0.746727 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.905697, + "std":0.002554, + "values":[ + 0.903273, + 0.908364, + 0.905455 + ] + }, + "oos_recall":{ + "mean":0.576667, + "std":0.012423, + "values":[ + 0.569, + 0.591, + 0.57 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.545813, + "std":0.0125, + "values":[ + 0.5381, + 0.560235, + 0.539105 + ] + }, + { + "mean":0.606933, + "std":0.012251, + "values":[ + 0.599372, + 0.621068, + 0.60036 + ] + } + ], + "oos_precision":{ + "mean":0.939161, + "std":0.003366, + "values":[ + 0.935855, + 0.942584, + 0.939044 + ] + }, + "oos_f1":{ + "mean":0.714533, + "std":0.01039, + "values":[ + 0.707711, + 0.72649, + 0.709396 + ] + }, + "oos_misroute_rate":{ + "mean":0.423333, + "std":0.012423, + "values":[ + 0.431, + 0.409, + 0.43 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.247394, + "std":0.009406, + "values":[ + 0.252364, + 0.236545, + 0.253273 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.091082, + "std":0.003459, + "values":[ + 0.092903, + 0.087094, + 0.09325 + ] + }, + "error_recovery_rate":{ + "mean":0.618675, + "std":0.007251, + "values":[ + 0.616715, + 0.612606, + 0.626705 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1360.666667, + "std":51.73329, + "values":[ + 1388, + 1301, + 1393 + ] + }, + "recoverable_errors":{ + "mean":842.0, + "std":39.887341, + "values":[ + 856, + 797, + 873 + ] + }, + "recovered_errors":{ + "mean":842.0, + "std":39.887341, + "values":[ + 856, + 797, + 873 + ] + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.028504, + "std":0.004865, + "values":[ + 0.023499, + 0.028796, + 0.033216 + ] + }, + "oos_auroc":{ + "mean":0.801951, + "std":0.036804, + "values":[ + 0.843795, + 0.787458, + 0.7746 + ] + }, + "oos_auprc":{ + "mean":0.110467, + "std":0.040814, + "values":[ + 0.156998, + 0.093676, + 0.080726 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.077612, + "std":0.009093, + "values":[ + 0.06818, + 0.078332, + 0.086324 + ] + }, + "oos_auroc":{ + "mean":0.81595, + "std":0.023327, + "values":[ + 0.841829, + 0.809482, + 0.79654 + ] + }, + "oos_auprc":{ + "mean":0.429412, + "std":0.039798, + "values":[ + 0.475314, + 0.408368, + 0.404554 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.027456, + "std":0.002299, + "values":[ + 0.025302, + 0.027188, + 0.029878 + ] + }, + "oos_auroc":{ + "mean":0.833187, + "std":0.017518, + "values":[ + 0.853415, + 0.823052, + 0.823093 + ] + }, + "oos_auprc":{ + "mean":0.143491, + "std":0.020345, + "values":[ + 0.165935, + 0.126262, + 0.138275 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.072608, + "std":0.00465, + "values":[ + 0.068822, + 0.071204, + 0.077798 + ] + }, + "oos_auroc":{ + "mean":0.845501, + "std":0.007569, + "values":[ + 0.849715, + 0.850024, + 0.836762 + ] + }, + "oos_auprc":{ + "mean":0.509751, + "std":0.014881, + "values":[ + 0.501314, + 0.526933, + 0.501007 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "margin", + "margin", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":2.820058, + "std":1.614574, + "values":[ + 3.747166, + 3.75729, + 0.955717 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.535806, + "std":0.074913, + "values":[ + 0.616452, + 0.522581, + 0.468387 + ] + }, + "validation_selective_risk":{ + "mean":0.014208, + "std":0.000439, + "values":[ + 0.014652, + 0.014198, + 0.013774 + ] + }, + "validation_risk_upper95":{ + "mean":0.019866, + "std":7e-05, + "values":[ + 0.019908, + 0.019906, + 0.019786 + ] + }, + "test":{ + "coverage":{ + "mean":0.466667, + "std":0.065288, + "values":[ + 0.536, + 0.457636, + 0.406364 + ] + }, + "selective_risk":{ + "mean":0.040089, + "std":0.004537, + "values":[ + 0.042062, + 0.043306, + 0.034899 + ] + }, + "accuracy_8":{ + "mean":0.856182, + "std":0.003632, + "values":[ + 0.855273, + 0.860182, + 0.853091 + ] + }, + "oos_recall":{ + "mean":0.526333, + "std":0.007095, + "values":[ + 0.52, + 0.525, + 0.534 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.495346, + "std":0.007091, + "values":[ + 0.489017, + 0.494012, + 0.50301 + ] + }, + { + "mean":0.557119, + "std":0.007044, + "values":[ + 0.55083, + 0.555797, + 0.56473 + ] + } + ], + "oos_precision":{ + "mean":0.834279, + "std":0.012171, + "values":[ + 0.829346, + 0.848142, + 0.825348 + ] + }, + "oos_f1":{ + "mean":0.645404, + "std":0.005362, + "values":[ + 0.639213, + 0.648548, + 0.648452 + ] + }, + "oos_misroute_rate":{ + "mean":0.473667, + "std":0.007095, + "values":[ + 0.48, + 0.475, + 0.466 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.078, + "std":0.017349, + "values":[ + 0.093, + 0.082, + 0.059 + ] + }, + "llm_call_rate":{ + "mean":0.533333, + "std":0.065288, + "values":[ + 0.464, + 0.542364, + 0.593636 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.196956, + "std":0.024133, + "values":[ + 0.17133, + 0.200292, + 0.219248 + ] + }, + "error_recovery_rate":{ + "mean":0.575278, + "std":0.014747, + "values":[ + 0.565562, + 0.568025, + 0.592247 + ] + }, + "recoverable_caught":{ + "mean":0.929767, + "std":0.014153, + "values":[ + 0.917056, + 0.927227, + 0.945017 + ] + }, + "small_errors":{ + "mean":1360.666667, + "std":51.73329, + "values":[ + 1388, + 1301, + 1393 + ] + }, + "recoverable_errors":{ + "mean":842.0, + "std":39.887341, + "values":[ + 856, + 797, + 873 + ] + }, + "recovered_errors":{ + "mean":783.0, + "std":43.03487, + "values":[ + 785, + 739, + 825 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "margin", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":0.962006, + "std":0.603048, + "values":[ + 1.655836, + 0.563929, + 0.666255 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.761935, + "std":0.041287, + "values":[ + 0.797742, + 0.77129, + 0.716774 + ] + }, + "validation_selective_risk":{ + "mean":0.042474, + "std":0.000178, + "values":[ + 0.042459, + 0.04266, + 0.042304 + ] + }, + "validation_risk_upper95":{ + "mean":0.049845, + "std":0.000181, + "values":[ + 0.049643, + 0.049991, + 0.049902 + ] + }, + "test":{ + "coverage":{ + "mean":0.689879, + "std":0.042173, + "values":[ + 0.728545, + 0.696182, + 0.644909 + ] + }, + "selective_risk":{ + "mean":0.092469, + "std":0.009561, + "values":[ + 0.100824, + 0.094542, + 0.082041 + ] + }, + "accuracy_8":{ + "mean":0.850848, + "std":0.002945, + "values":[ + 0.847455, + 0.852364, + 0.852727 + ] + }, + "oos_recall":{ + "mean":0.426, + "std":0.026458, + "values":[ + 0.396, + 0.436, + 0.446 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.395721, + "std":0.02609, + "values":[ + 0.366141, + 0.405567, + 0.415456 + ] + }, + { + "mean":0.456845, + "std":0.026622, + "values":[ + 0.426655, + 0.466922, + 0.476957 + ] + } + ], + "oos_precision":{ + "mean":0.884003, + "std":0.018325, + "values":[ + 0.878049, + 0.904564, + 0.869396 + ] + }, + "oos_f1":{ + "mean":0.574594, + "std":0.024917, + "values":[ + 0.54583, + 0.588394, + 0.589557 + ] + }, + "oos_misroute_rate":{ + "mean":0.574, + "std":0.026458, + "values":[ + 0.604, + 0.564, + 0.554 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.259333, + "std":0.055365, + "values":[ + 0.318, + 0.252, + 0.208 + ] + }, + "llm_call_rate":{ + "mean":0.310121, + "std":0.042173, + "values":[ + 0.271455, + 0.303818, + 0.355091 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.11447, + "std":0.015577, + "values":[ + 0.100182, + 0.11215, + 0.131076 + ] + }, + "error_recovery_rate":{ + "mean":0.46879, + "std":0.023626, + "values":[ + 0.456052, + 0.454266, + 0.496052 + ] + }, + "recoverable_caught":{ + "mean":0.757513, + "std":0.029471, + "values":[ + 0.739486, + 0.741531, + 0.791523 + ] + }, + "small_errors":{ + "mean":1360.666667, + "std":51.73329, + "values":[ + 1388, + 1301, + 1393 + ] + }, + "recoverable_errors":{ + "mean":842.0, + "std":39.887341, + "values":[ + 856, + 797, + 873 + ] + }, + "recovered_errors":{ + "mean":638.333333, + "std":50.21288, + "values":[ + 633, + 591, + 691 + ] + } + } + } + } + } + } + } + }, + "tfidf-centroid/k5":{ + "model":"tfidf-centroid", + "k":5, + "oos_train":null, + "kind":"tfidf", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "tfidf-centroid-k5-seed42", + "tfidf-centroid-k5-seed43", + "tfidf-centroid-k5-seed44" + ], + "logits_sha256":[ + "cac96d18d9cad091f8dd3822554c9b94ae38e08d90688d161f52560791ccb63b", + "dbfdd8f7f5107d61cae3e48f2e7a99916ecdf6a094047b4053e65fe43bc3ec1d", + "e3e751ba605ba46a492f6f8ba0759c63b41239d46bf01f9513dc0d1d5db175ba" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":4.235868, + "std":0.073504, + "values":[ + 4.317316, + 4.174468, + 4.21582 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":null, + "calibrated":{ + "ece_151":{ + "mean":0.07594, + "std":0.009698, + "values":[ + 0.087047, + 0.069151, + 0.071622 + ] + }, + "ece_8_summed":{ + "mean":0.037196, + "std":0.001077, + "values":[ + 0.038107, + 0.037475, + 0.036007 + ] + } + } + } + }, + "selected_aggregation":[ + "argmax", + "summed", + "summed" + ], + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.805269, + "std":0.003231, + "values":[ + 0.806452, + 0.801613, + 0.807742 + ] + }, + "summed":{ + "mean":0.806129, + "std":0.004029, + "values":[ + 0.802903, + 0.804839, + 0.810645 + ] + } + }, + "signals_reported":[ + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.805269, + "std":0.003231 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.316, + "std":0.00619 + }, + "accuracy_8":{ + "mean":0.684, + "std":0.00619 + }, + "oos_recall":{ + "mean":0.009333, + "std":0.009238 + }, + "oos_recall_wilson95":[ + { + "mean":0.005366, + "std":0.006597 + }, + { + "mean":0.017057, + "std":0.011807 + } + ], + "oos_precision":{ + "mean":0.944444, + "std":0.096225 + }, + "oos_f1":{ + "mean":0.018333, + "std":0.017952 + }, + "oos_misroute_rate":{ + "mean":0.990667, + "std":0.009238 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.990667, + "std":0.009238 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1738.0, + "std":34.044089 + }, + "recoverable_errors":{ + "mean":1155.0, + "std":28.930952 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.684, + "std":0.00619 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.894, + "std":0.00142 + }, + "oos_recall":{ + "mean":0.569667, + "std":0.000577 + }, + "oos_recall_wilson95":[ + { + "mean":0.53877, + "std":0.00058 + }, + { + "mean":0.60003, + "std":0.00057 + } + ], + "oos_precision":{ + "mean":0.894312, + "std":0.004207 + }, + "oos_f1":{ + "mean":0.69599, + "std":0.000878 + }, + "oos_misroute_rate":{ + "mean":0.430333, + "std":0.000577 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.316, + "std":0.00619 + }, + "llm_cost_usd_per_1k":{ + "mean":0.116433, + "std":0.002294 + }, + "error_recovery_rate":{ + "mean":0.664513, + "std":0.004612 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1738.0, + "std":34.044089 + }, + "recoverable_errors":{ + "mean":1155.0, + "std":28.930952 + }, + "recovered_errors":{ + "mean":1155.0, + "std":28.930952 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.073641, + "std":0.000936 + }, + "oos_auroc":{ + "mean":0.715485, + "std":0.009754 + }, + "oos_auprc":{ + "mean":0.06112, + "std":0.002138 + } + }, + "test":{ + "aurc":{ + "mean":0.13745, + "std":0.002605 + }, + "oos_auroc":{ + "mean":0.729705, + "std":0.008185 + }, + "oos_auprc":{ + "mean":0.313133, + "std":0.01042 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.066726, + "std":0.001594 + }, + "oos_auroc":{ + "mean":0.774111, + "std":0.022311 + }, + "oos_auprc":{ + "mean":0.09415, + "std":0.010805 + } + }, + "test":{ + "aurc":{ + "mean":0.1265, + "std":0.003848 + }, + "oos_auroc":{ + "mean":0.768809, + "std":0.011574 + }, + "oos_auprc":{ + "mean":0.376608, + "std":0.020621 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":5.393382, + "std":0.552215 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.217527, + "std":0.040097 + }, + "validation_selective_risk":{ + "mean":0.010253, + "std":0.001215 + }, + "validation_risk_upper95":{ + "mean":0.018956, + "std":0.000826 + }, + "test":{ + "coverage":{ + "mean":0.186121, + "std":0.037334 + }, + "selective_risk":{ + "mean":0.035586, + "std":0.007181 + }, + "accuracy_8":{ + "mean":0.831939, + "std":0.005249 + }, + "oos_recall":{ + "mean":0.556667, + "std":0.002082 + }, + "oos_recall_wilson95":[ + { + "mean":0.525718, + "std":0.002088 + }, + { + "mean":0.587182, + "std":0.002059 + } + ], + "oos_precision":{ + "mean":0.777299, + "std":0.014711 + }, + "oos_f1":{ + "mean":0.648694, + "std":0.004896 + }, + "oos_misroute_rate":{ + "mean":0.443333, + "std":0.002082 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.021667, + "std":0.004933 + }, + "llm_call_rate":{ + "mean":0.813879, + "std":0.037334 + }, + "llm_cost_usd_per_1k":{ + "mean":0.300694, + "std":0.013798 + }, + "error_recovery_rate":{ + "mean":0.653029, + "std":0.001532 + }, + "recoverable_caught":{ + "mean":0.982743, + "std":0.005414 + }, + "small_errors":{ + "mean":1738.0, + "std":34.044089 + }, + "recoverable_errors":{ + "mean":1155.0, + "std":28.930952 + }, + "recovered_errors":{ + "mean":1135.0, + "std":24.879711 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":5.393382, + "std":0.552215 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.217527, + "std":0.040097 + }, + "validation_selective_risk":{ + "mean":0.010253, + "std":0.001215 + }, + "validation_risk_upper95":{ + "mean":0.018956, + "std":0.000826 + }, + "test":{ + "coverage":{ + "mean":0.186121, + "std":0.037334 + }, + "selective_risk":{ + "mean":0.035586, + "std":0.007181 + }, + "accuracy_8":{ + "mean":0.831939, + "std":0.005249 + }, + "oos_recall":{ + "mean":0.556667, + "std":0.002082 + }, + "oos_recall_wilson95":[ + { + "mean":0.525718, + "std":0.002088 + }, + { + "mean":0.587182, + "std":0.002059 + } + ], + "oos_precision":{ + "mean":0.777299, + "std":0.014711 + }, + "oos_f1":{ + "mean":0.648694, + "std":0.004896 + }, + "oos_misroute_rate":{ + "mean":0.443333, + "std":0.002082 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.021667, + "std":0.004933 + }, + "llm_call_rate":{ + "mean":0.813879, + "std":0.037334 + }, + "llm_cost_usd_per_1k":{ + "mean":0.300694, + "std":0.013798 + }, + "error_recovery_rate":{ + "mean":0.653029, + "std":0.001532 + }, + "recoverable_caught":{ + "mean":0.982743, + "std":0.005414 + }, + "small_errors":{ + "mean":1738.0, + "std":34.044089 + }, + "recoverable_errors":{ + "mean":1155.0, + "std":28.930952 + }, + "recovered_errors":{ + "mean":1135.0, + "std":24.879711 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.988566, + "std":0.007348 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.163548, + "std":0.045632 + }, + "validation_selective_risk":{ + "mean":0.008171, + "std":0.002225 + }, + "validation_risk_upper95":{ + "mean":0.018088, + "std":0.001305 + }, + "test":{ + "coverage":{ + "mean":0.139697, + "std":0.036155 + }, + "selective_risk":{ + "mean":0.030081, + "std":0.008388 + }, + "accuracy_8":{ + "mean":0.828061, + "std":0.002956 + }, + "oos_recall":{ + "mean":0.561, + "std":0.003606 + }, + "oos_recall_wilson95":[ + { + "mean":0.530066, + "std":0.003619 + }, + { + "mean":0.591467, + "std":0.003564 + } + ], + "oos_precision":{ + "mean":0.766113, + "std":0.00673 + }, + "oos_f1":{ + "mean":0.647681, + "std":0.000421 + }, + "oos_misroute_rate":{ + "mean":0.439, + "std":0.003606 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.012667, + "std":0.006658 + }, + "llm_call_rate":{ + "mean":0.860303, + "std":0.036155 + }, + "llm_cost_usd_per_1k":{ + "mean":0.317874, + "std":0.013362 + }, + "error_recovery_rate":{ + "mean":0.656287, + "std":0.001785 + }, + "recoverable_caught":{ + "mean":0.987643, + "std":0.005113 + }, + "small_errors":{ + "mean":1738.0, + "std":34.044089 + }, + "recoverable_errors":{ + "mean":1155.0, + "std":28.930952 + }, + "recovered_errors":{ + "mean":1140.666667, + "std":25.423087 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.842134, + "std":0.023514 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.402473, + "std":0.020842 + }, + "validation_selective_risk":{ + "mean":0.039246, + "std":0.000978 + }, + "validation_risk_upper95":{ + "mean":0.049337, + "std":0.000872 + }, + "test":{ + "coverage":{ + "mean":0.353212, + "std":0.022652 + }, + "selective_risk":{ + "mean":0.065921, + "std":0.00327 + }, + "accuracy_8":{ + "mean":0.837091, + "std":0.004821 + }, + "oos_recall":{ + "mean":0.529333, + "std":0.004726 + }, + "oos_recall_wilson95":[ + { + "mean":0.498344, + "std":0.004724 + }, + { + "mean":0.560098, + "std":0.004691 + } + ], + "oos_precision":{ + "mean":0.793746, + "std":0.011369 + }, + "oos_f1":{ + "mean":0.635077, + "std":0.003596 + }, + "oos_misroute_rate":{ + "mean":0.470667, + "std":0.004726 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.071667, + "std":0.012503 + }, + "llm_call_rate":{ + "mean":0.646788, + "std":0.022652 + }, + "llm_cost_usd_per_1k":{ + "mean":0.238899, + "std":0.008381 + }, + "error_recovery_rate":{ + "mean":0.618062, + "std":0.007137 + }, + "recoverable_caught":{ + "mean":0.930111, + "std":0.01058 + }, + "small_errors":{ + "mean":1738.0, + "std":34.044089 + }, + "recoverable_errors":{ + "mean":1155.0, + "std":28.930952 + }, + "recovered_errors":{ + "mean":1074.333333, + "std":32.501282 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":3.447295, + "std":0.134807 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.370645, + "std":0.01346 + }, + "validation_selective_risk":{ + "mean":0.039157, + "std":0.000353 + }, + "validation_risk_upper95":{ + "mean":0.049707, + "std":0.000234 + }, + "test":{ + "coverage":{ + "mean":0.329273, + "std":0.009053 + }, + "selective_risk":{ + "mean":0.065931, + "std":0.00293 + }, + "accuracy_8":{ + "mean":0.835697, + "std":0.002612 + }, + "oos_recall":{ + "mean":0.528333, + "std":0.001528 + }, + "oos_recall_wilson95":[ + { + "mean":0.497344, + "std":0.001527 + }, + { + "mean":0.559106, + "std":0.001516 + } + ], + "oos_precision":{ + "mean":0.795351, + "std":0.008429 + }, + "oos_f1":{ + "mean":0.634895, + "std":0.00232 + }, + "oos_misroute_rate":{ + "mean":0.471667, + "std":0.001528 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.072333, + "std":0.003786 + }, + "llm_call_rate":{ + "mean":0.670727, + "std":0.009053 + }, + "llm_cost_usd_per_1k":{ + "mean":0.247755, + "std":0.003344 + }, + "error_recovery_rate":{ + "mean":0.621909, + "std":0.005552 + }, + "recoverable_caught":{ + "mean":0.935883, + "std":0.004001 + }, + "small_errors":{ + "mean":1738.0, + "std":34.044089 + }, + "recoverable_errors":{ + "mean":1155.0, + "std":28.930952 + }, + "recovered_errors":{ + "mean":1081.0, + "std":30.512293 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.842134, + "std":0.023514 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.402473, + "std":0.020842 + }, + "validation_selective_risk":{ + "mean":0.039246, + "std":0.000978 + }, + "validation_risk_upper95":{ + "mean":0.049337, + "std":0.000872 + }, + "test":{ + "coverage":{ + "mean":0.353212, + "std":0.022652 + }, + "selective_risk":{ + "mean":0.065921, + "std":0.00327 + }, + "accuracy_8":{ + "mean":0.837091, + "std":0.004821 + }, + "oos_recall":{ + "mean":0.529333, + "std":0.004726 + }, + "oos_recall_wilson95":[ + { + "mean":0.498344, + "std":0.004724 + }, + { + "mean":0.560098, + "std":0.004691 + } + ], + "oos_precision":{ + "mean":0.793746, + "std":0.011369 + }, + "oos_f1":{ + "mean":0.635077, + "std":0.003596 + }, + "oos_misroute_rate":{ + "mean":0.470667, + "std":0.004726 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.071667, + "std":0.012503 + }, + "llm_call_rate":{ + "mean":0.646788, + "std":0.022652 + }, + "llm_cost_usd_per_1k":{ + "mean":0.238899, + "std":0.008381 + }, + "error_recovery_rate":{ + "mean":0.618062, + "std":0.007137 + }, + "recoverable_caught":{ + "mean":0.930111, + "std":0.01058 + }, + "small_errors":{ + "mean":1738.0, + "std":34.044089 + }, + "recoverable_errors":{ + "mean":1155.0, + "std":28.930952 + }, + "recovered_errors":{ + "mean":1074.333333, + "std":32.501282 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.806129, + "std":0.004029 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.317879, + "std":0.005437 + }, + "accuracy_8":{ + "mean":0.682121, + "std":0.005437 + }, + "oos_recall":{ + "mean":0.003, + "std":0.004359 + }, + "oos_recall_wilson95":[ + { + "mean":0.001412, + "std":0.002294 + }, + { + "mean":0.008392, + "std":0.006399 + } + ], + "oos_precision":{ + "mean":0.666667, + "std":0.57735 + }, + "oos_f1":{ + "mean":0.005957, + "std":0.008645 + }, + "oos_misroute_rate":{ + "mean":0.997, + "std":0.004359 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.997, + "std":0.004359 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1748.333333, + "std":29.905406 + }, + "recoverable_errors":{ + "mean":1160.333333, + "std":24.583192 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.682121, + "std":0.005437 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.893091, + "std":0.002271 + }, + "oos_recall":{ + "mean":0.568, + "std":0.0 + }, + "oos_recall_wilson95":[ + { + "mean":0.537095, + "std":0.0 + }, + { + "mean":0.598384, + "std":0.0 + } + ], + "oos_precision":{ + "mean":0.893099, + "std":0.004849 + }, + "oos_f1":{ + "mean":0.694379, + "std":0.001468 + }, + "oos_misroute_rate":{ + "mean":0.432, + "std":0.0 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.317879, + "std":0.005437 + }, + "llm_cost_usd_per_1k":{ + "mean":0.117134, + "std":0.002011 + }, + "error_recovery_rate":{ + "mean":0.663662, + "std":0.005923 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1748.333333, + "std":29.905406 + }, + "recoverable_errors":{ + "mean":1160.333333, + "std":24.583192 + }, + "recovered_errors":{ + "mean":1160.333333, + "std":24.583192 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.052452, + "std":0.003244 + }, + "oos_auroc":{ + "mean":0.792295, + "std":0.018979 + }, + "oos_auprc":{ + "mean":0.094886, + "std":0.01288 + } + }, + "test":{ + "aurc":{ + "mean":0.113493, + "std":0.002095 + }, + "oos_auroc":{ + "mean":0.774836, + "std":0.006868 + }, + "oos_auprc":{ + "mean":0.363666, + "std":0.007367 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.054968, + "std":0.002444 + }, + "oos_auroc":{ + "mean":0.798789, + "std":0.021009 + }, + "oos_auprc":{ + "mean":0.115659, + "std":0.008916 + } + }, + "test":{ + "aurc":{ + "mean":0.115056, + "std":0.002308 + }, + "oos_auroc":{ + "mean":0.781752, + "std":0.006551 + }, + "oos_auprc":{ + "mean":0.386335, + "std":0.010119 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":4.933781, + "std":0.29405 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.291075, + "std":0.029898 + }, + "validation_selective_risk":{ + "mean":0.011437, + "std":0.000927 + }, + "validation_risk_upper95":{ + "mean":0.018917, + "std":0.001145 + }, + "test":{ + "coverage":{ + "mean":0.250667, + "std":0.016625 + }, + "selective_risk":{ + "mean":0.032536, + "std":0.005565 + }, + "accuracy_8":{ + "mean":0.836121, + "std":0.003274 + }, + "oos_recall":{ + "mean":0.554, + "std":0.005292 + }, + "oos_recall_wilson95":[ + { + "mean":0.523044, + "std":0.005308 + }, + { + "mean":0.584543, + "std":0.005235 + } + ], + "oos_precision":{ + "mean":0.773047, + "std":0.003865 + }, + "oos_f1":{ + "mean":0.645429, + "std":0.003192 + }, + "oos_misroute_rate":{ + "mean":0.446, + "std":0.005292 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.027333, + "std":0.008083 + }, + "llm_call_rate":{ + "mean":0.749333, + "std":0.016625 + }, + "llm_cost_usd_per_1k":{ + "mean":0.276836, + "std":0.00614 + }, + "error_recovery_rate":{ + "mean":0.648616, + "std":0.001151 + }, + "recoverable_caught":{ + "mean":0.977371, + "std":0.006981 + }, + "small_errors":{ + "mean":1748.333333, + "std":29.905406 + }, + "recoverable_errors":{ + "mean":1160.333333, + "std":24.583192 + }, + "recovered_errors":{ + "mean":1134.0, + "std":19.697716 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":4.933781, + "std":0.29405 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.291075, + "std":0.029898 + }, + "validation_selective_risk":{ + "mean":0.011437, + "std":0.000927 + }, + "validation_risk_upper95":{ + "mean":0.018917, + "std":0.001145 + }, + "test":{ + "coverage":{ + "mean":0.250667, + "std":0.016625 + }, + "selective_risk":{ + "mean":0.032536, + "std":0.005565 + }, + "accuracy_8":{ + "mean":0.836121, + "std":0.003274 + }, + "oos_recall":{ + "mean":0.554, + "std":0.005292 + }, + "oos_recall_wilson95":[ + { + "mean":0.523044, + "std":0.005308 + }, + { + "mean":0.584543, + "std":0.005235 + } + ], + "oos_precision":{ + "mean":0.773047, + "std":0.003865 + }, + "oos_f1":{ + "mean":0.645429, + "std":0.003192 + }, + "oos_misroute_rate":{ + "mean":0.446, + "std":0.005292 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.027333, + "std":0.008083 + }, + "llm_call_rate":{ + "mean":0.749333, + "std":0.016625 + }, + "llm_cost_usd_per_1k":{ + "mean":0.276836, + "std":0.00614 + }, + "error_recovery_rate":{ + "mean":0.648616, + "std":0.001151 + }, + "recoverable_caught":{ + "mean":0.977371, + "std":0.006981 + }, + "small_errors":{ + "mean":1748.333333, + "std":29.905406 + }, + "recoverable_errors":{ + "mean":1160.333333, + "std":24.583192 + }, + "recovered_errors":{ + "mean":1134.0, + "std":19.697716 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.987303, + "std":0.005738 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.249462, + "std":0.037322 + }, + "validation_selective_risk":{ + "mean":0.011175, + "std":0.000367 + }, + "validation_risk_upper95":{ + "mean":0.01938, + "std":0.000429 + }, + "test":{ + "coverage":{ + "mean":0.216667, + "std":0.025328 + }, + "selective_risk":{ + "mean":0.03336, + "std":0.004881 + }, + "accuracy_8":{ + "mean":0.833636, + "std":0.002834 + }, + "oos_recall":{ + "mean":0.555333, + "std":0.004933 + }, + "oos_recall_wilson95":[ + { + "mean":0.524381, + "std":0.004949 + }, + { + "mean":0.585862, + "std":0.004879 + } + ], + "oos_precision":{ + "mean":0.76813, + "std":0.003862 + }, + "oos_f1":{ + "mean":0.644605, + "std":0.002155 + }, + "oos_misroute_rate":{ + "mean":0.444667, + "std":0.004933 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.023667, + "std":0.006807 + }, + "llm_call_rate":{ + "mean":0.783333, + "std":0.025328 + }, + "llm_cost_usd_per_1k":{ + "mean":0.28942, + "std":0.009364 + }, + "error_recovery_rate":{ + "mean":0.650149, + "std":0.002475 + }, + "recoverable_caught":{ + "mean":0.979671, + "std":0.005706 + }, + "small_errors":{ + "mean":1748.333333, + "std":29.905406 + }, + "recoverable_errors":{ + "mean":1160.333333, + "std":24.583192 + }, + "recovered_errors":{ + "mean":1136.666667, + "std":18.929694 + } + } + } + } + }, + "0.05":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":2.358516, + "std":0.085141 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.559355, + "std":0.016603 + }, + "validation_selective_risk":{ + "mean":0.041139, + "std":0.000176 + }, + "validation_risk_upper95":{ + "mean":0.049727, + "std":0.000255 + }, + "test":{ + "coverage":{ + "mean":0.493394, + "std":0.017211 + }, + "selective_risk":{ + "mean":0.079743, + "std":0.001124 + }, + "accuracy_8":{ + "mean":0.839212, + "std":0.004572 + }, + "oos_recall":{ + "mean":0.504667, + "std":0.008622 + }, + "oos_recall_wilson95":[ + { + "mean":0.473722, + "std":0.008595 + }, + { + "mean":0.535575, + "std":0.008582 + } + ], + "oos_precision":{ + "mean":0.807446, + "std":0.006176 + }, + "oos_f1":{ + "mean":0.621107, + "std":0.007965 + }, + "oos_misroute_rate":{ + "mean":0.495333, + "std":0.008622 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.132667, + "std":0.004041 + }, + "llm_call_rate":{ + "mean":0.506606, + "std":0.017211 + }, + "llm_cost_usd_per_1k":{ + "mean":0.187042, + "std":0.006399 + }, + "error_recovery_rate":{ + "mean":0.591908, + "std":0.010305 + }, + "recoverable_caught":{ + "mean":0.89185, + "std":0.009624 + }, + "small_errors":{ + "mean":1748.333333, + "std":29.905406 + }, + "recoverable_errors":{ + "mean":1160.333333, + "std":24.583192 + }, + "recovered_errors":{ + "mean":1035.0, + "std":32.924155 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":2.358516, + "std":0.085141 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.559355, + "std":0.016603 + }, + "validation_selective_risk":{ + "mean":0.041139, + "std":0.000176 + }, + "validation_risk_upper95":{ + "mean":0.049727, + "std":0.000255 + }, + "test":{ + "coverage":{ + "mean":0.493394, + "std":0.017211 + }, + "selective_risk":{ + "mean":0.079743, + "std":0.001124 + }, + "accuracy_8":{ + "mean":0.839212, + "std":0.004572 + }, + "oos_recall":{ + "mean":0.504667, + "std":0.008622 + }, + "oos_recall_wilson95":[ + { + "mean":0.473722, + "std":0.008595 + }, + { + "mean":0.535575, + "std":0.008582 + } + ], + "oos_precision":{ + "mean":0.807446, + "std":0.006176 + }, + "oos_f1":{ + "mean":0.621107, + "std":0.007965 + }, + "oos_misroute_rate":{ + "mean":0.495333, + "std":0.008622 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.132667, + "std":0.004041 + }, + "llm_call_rate":{ + "mean":0.506606, + "std":0.017211 + }, + "llm_cost_usd_per_1k":{ + "mean":0.187042, + "std":0.006399 + }, + "error_recovery_rate":{ + "mean":0.591908, + "std":0.010305 + }, + "recoverable_caught":{ + "mean":0.89185, + "std":0.009624 + }, + "small_errors":{ + "mean":1748.333333, + "std":29.905406 + }, + "recoverable_errors":{ + "mean":1160.333333, + "std":24.583192 + }, + "recovered_errors":{ + "mean":1035.0, + "std":32.924155 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.837979, + "std":0.0135 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.53129, + "std":0.019293 + }, + "validation_selective_risk":{ + "mean":0.040876, + "std":0.000282 + }, + "validation_risk_upper95":{ + "mean":0.049686, + "std":0.000144 + }, + "test":{ + "coverage":{ + "mean":0.463394, + "std":0.020419 + }, + "selective_risk":{ + "mean":0.075186, + "std":0.001202 + }, + "accuracy_8":{ + "mean":0.839212, + "std":0.005684 + }, + "oos_recall":{ + "mean":0.512, + "std":0.006557 + }, + "oos_recall_wilson95":[ + { + "mean":0.481034, + "std":0.006541 + }, + { + "mean":0.542874, + "std":0.006523 + } + ], + "oos_precision":{ + "mean":0.802498, + "std":0.008451 + }, + "oos_f1":{ + "mean":0.625148, + "std":0.007375 + }, + "oos_misroute_rate":{ + "mean":0.488, + "std":0.006557 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.112667, + "std":0.007767 + }, + "llm_call_rate":{ + "mean":0.536606, + "std":0.020419 + }, + "llm_cost_usd_per_1k":{ + "mean":0.198157, + "std":0.007574 + }, + "error_recovery_rate":{ + "mean":0.598184, + "std":0.010626 + }, + "recoverable_caught":{ + "mean":0.901313, + "std":0.011299 + }, + "small_errors":{ + "mean":1748.333333, + "std":29.905406 + }, + "recoverable_errors":{ + "mean":1160.333333, + "std":24.583192 + }, + "recovered_errors":{ + "mean":1046.0, + "std":34.698703 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.317455, + "std":0.004892, + "values":[ + 0.321455, + 0.318909, + 0.312 + ] + }, + "accuracy_8":{ + "mean":0.682545, + "std":0.004892, + "values":[ + 0.678545, + 0.681091, + 0.688 + ] + }, + "oos_recall":{ + "mean":0.007, + "std":0.011269, + "values":[ + 0.02, + 0.0, + 0.001 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.004387, + "std":0.007446, + "values":[ + 0.012984, + 0.0, + 0.000177 + ] + }, + { + "mean":0.013387, + "std":0.015013, + "values":[ + 0.03069, + 0.003827, + 0.005643 + ] + } + ], + "oos_precision":{ + "mean":0.611111, + "std":0.535758, + "values":[ + 0.833333, + 0.0, + 1.0 + ] + }, + "oos_f1":{ + "mean":0.013687, + "std":0.021999, + "values":[ + 0.039062, + 0.0, + 0.001998 + ] + }, + "oos_misroute_rate":{ + "mean":0.993, + "std":0.011269, + "values":[ + 0.98, + 1.0, + 0.999 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.993, + "std":0.011269, + "values":[ + 0.98, + 1.0, + 0.999 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1746.0, + "std":26.907248, + "values":[ + 1768, + 1754, + 1716 + ] + }, + "recoverable_errors":{ + "mean":1161.333333, + "std":25.403412, + "values":[ + 1176, + 1176, + 1132 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.682545, + "std":0.004892, + "values":[ + 0.678545, + 0.681091, + 0.688 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.893697, + "std":0.001277, + "values":[ + 0.892364, + 0.894909, + 0.893818 + ] + }, + "oos_recall":{ + "mean":0.568667, + "std":0.001155, + "values":[ + 0.57, + 0.568, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537765, + "std":0.00116, + "values":[ + 0.539105, + 0.537095, + 0.537095 + ] + }, + { + "mean":0.599043, + "std":0.00114, + "values":[ + 0.60036, + 0.598384, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.894141, + "std":0.003045, + "values":[ + 0.890625, + 0.895899, + 0.895899 + ] + }, + "oos_f1":{ + "mean":0.695192, + "std":6e-05, + "values":[ + 0.695122, + 0.695226, + 0.695226 + ] + }, + "oos_misroute_rate":{ + "mean":0.431333, + "std":0.001155, + "values":[ + 0.43, + 0.432, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.317455, + "std":0.004892, + "values":[ + 0.321455, + 0.318909, + 0.312 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.116977, + "std":0.00181, + "values":[ + 0.118458, + 0.117514, + 0.11496 + ] + }, + "error_recovery_rate":{ + "mean":0.6651, + "std":0.005397, + "values":[ + 0.665158, + 0.670468, + 0.659674 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1746.0, + "std":26.907248, + "values":[ + 1768, + 1754, + 1716 + ] + }, + "recoverable_errors":{ + "mean":1161.333333, + "std":25.403412, + "values":[ + 1176, + 1176, + 1132 + ] + }, + "recovered_errors":{ + "mean":1161.333333, + "std":25.403412, + "values":[ + 1176, + 1176, + 1132 + ] + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.058448, + "std":0.013501, + "values":[ + 0.073992, + 0.04964, + 0.051713 + ] + }, + "oos_auroc":{ + "mean":0.770668, + "std":0.056436, + "values":[ + 0.705503, + 0.80283, + 0.80367 + ] + }, + "oos_auprc":{ + "mean":0.086916, + "std":0.023614, + "values":[ + 0.062562, + 0.109713, + 0.088474 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.120968, + "std":0.015015, + "values":[ + 0.138301, + 0.112672, + 0.111932 + ] + }, + "oos_auroc":{ + "mean":0.760008, + "std":0.032461, + "values":[ + 0.722553, + 0.777484, + 0.779986 + ] + }, + "oos_auprc":{ + "mean":0.345778, + "std":0.037802, + "values":[ + 0.302274, + 0.370607, + 0.364455 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.058555, + "std":0.008643, + "values":[ + 0.068528, + 0.053252, + 0.053884 + ] + }, + "oos_auroc":{ + "mean":0.790177, + "std":0.035664, + "values":[ + 0.74942, + 0.805452, + 0.81566 + ] + }, + "oos_auprc":{ + "mean":0.10672, + "std":0.021463, + "values":[ + 0.083568, + 0.125953, + 0.110637 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.119108, + "std":0.009298, + "values":[ + 0.129834, + 0.114162, + 0.113329 + ] + }, + "oos_auroc":{ + "mean":0.776343, + "std":0.015487, + "values":[ + 0.758799, + 0.78212, + 0.788112 + ] + }, + "oos_auprc":{ + "mean":0.380462, + "std":0.019248, + "values":[ + 0.359372, + 0.384932, + 0.397082 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":4.951801, + "std":0.274003, + "values":[ + 4.759164, + 4.830757, + 5.265481 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.276452, + "std":0.030523, + "values":[ + 0.260968, + 0.311613, + 0.256774 + ] + }, + "validation_selective_risk":{ + "mean":0.011618, + "std":0.000703, + "values":[ + 0.011125, + 0.012422, + 0.011307 + ] + }, + "validation_risk_upper95":{ + "mean":0.019385, + "std":0.000387, + "values":[ + 0.019025, + 0.019794, + 0.019334 + ] + }, + "test":{ + "coverage":{ + "mean":0.24103, + "std":0.018519, + "values":[ + 0.229091, + 0.262364, + 0.231636 + ] + }, + "selective_risk":{ + "mean":0.034516, + "std":0.004112, + "values":[ + 0.034127, + 0.038808, + 0.030612 + ] + }, + "accuracy_8":{ + "mean":0.835636, + "std":0.002565, + "values":[ + 0.838, + 0.836, + 0.832909 + ] + }, + "oos_recall":{ + "mean":0.555333, + "std":0.005033, + "values":[ + 0.556, + 0.55, + 0.56 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.524381, + "std":0.005048, + "values":[ + 0.525049, + 0.519032, + 0.529062 + ] + }, + { + "mean":0.585862, + "std":0.00498, + "values":[ + 0.586523, + 0.580585, + 0.590479 + ] + } + ], + "oos_precision":{ + "mean":0.778654, + "std":0.013548, + "values":[ + 0.794286, + 0.771388, + 0.770289 + ] + }, + "oos_f1":{ + "mean":0.648263, + "std":0.005989, + "values":[ + 0.654118, + 0.642148, + 0.648523 + ] + }, + "oos_misroute_rate":{ + "mean":0.444667, + "std":0.005033, + "values":[ + 0.444, + 0.45, + 0.44 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.027, + "std":0.008185, + "values":[ + 0.025, + 0.036, + 0.02 + ] + }, + "llm_call_rate":{ + "mean":0.75897, + "std":0.018519, + "values":[ + 0.770909, + 0.737636, + 0.768364 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.280397, + "std":0.006843, + "values":[ + 0.284812, + 0.272515, + 0.283865 + ] + }, + "error_recovery_rate":{ + "mean":0.650791, + "std":0.00328, + "values":[ + 0.654412, + 0.649943, + 0.648019 + ] + }, + "recoverable_caught":{ + "mean":0.978521, + "std":0.007946, + "values":[ + 0.983844, + 0.969388, + 0.982332 + ] + }, + "small_errors":{ + "mean":1746.0, + "std":26.907248, + "values":[ + 1768, + 1754, + 1716 + ] + }, + "recoverable_errors":{ + "mean":1161.333333, + "std":25.403412, + "values":[ + 1176, + 1176, + 1132 + ] + }, + "recovered_errors":{ + "mean":1136.333333, + "std":22.722969, + "values":[ + 1157, + 1140, + 1112 + ] + } + } + } + }, + "0.05":{ + "selected_signal":[ + "msp_t", + "margin", + "margin" + ], + "hybrid":{ + "tau":{ + "mean":1.83595, + "std":0.849718, + "values":[ + 0.857446, + 2.387806, + 2.262597 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.508387, + "std":0.103834, + "values":[ + 0.38871, + 0.561935, + 0.574516 + ] + }, + "validation_selective_risk":{ + "mean":0.040165, + "std":0.001732, + "values":[ + 0.038174, + 0.041332, + 0.040988 + ] + }, + "validation_risk_upper95":{ + "mean":0.049229, + "std":0.000809, + "values":[ + 0.048337, + 0.049914, + 0.049437 + ] + }, + "test":{ + "coverage":{ + "mean":0.446485, + "std":0.096936, + "values":[ + 0.334909, + 0.494545, + 0.51 + ] + }, + "selective_risk":{ + "mean":0.073902, + "std":0.008993, + "values":[ + 0.063518, + 0.079044, + 0.079144 + ] + }, + "accuracy_8":{ + "mean":0.839091, + "std":0.004639, + "values":[ + 0.837273, + 0.844364, + 0.835636 + ] + }, + "oos_recall":{ + "mean":0.514667, + "std":0.018009, + "values":[ + 0.533, + 0.514, + 0.497 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.483706, + "std":0.017974, + "values":[ + 0.50201, + 0.483028, + 0.466081 + ] + }, + { + "mean":0.545515, + "std":0.017907, + "values":[ + 0.563737, + 0.544865, + 0.527942 + ] + } + ], + "oos_precision":{ + "mean":0.802984, + "std":0.006813, + "values":[ + 0.797904, + 0.810726, + 0.800322 + ] + }, + "oos_f1":{ + "mean":0.62714, + "std":0.013058, + "values":[ + 0.639089, + 0.629131, + 0.613202 + ] + }, + "oos_misroute_rate":{ + "mean":0.485333, + "std":0.018009, + "values":[ + 0.467, + 0.486, + 0.503 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.110667, + "std":0.041356, + "values":[ + 0.063, + 0.132, + 0.137 + ] + }, + "llm_call_rate":{ + "mean":0.553515, + "std":0.096936, + "values":[ + 0.665091, + 0.505455, + 0.49 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.204382, + "std":0.035877, + "values":[ + 0.245678, + 0.186581, + 0.180886 + ] + }, + "error_recovery_rate":{ + "mean":0.602872, + "std":0.022597, + "values":[ + 0.626131, + 0.601482, + 0.581002 + ] + }, + "recoverable_caught":{ + "mean":0.906392, + "std":0.031341, + "values":[ + 0.941327, + 0.897109, + 0.880742 + ] + }, + "small_errors":{ + "mean":1746.0, + "std":26.907248, + "values":[ + 1768, + 1754, + 1716 + ] + }, + "recoverable_errors":{ + "mean":1161.333333, + "std":25.403412, + "values":[ + 1176, + 1176, + 1132 + ] + }, + "recovered_errors":{ + "mean":1053.0, + "std":55.027266, + "values":[ + 1107, + 1055, + 997 + ] + } + } + } + } + } + } + } + }, + "tfidf-centroid/k50":{ + "model":"tfidf-centroid", + "k":50, + "oos_train":null, + "kind":"tfidf", + "seeds":[ + 42, + 43, + 44 + ], + "run_names":[ + "tfidf-centroid-k50-seed42", + "tfidf-centroid-k50-seed43", + "tfidf-centroid-k50-seed44" + ], + "logits_sha256":[ + "27608c5f570bcb13a37d9cff167a0c2980f2915bc0fbce190c0027b0e52c3b93", + "9f8eed2728260b4b58e05b7e7d70b79b4b5f74d6b9f7a01decc8cff13e42aa2f", + "3fa2c12a4a881fe0c9148c86acc9be6edd315d968621aecf6f7936c0927d79ab" + ], + "result":{ + "calibration":{ + "temperature":{ + "mean":3.110889, + "std":0.040684, + "values":[ + 3.071946, + 3.107605, + 3.153116 + ] + }, + "hit_search_bound":false, + "test_ece":{ + "uncalibrated":null, + "calibrated":{ + "ece_151":{ + "mean":0.073844, + "std":0.005567, + "values":[ + 0.075536, + 0.078367, + 0.067627 + ] + }, + "ece_8_summed":{ + "mean":0.066992, + "std":0.002042, + "values":[ + 0.066748, + 0.069145, + 0.065082 + ] + } + } + } + }, + "selected_aggregation":"argmax", + "aggregation_validation_accuracy_8":{ + "argmax":{ + "mean":0.887097, + "std":0.005369, + "values":[ + 0.884839, + 0.893226, + 0.883226 + ] + }, + "summed":{ + "mean":0.884194, + "std":0.005597, + "values":[ + 0.882903, + 0.890323, + 0.879355 + ] + } + }, + "signals_reported":[ + "margin", + "msp_t" + ], + "argmax":{ + "validation_accuracy_8":{ + "mean":0.887097, + "std":0.005369 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.235455, + "std":0.003933 + }, + "accuracy_8":{ + "mean":0.764545, + "std":0.003933 + }, + "oos_recall":{ + "mean":0.061667, + "std":0.018583 + }, + "oos_recall_wilson95":[ + { + "mean":0.048487, + "std":0.016518 + }, + { + "mean":0.078201, + "std":0.020506 + } + ], + "oos_precision":{ + "mean":0.937901, + "std":0.035233 + }, + "oos_f1":{ + "mean":0.115419, + "std":0.032746 + }, + "oos_misroute_rate":{ + "mean":0.938333, + "std":0.018583 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.938333, + "std":0.018583 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308 + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.764545, + "std":0.003933 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.908606, + "std":0.00129 + }, + "oos_recall":{ + "mean":0.582, + "std":0.003606 + }, + "oos_recall_wilson95":[ + { + "mean":0.551173, + "std":0.003629 + }, + { + "mean":0.612199, + "std":0.003554 + } + ], + "oos_precision":{ + "mean":0.944297, + "std":0.001725 + }, + "oos_f1":{ + "mean":0.720144, + "std":0.002774 + }, + "oos_misroute_rate":{ + "mean":0.418, + "std":0.003606 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.235455, + "std":0.003933 + }, + "llm_cost_usd_per_1k":{ + "mean":0.086702, + "std":0.001444 + }, + "error_recovery_rate":{ + "mean":0.611829, + "std":0.001122 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308 + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662 + }, + "recovered_errors":{ + "mean":792.333333, + "std":14.571662 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.029636, + "std":0.001737 + }, + "oos_auroc":{ + "mean":0.787083, + "std":0.00459 + }, + "oos_auprc":{ + "mean":0.097686, + "std":0.012867 + } + }, + "test":{ + "aurc":{ + "mean":0.075864, + "std":0.002053 + }, + "oos_auroc":{ + "mean":0.817547, + "std":0.003971 + }, + "oos_auprc":{ + "mean":0.43433, + "std":0.005817 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.028559, + "std":0.001878 + }, + "oos_auroc":{ + "mean":0.832308, + "std":0.001097 + }, + "oos_auprc":{ + "mean":0.146969, + "std":0.010386 + } + }, + "test":{ + "aurc":{ + "mean":0.070139, + "std":0.001875 + }, + "oos_auroc":{ + "mean":0.853279, + "std":0.003372 + }, + "oos_auprc":{ + "mean":0.54173, + "std":0.008359 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "margin", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":1.916379, + "std":1.633921 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.485376, + "std":0.051151 + }, + "validation_selective_risk":{ + "mean":0.013937, + "std":0.000276 + }, + "validation_risk_upper95":{ + "mean":0.019871, + "std":4.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.422667, + "std":0.056593 + }, + "selective_risk":{ + "mean":0.037015, + "std":0.00646 + }, + "accuracy_8":{ + "mean":0.853818, + "std":0.005042 + }, + "oos_recall":{ + "mean":0.532667, + "std":0.008737 + }, + "oos_recall_wilson95":[ + { + "mean":0.50168, + "std":0.008737 + }, + { + "mean":0.563404, + "std":0.00867 + } + ], + "oos_precision":{ + "mean":0.828533, + "std":0.021502 + }, + "oos_f1":{ + "mean":0.648279, + "std":0.004192 + }, + "oos_misroute_rate":{ + "mean":0.467333, + "std":0.008737 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.065333, + "std":0.021572 + }, + "llm_call_rate":{ + "mean":0.577333, + "std":0.056593 + }, + "llm_cost_usd_per_1k":{ + "mean":0.213233, + "std":0.020936 + }, + "error_recovery_rate":{ + "mean":0.574389, + "std":0.012351 + }, + "recoverable_caught":{ + "mean":0.938784, + "std":0.018485 + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308 + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662 + }, + "recovered_errors":{ + "mean":744.0, + "std":27.784888 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":4.879453, + "std":1.186614 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.454516, + "std":0.094483 + }, + "validation_selective_risk":{ + "mean":0.013007, + "std":0.001835 + }, + "validation_risk_upper95":{ + "mean":0.019047, + "std":0.001447 + }, + "test":{ + "coverage":{ + "mean":0.399758, + "std":0.088578 + }, + "selective_risk":{ + "mean":0.036066, + "std":0.009448 + }, + "accuracy_8":{ + "mean":0.852545, + "std":0.007002 + }, + "oos_recall":{ + "mean":0.535333, + "std":0.01365 + }, + "oos_recall_wilson95":[ + { + "mean":0.504352, + "std":0.013661 + }, + { + "mean":0.566044, + "std":0.013536 + } + ], + "oos_precision":{ + "mean":0.826033, + "std":0.022724 + }, + "oos_f1":{ + "mean":0.649373, + "std":0.004872 + }, + "oos_misroute_rate":{ + "mean":0.464667, + "std":0.01365 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.063, + "std":0.029206 + }, + "llm_call_rate":{ + "mean":0.600242, + "std":0.088578 + }, + "llm_cost_usd_per_1k":{ + "mean":0.221696, + "std":0.032761 + }, + "error_recovery_rate":{ + "mean":0.578235, + "std":0.017409 + }, + "recoverable_caught":{ + "mean":0.945062, + "std":0.026838 + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308 + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662 + }, + "recovered_errors":{ + "mean":749.0, + "std":32.695565 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.96838, + "std":0.00883 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.473118, + "std":0.030658 + }, + "validation_selective_risk":{ + "mean":0.013858, + "std":0.000141 + }, + "validation_risk_upper95":{ + "mean":0.019856, + "std":6.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.406727, + "std":0.029882 + }, + "selective_risk":{ + "mean":0.033797, + "std":0.001074 + }, + "accuracy_8":{ + "mean":0.852, + "std":0.001898 + }, + "oos_recall":{ + "mean":0.536333, + "std":0.003215 + }, + "oos_recall_wilson95":[ + { + "mean":0.505345, + "std":0.003217 + }, + { + "mean":0.567043, + "std":0.003187 + } + ], + "oos_precision":{ + "mean":0.818871, + "std":0.008259 + }, + "oos_f1":{ + "mean":0.648141, + "std":0.004209 + }, + "oos_misroute_rate":{ + "mean":0.463667, + "std":0.003215 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.055667, + "std":0.005508 + }, + "llm_call_rate":{ + "mean":0.593273, + "std":0.029882 + }, + "llm_cost_usd_per_1k":{ + "mean":0.219132, + "std":0.011055 + }, + "error_recovery_rate":{ + "mean":0.578848, + "std":0.004715 + }, + "recoverable_caught":{ + "mean":0.946087, + "std":0.005995 + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308 + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662 + }, + "recovered_errors":{ + "mean":749.666667, + "std":18.009257 + } + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.664378, + "std":0.015827 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.743118, + "std":0.011379 + }, + "validation_selective_risk":{ + "mean":0.042396, + "std":0.00018 + }, + "validation_risk_upper95":{ + "mean":0.049856, + "std":0.000207 + }, + "test":{ + "coverage":{ + "mean":0.670364, + "std":0.016193 + }, + "selective_risk":{ + "mean":0.080051, + "std":0.003097 + }, + "accuracy_8":{ + "mean":0.857394, + "std":0.001857 + }, + "oos_recall":{ + "mean":0.449333, + "std":0.004509 + }, + "oos_recall_wilson95":[ + { + "mean":0.418756, + "std":0.004464 + }, + { + "mean":0.480299, + "std":0.00452 + } + ], + "oos_precision":{ + "mean":0.875396, + "std":0.010982 + }, + "oos_f1":{ + "mean":0.593832, + "std":0.005212 + }, + "oos_misroute_rate":{ + "mean":0.550667, + "std":0.004509 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.216, + "std":0.010583 + }, + "llm_call_rate":{ + "mean":0.329636, + "std":0.016193 + }, + "llm_cost_usd_per_1k":{ + "mean":0.121637, + "std":0.005992 + }, + "error_recovery_rate":{ + "mean":0.478441, + "std":0.01094 + }, + "recoverable_caught":{ + "mean":0.781969, + "std":0.016713 + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308 + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662 + }, + "recovered_errors":{ + "mean":619.666667, + "std":21.501938 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":1.821899, + "std":0.080179 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.735161, + "std":0.010925 + }, + "validation_selective_risk":{ + "mean":0.042416, + "std":7.3e-05 + }, + "validation_risk_upper95":{ + "mean":0.04992, + "std":7.1e-05 + }, + "test":{ + "coverage":{ + "mean":0.668727, + "std":0.014088 + }, + "selective_risk":{ + "mean":0.092958, + "std":0.002056 + }, + "accuracy_8":{ + "mean":0.854, + "std":0.002622 + }, + "oos_recall":{ + "mean":0.424333, + "std":0.006807 + }, + "oos_recall_wilson95":[ + { + "mean":0.394049, + "std":0.006718 + }, + { + "mean":0.455197, + "std":0.006844 + } + ], + "oos_precision":{ + "mean":0.88069, + "std":0.020183 + }, + "oos_f1":{ + "mean":0.572647, + "std":0.007013 + }, + "oos_misroute_rate":{ + "mean":0.575667, + "std":0.006807 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.262667, + "std":0.008505 + }, + "llm_call_rate":{ + "mean":0.331273, + "std":0.014088 + }, + "llm_cost_usd_per_1k":{ + "mean":0.122276, + "std":0.005199 + }, + "error_recovery_rate":{ + "mean":0.458834, + "std":0.012657 + }, + "recoverable_caught":{ + "mean":0.749916, + "std":0.019377 + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308 + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662 + }, + "recovered_errors":{ + "mean":594.333333, + "std":24.785749 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.664378, + "std":0.015827 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.743118, + "std":0.011379 + }, + "validation_selective_risk":{ + "mean":0.042396, + "std":0.00018 + }, + "validation_risk_upper95":{ + "mean":0.049856, + "std":0.000207 + }, + "test":{ + "coverage":{ + "mean":0.670364, + "std":0.016193 + }, + "selective_risk":{ + "mean":0.080051, + "std":0.003097 + }, + "accuracy_8":{ + "mean":0.857394, + "std":0.001857 + }, + "oos_recall":{ + "mean":0.449333, + "std":0.004509 + }, + "oos_recall_wilson95":[ + { + "mean":0.418756, + "std":0.004464 + }, + { + "mean":0.480299, + "std":0.00452 + } + ], + "oos_precision":{ + "mean":0.875396, + "std":0.010982 + }, + "oos_f1":{ + "mean":0.593832, + "std":0.005212 + }, + "oos_misroute_rate":{ + "mean":0.550667, + "std":0.004509 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.216, + "std":0.010583 + }, + "llm_call_rate":{ + "mean":0.329636, + "std":0.016193 + }, + "llm_cost_usd_per_1k":{ + "mean":0.121637, + "std":0.005992 + }, + "error_recovery_rate":{ + "mean":0.478441, + "std":0.01094 + }, + "recoverable_caught":{ + "mean":0.781969, + "std":0.016713 + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308 + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662 + }, + "recovered_errors":{ + "mean":619.666667, + "std":21.501938 + } + } + } + } + } + } + }, + "summed":{ + "validation_accuracy_8":{ + "mean":0.884194, + "std":0.005597 + }, + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0 + }, + "selective_risk":{ + "mean":0.243333, + "std":0.003772 + }, + "accuracy_8":{ + "mean":0.756667, + "std":0.003772 + }, + "oos_recall":{ + "mean":0.018333, + "std":0.011372 + }, + "oos_recall_wilson95":[ + { + "mean":0.011926, + "std":0.00893 + }, + { + "mean":0.028428, + "std":0.013731 + } + ], + "oos_precision":{ + "mean":0.939394, + "std":0.104973 + }, + "oos_f1":{ + "mean":0.035832, + "std":0.021852 + }, + "oos_misroute_rate":{ + "mean":0.981667, + "std":0.011372 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.981667, + "std":0.011372 + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0 + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0 + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0 + }, + "small_errors":{ + "mean":1338.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":823.666667, + "std":12.342339 + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0 + } + }, + "oracle":{ + "coverage":{ + "mean":0.756667, + "std":0.003772 + }, + "selective_risk":{ + "mean":0.0, + "std":0.0 + }, + "accuracy_8":{ + "mean":0.906424, + "std":0.001546 + }, + "oos_recall":{ + "mean":0.571, + "std":0.001732 + }, + "oos_recall_wilson95":[ + { + "mean":0.54011, + "std":0.001741 + }, + { + "mean":0.601347, + "std":0.00171 + } + ], + "oos_precision":{ + "mean":0.942766, + "std":0.003351 + }, + "oos_f1":{ + "mean":0.711231, + "std":0.002135 + }, + "oos_misroute_rate":{ + "mean":0.429, + "std":0.001732 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0 + }, + "llm_call_rate":{ + "mean":0.243333, + "std":0.003772 + }, + "llm_cost_usd_per_1k":{ + "mean":0.089585, + "std":0.001382 + }, + "error_recovery_rate":{ + "mean":0.615446, + "std":0.000828 + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0 + }, + "small_errors":{ + "mean":1338.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":823.666667, + "std":12.342339 + }, + "recovered_errors":{ + "mean":823.666667, + "std":12.342339 + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.021927, + "std":0.001347 + }, + "oos_auroc":{ + "mean":0.846503, + "std":0.005139 + }, + "oos_auprc":{ + "mean":0.163497, + "std":0.020703 + } + }, + "test":{ + "aurc":{ + "mean":0.06418, + "std":0.002255 + }, + "oos_auroc":{ + "mean":0.855263, + "std":0.0061 + }, + "oos_auprc":{ + "mean":0.510526, + "std":0.013172 + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.023319, + "std":0.001634 + }, + "oos_auroc":{ + "mean":0.855831, + "std":0.002877 + }, + "oos_auprc":{ + "mean":0.204029, + "std":0.010327 + } + }, + "test":{ + "aurc":{ + "mean":0.064652, + "std":0.002142 + }, + "oos_auroc":{ + "mean":0.862896, + "std":0.004916 + }, + "oos_auprc":{ + "mean":0.550143, + "std":0.007737 + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":3.816346, + "std":0.188103 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.641613, + "std":0.013652 + }, + "validation_selective_risk":{ + "mean":0.014578, + "std":0.000199 + }, + "validation_risk_upper95":{ + "mean":0.019704, + "std":0.00017 + }, + "test":{ + "coverage":{ + "mean":0.567091, + "std":0.022218 + }, + "selective_risk":{ + "mean":0.043346, + "std":0.00167 + }, + "accuracy_8":{ + "mean":0.862545, + "std":0.001192 + }, + "oos_recall":{ + "mean":0.513667, + "std":0.004509 + }, + "oos_recall_wilson95":[ + { + "mean":0.482696, + "std":0.004499 + }, + { + "mean":0.544533, + "std":0.004485 + } + ], + "oos_precision":{ + "mean":0.855224, + "std":0.0088 + }, + "oos_f1":{ + "mean":0.641814, + "std":0.004192 + }, + "oos_misroute_rate":{ + "mean":0.486333, + "std":0.004509 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.106333, + "std":0.008327 + }, + "llm_call_rate":{ + "mean":0.432909, + "std":0.022218 + }, + "llm_cost_usd_per_1k":{ + "mean":0.159838, + "std":0.008222 + }, + "error_recovery_rate":{ + "mean":0.56164, + "std":0.00471 + }, + "recoverable_caught":{ + "mean":0.912569, + "std":0.006604 + }, + "small_errors":{ + "mean":1338.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":823.666667, + "std":12.342339 + }, + "recovered_errors":{ + "mean":751.666667, + "std":13.650397 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":3.816346, + "std":0.188103 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.641613, + "std":0.013652 + }, + "validation_selective_risk":{ + "mean":0.014578, + "std":0.000199 + }, + "validation_risk_upper95":{ + "mean":0.019704, + "std":0.00017 + }, + "test":{ + "coverage":{ + "mean":0.567091, + "std":0.022218 + }, + "selective_risk":{ + "mean":0.043346, + "std":0.00167 + }, + "accuracy_8":{ + "mean":0.862545, + "std":0.001192 + }, + "oos_recall":{ + "mean":0.513667, + "std":0.004509 + }, + "oos_recall_wilson95":[ + { + "mean":0.482696, + "std":0.004499 + }, + { + "mean":0.544533, + "std":0.004485 + } + ], + "oos_precision":{ + "mean":0.855224, + "std":0.0088 + }, + "oos_f1":{ + "mean":0.641814, + "std":0.004192 + }, + "oos_misroute_rate":{ + "mean":0.486333, + "std":0.004509 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.106333, + "std":0.008327 + }, + "llm_call_rate":{ + "mean":0.432909, + "std":0.022218 + }, + "llm_cost_usd_per_1k":{ + "mean":0.159838, + "std":0.008222 + }, + "error_recovery_rate":{ + "mean":0.56164, + "std":0.00471 + }, + "recoverable_caught":{ + "mean":0.912569, + "std":0.006604 + }, + "small_errors":{ + "mean":1338.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":823.666667, + "std":12.342339 + }, + "recovered_errors":{ + "mean":751.666667, + "std":13.650397 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.972941, + "std":0.005127 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.584516, + "std":0.0179 + }, + "validation_selective_risk":{ + "mean":0.014347, + "std":0.000171 + }, + "validation_risk_upper95":{ + "mean":0.019721, + "std":0.000167 + }, + "test":{ + "coverage":{ + "mean":0.513636, + "std":0.021434 + }, + "selective_risk":{ + "mean":0.038683, + "std":0.000721 + }, + "accuracy_8":{ + "mean":0.858485, + "std":0.00021 + }, + "oos_recall":{ + "mean":0.525333, + "std":0.004509 + }, + "oos_recall_wilson95":[ + { + "mean":0.494346, + "std":0.004506 + }, + { + "mean":0.556127, + "std":0.004478 + } + ], + "oos_precision":{ + "mean":0.831218, + "std":0.004846 + }, + "oos_f1":{ + "mean":0.643787, + "std":0.004735 + }, + "oos_misroute_rate":{ + "mean":0.474667, + "std":0.004509 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.082333, + "std":0.005508 + }, + "llm_call_rate":{ + "mean":0.486364, + "std":0.021434 + }, + "llm_cost_usd_per_1k":{ + "mean":0.179609, + "std":0.007949 + }, + "error_recovery_rate":{ + "mean":0.571854, + "std":0.004271 + }, + "recoverable_caught":{ + "mean":0.929166, + "std":0.005844 + }, + "small_errors":{ + "mean":1338.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":823.666667, + "std":12.342339 + }, + "recovered_errors":{ + "mean":765.333333, + "std":13.316656 + } + } + } + } + }, + "0.05":{ + "selected_signal":"margin", + "hybrid":{ + "tau":{ + "mean":1.570578, + "std":0.144542 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.827849, + "std":0.012436 + }, + "validation_selective_risk":{ + "mean":0.042731, + "std":0.000171 + }, + "validation_risk_upper95":{ + "mean":0.049794, + "std":0.000129 + }, + "test":{ + "coverage":{ + "mean":0.758242, + "std":0.019092 + }, + "selective_risk":{ + "mean":0.102356, + "std":0.007069 + }, + "accuracy_8":{ + "mean":0.851576, + "std":0.003951 + }, + "oos_recall":{ + "mean":0.393, + "std":0.013077 + }, + "oos_recall_wilson95":[ + { + "mean":0.3632, + "std":0.012856 + }, + { + "mean":0.423619, + "std":0.013197 + } + ], + "oos_precision":{ + "mean":0.895481, + "std":0.015461 + }, + "oos_f1":{ + "mean":0.546131, + "std":0.012128 + }, + "oos_misroute_rate":{ + "mean":0.607, + "std":0.013077 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.335, + "std":0.028054 + }, + "llm_call_rate":{ + "mean":0.241758, + "std":0.019092 + }, + "llm_cost_usd_per_1k":{ + "mean":0.089154, + "std":0.007049 + }, + "error_recovery_rate":{ + "mean":0.440243, + "std":0.019131 + }, + "recoverable_caught":{ + "mean":0.71531, + "std":0.030546 + }, + "small_errors":{ + "mean":1338.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":823.666667, + "std":12.342339 + }, + "recovered_errors":{ + "mean":589.333333, + "std":31.262331 + } + } + }, + "by_signal":{ + "margin":{ + "tau":{ + "mean":1.570578, + "std":0.144542 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.827849, + "std":0.012436 + }, + "validation_selective_risk":{ + "mean":0.042731, + "std":0.000171 + }, + "validation_risk_upper95":{ + "mean":0.049794, + "std":0.000129 + }, + "test":{ + "coverage":{ + "mean":0.758242, + "std":0.019092 + }, + "selective_risk":{ + "mean":0.102356, + "std":0.007069 + }, + "accuracy_8":{ + "mean":0.851576, + "std":0.003951 + }, + "oos_recall":{ + "mean":0.393, + "std":0.013077 + }, + "oos_recall_wilson95":[ + { + "mean":0.3632, + "std":0.012856 + }, + { + "mean":0.423619, + "std":0.013197 + } + ], + "oos_precision":{ + "mean":0.895481, + "std":0.015461 + }, + "oos_f1":{ + "mean":0.546131, + "std":0.012128 + }, + "oos_misroute_rate":{ + "mean":0.607, + "std":0.013077 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.335, + "std":0.028054 + }, + "llm_call_rate":{ + "mean":0.241758, + "std":0.019092 + }, + "llm_cost_usd_per_1k":{ + "mean":0.089154, + "std":0.007049 + }, + "error_recovery_rate":{ + "mean":0.440243, + "std":0.019131 + }, + "recoverable_caught":{ + "mean":0.71531, + "std":0.030546 + }, + "small_errors":{ + "mean":1338.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":823.666667, + "std":12.342339 + }, + "recovered_errors":{ + "mean":589.333333, + "std":31.262331 + } + } + }, + "msp_t":{ + "tau":{ + "mean":0.720314, + "std":0.018943 + }, + "feasible":true, + "validation_coverage":{ + "mean":0.81172, + "std":0.009733 + }, + "validation_selective_risk":{ + "mean":0.042654, + "std":0.000162 + }, + "validation_risk_upper95":{ + "mean":0.049786, + "std":0.000157 + }, + "test":{ + "coverage":{ + "mean":0.733091, + "std":0.014331 + }, + "selective_risk":{ + "mean":0.090394, + "std":0.003892 + }, + "accuracy_8":{ + "mean":0.854788, + "std":0.002123 + }, + "oos_recall":{ + "mean":0.428333, + "std":0.004726 + }, + "oos_recall_wilson95":[ + { + "mean":0.397996, + "std":0.004665 + }, + { + "mean":0.459219, + "std":0.004751 + } + ], + "oos_precision":{ + "mean":0.884615, + "std":0.014132 + }, + "oos_f1":{ + "mean":0.577132, + "std":0.002381 + }, + "oos_misroute_rate":{ + "mean":0.571667, + "std":0.004726 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.274, + "std":0.013 + }, + "llm_call_rate":{ + "mean":0.266909, + "std":0.014331 + }, + "llm_cost_usd_per_1k":{ + "mean":0.098441, + "std":0.005286 + }, + "error_recovery_rate":{ + "mean":0.466883, + "std":0.012918 + }, + "recoverable_caught":{ + "mean":0.758608, + "std":0.02092 + }, + "small_errors":{ + "mean":1338.333333, + "std":20.744477 + }, + "recoverable_errors":{ + "mean":823.666667, + "std":12.342339 + }, + "recovered_errors":{ + "mean":625.0, + "std":26.0 + } + } + } + } + } + } + }, + "final":{ + "small_only":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.235455, + "std":0.003933, + "values":[ + 0.231091, + 0.238727, + 0.236545 + ] + }, + "accuracy_8":{ + "mean":0.764545, + "std":0.003933, + "values":[ + 0.768909, + 0.761273, + 0.763455 + ] + }, + "oos_recall":{ + "mean":0.061667, + "std":0.018583, + "values":[ + 0.083, + 0.049, + 0.053 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.048487, + "std":0.016518, + "values":[ + 0.067455, + 0.037261, + 0.040746 + ] + }, + { + "mean":0.078201, + "std":0.020506, + "values":[ + 0.101737, + 0.064191, + 0.068675 + ] + } + ], + "oos_precision":{ + "mean":0.937901, + "std":0.035233, + "values":[ + 0.976471, + 0.907407, + 0.929825 + ] + }, + "oos_f1":{ + "mean":0.115419, + "std":0.032746, + "values":[ + 0.152995, + 0.092979, + 0.100284 + ] + }, + "oos_misroute_rate":{ + "mean":0.938333, + "std":0.018583, + "values":[ + 0.917, + 0.951, + 0.947 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.938333, + "std":0.018583, + "values":[ + 0.917, + 0.951, + 0.947 + ] + }, + "llm_call_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "error_recovery_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "recoverable_caught":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308, + "values":[ + 1271, + 1313, + 1301 + ] + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662, + "values":[ + 776, + 804, + 797 + ] + }, + "recovered_errors":{ + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.764545, + "std":0.003933, + "values":[ + 0.768909, + 0.761273, + 0.763455 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.908606, + "std":0.00129, + "values":[ + 0.91, + 0.907455, + 0.908364 + ] + }, + "oos_recall":{ + "mean":0.582, + "std":0.003606, + "values":[ + 0.586, + 0.579, + 0.581 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.551173, + "std":0.003629, + "values":[ + 0.555199, + 0.548154, + 0.550166 + ] + }, + { + "mean":0.612199, + "std":0.003554, + "values":[ + 0.616143, + 0.609242, + 0.611214 + ] + } + ], + "oos_precision":{ + "mean":0.944297, + "std":0.001725, + "values":[ + 0.943639, + 0.942997, + 0.946254 + ] + }, + "oos_f1":{ + "mean":0.720144, + "std":0.002774, + "values":[ + 0.72301, + 0.717472, + 0.71995 + ] + }, + "oos_misroute_rate":{ + "mean":0.418, + "std":0.003606, + "values":[ + 0.414, + 0.421, + 0.419 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.235455, + "std":0.003933, + "values":[ + 0.231091, + 0.238727, + 0.236545 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.086702, + "std":0.001444, + "values":[ + 0.085101, + 0.087904, + 0.087102 + ] + }, + "error_recovery_rate":{ + "mean":0.611829, + "std":0.001122, + "values":[ + 0.610543, + 0.612338, + 0.612606 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308, + "values":[ + 1271, + 1313, + 1301 + ] + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662, + "values":[ + 776, + 804, + 797 + ] + }, + "recovered_errors":{ + "mean":792.333333, + "std":14.571662, + "values":[ + 776, + 804, + 797 + ] + } + }, + "signals":{ + "margin":{ + "validation":{ + "aurc":{ + "mean":0.029636, + "std":0.001737, + "values":[ + 0.028517, + 0.028753, + 0.031637 + ] + }, + "oos_auroc":{ + "mean":0.787083, + "std":0.00459, + "values":[ + 0.782403, + 0.787268, + 0.791578 + ] + }, + "oos_auprc":{ + "mean":0.097686, + "std":0.012867, + "values":[ + 0.083583, + 0.100692, + 0.108784 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.075864, + "std":0.002053, + "values":[ + 0.073496, + 0.076966, + 0.077132 + ] + }, + "oos_auroc":{ + "mean":0.817547, + "std":0.003971, + "values":[ + 0.821946, + 0.816465, + 0.814229 + ] + }, + "oos_auprc":{ + "mean":0.43433, + "std":0.005817, + "values":[ + 0.430422, + 0.441016, + 0.431554 + ] + } + } + }, + "msp_t":{ + "validation":{ + "aurc":{ + "mean":0.028559, + "std":0.001878, + "values":[ + 0.027529, + 0.027421, + 0.030726 + ] + }, + "oos_auroc":{ + "mean":0.832308, + "std":0.001097, + "values":[ + 0.832973, + 0.831042, + 0.832908 + ] + }, + "oos_auprc":{ + "mean":0.146969, + "std":0.010386, + "values":[ + 0.136715, + 0.146712, + 0.157481 + ] + } + }, + "test":{ + "aurc":{ + "mean":0.070139, + "std":0.001875, + "values":[ + 0.06798, + 0.071347, + 0.071091 + ] + }, + "oos_auroc":{ + "mean":0.853279, + "std":0.003372, + "values":[ + 0.857123, + 0.85082, + 0.851895 + ] + }, + "oos_auprc":{ + "mean":0.54173, + "std":0.008359, + "values":[ + 0.537433, + 0.536395, + 0.551363 + ] + } + } + } + }, + "fallback":{ + "0.02":{ + "selected_signal":[ + "margin", + "msp_t", + "msp_t" + ], + "hybrid":{ + "tau":{ + "mean":1.916379, + "std":1.633921, + "values":[ + 3.803064, + 0.96944, + 0.976632 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.485376, + "std":0.051151, + "values":[ + 0.543226, + 0.466774, + 0.446129 + ] + }, + "validation_selective_risk":{ + "mean":0.013937, + "std":0.000276, + "values":[ + 0.014252, + 0.013822, + 0.013738 + ] + }, + "validation_risk_upper95":{ + "mean":0.019871, + "std":4.1e-05, + "values":[ + 0.019842, + 0.019854, + 0.019918 + ] + }, + "test":{ + "coverage":{ + "mean":0.422667, + "std":0.056593, + "values":[ + 0.486909, + 0.400909, + 0.380182 + ] + }, + "selective_risk":{ + "mean":0.037015, + "std":0.00646, + "values":[ + 0.044436, + 0.032653, + 0.033955 + ] + }, + "accuracy_8":{ + "mean":0.853818, + "std":0.005042, + "values":[ + 0.859636, + 0.850727, + 0.851091 + ] + }, + "oos_recall":{ + "mean":0.532667, + "std":0.008737, + "values":[ + 0.523, + 0.535, + 0.54 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.50168, + "std":0.008737, + "values":[ + 0.492014, + 0.504011, + 0.509015 + ] + }, + { + "mean":0.563404, + "std":0.00867, + "values":[ + 0.55381, + 0.565721, + 0.570679 + ] + } + ], + "oos_precision":{ + "mean":0.828533, + "std":0.021502, + "values":[ + 0.851792, + 0.80938, + 0.824427 + ] + }, + "oos_f1":{ + "mean":0.648279, + "std":0.004192, + "values":[ + 0.648079, + 0.64419, + 0.652568 + ] + }, + "oos_misroute_rate":{ + "mean":0.467333, + "std":0.008737, + "values":[ + 0.477, + 0.465, + 0.46 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.065333, + "std":0.021572, + "values":[ + 0.09, + 0.056, + 0.05 + ] + }, + "llm_call_rate":{ + "mean":0.577333, + "std":0.056593, + "values":[ + 0.513091, + 0.599091, + 0.619818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.213233, + "std":0.020936, + "values":[ + 0.189471, + 0.221261, + 0.228967 + ] + }, + "error_recovery_rate":{ + "mean":0.574389, + "std":0.012351, + "values":[ + 0.560189, + 0.58035, + 0.582629 + ] + }, + "recoverable_caught":{ + "mean":0.938784, + "std":0.018485, + "values":[ + 0.917526, + 0.947761, + 0.951066 + ] + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308, + "values":[ + 1271, + 1313, + 1301 + ] + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662, + "values":[ + 776, + 804, + 797 + ] + }, + "recovered_errors":{ + "mean":744.0, + "std":27.784888, + "values":[ + 712, + 762, + 758 + ] + } + } + } + }, + "0.05":{ + "selected_signal":"msp_t", + "hybrid":{ + "tau":{ + "mean":0.664378, + "std":0.015827, + "values":[ + 0.654798, + 0.65569, + 0.682647 + ] + }, + "feasible":true, + "validation_coverage":{ + "mean":0.743118, + "std":0.011379, + "values":[ + 0.749032, + 0.750323, + 0.73 + ] + }, + "validation_selective_risk":{ + "mean":0.042396, + "std":0.00018, + "values":[ + 0.042205, + 0.042562, + 0.042422 + ] + }, + "validation_risk_upper95":{ + "mean":0.049856, + "std":0.000207, + "values":[ + 0.049617, + 0.049995, + 0.049954 + ] + }, + "test":{ + "coverage":{ + "mean":0.670364, + "std":0.016193, + "values":[ + 0.684182, + 0.674364, + 0.652545 + ] + }, + "selective_risk":{ + "mean":0.080051, + "std":0.003097, + "values":[ + 0.082647, + 0.080884, + 0.076623 + ] + }, + "accuracy_8":{ + "mean":0.857394, + "std":0.001857, + "values":[ + 0.858727, + 0.855273, + 0.858182 + ] + }, + "oos_recall":{ + "mean":0.449333, + "std":0.004509, + "values":[ + 0.449, + 0.445, + 0.454 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.418756, + "std":0.004464, + "values":[ + 0.418425, + 0.414466, + 0.423376 + ] + }, + { + "mean":0.480299, + "std":0.00452, + "values":[ + 0.479965, + 0.475954, + 0.484976 + ] + } + ], + "oos_precision":{ + "mean":0.875396, + "std":0.010982, + "values":[ + 0.887352, + 0.865759, + 0.873077 + ] + }, + "oos_f1":{ + "mean":0.593832, + "std":0.005212, + "values":[ + 0.596282, + 0.587847, + 0.597368 + ] + }, + "oos_misroute_rate":{ + "mean":0.550667, + "std":0.004509, + "values":[ + 0.551, + 0.555, + 0.546 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.216, + "std":0.010583, + "values":[ + 0.224, + 0.22, + 0.204 + ] + }, + "llm_call_rate":{ + "mean":0.329636, + "std":0.016193, + "values":[ + 0.315818, + 0.325636, + 0.347455 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.121637, + "std":0.005992, + "values":[ + 0.116526, + 0.120153, + 0.128231 + ] + }, + "error_recovery_rate":{ + "mean":0.478441, + "std":0.01094, + "values":[ + 0.468922, + 0.476009, + 0.490392 + ] + }, + "recoverable_caught":{ + "mean":0.781969, + "std":0.016713, + "values":[ + 0.768041, + 0.777363, + 0.800502 + ] + }, + "small_errors":{ + "mean":1295.0, + "std":21.633308, + "values":[ + 1271, + 1313, + 1301 + ] + }, + "recoverable_errors":{ + "mean":792.333333, + "std":14.571662, + "values":[ + 776, + 804, + 797 + ] + }, + "recovered_errors":{ + "mean":619.666667, + "std":21.501938, + "values":[ + 596, + 625, + 638 + ] + } + } + } + } + } + } + } + } + } +} diff --git a/src/tinyrouter/analysis.py b/src/tinyrouter/analysis.py new file mode 100644 index 0000000..88c2a5e --- /dev/null +++ b/src/tinyrouter/analysis.py @@ -0,0 +1,347 @@ +"""RQ2 to RQ4 for one archived run: 8-way routing, OOS, uncertainty, LLM fallback. + +Everything is computed from one logits archive (AC3) and the stored Haiku +predictions; nothing is trained and no API is called. Every choice is +made on validation and applied unchanged to test (AC4), and each choosing +function refuses any other split with ``LeakageError``: + +- the temperature (``calibrate.fit_temperature``); +- the 8-way aggregation (``select_aggregation``): higher validation 8-way + accuracy, ties go to argmax, the PLAN's default; +- the deferral threshold per signal and target risk + (``selective.select_threshold``); +- which signal the hybrid router uses (``select_signal``): largest + validation coverage at the target, then lower validation AURC, then the + order of ``SIGNALS``. + +Aggregations. ``argmax``: the argmax intent's agent; the signals are read +off the 151-way distribution. ``summed``: the argmax of the 8 agent +probabilities summed over each agent's intents (temperature-scaled, since +that argmax depends on T); the signals are read off the 8-way +distribution. Signals (higher = more confident): ``msp`` max probability +and ``entropy`` minus the entropy, both at T = 1; ``msp_t`` max +probability at the fitted T; ``margin`` the gap between the two largest +log-probabilities at the fitted T. For argmax that gap is +``(z1 - z2) / T``, which ranks rows exactly as the raw score margin does, +so it is the "score margin" PLAN section 4.1 asks for on TF-IDF. + +Which signals a run gets (PLAN section 4.1): encoders all four; TF-IDF +only ``msp_t`` and ``margin`` (its 100x cosine scale distorts the T = 1 +signals and the raw ECE); majority none, and no temperature (every row +has the same scores, so every signal is a constant). +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass + +import numpy as np + +from tinyrouter.calibrate import SplitLogits, TemperatureBoundWarning, fit_temperature +from tinyrouter.labels import AGENTS, OOS, LabelSpace +from tinyrouter.metrics import expected_calibration_error, log_softmax, softmax, wilson_interval +from tinyrouter.selective import ( + COVERAGE_GRID, + ThresholdChoice, + aurc, + auroc, + average_precision, + coverage_thresholds, + detection_counts, + high_confidence_oos_misroute, + require_validation, + risk_at_coverages, + select_threshold, +) + +AGGREGATIONS = ("argmax", "summed") +SIGNALS = ("msp", "entropy", "margin", "msp_t") +SIGNALS_BY_KIND: dict[str, tuple[str, ...]] = { + "encoder": SIGNALS, + "tfidf": ("margin", "msp_t"), + "majority": (), +} +TARGET_RISKS = (0.02, 0.05) +OOS_AGENT = AGENTS.index(OOS) + + +@dataclass(frozen=True) +class Routed: + """One split routed in the 8-way space by one aggregation.""" + + split: str + pred: np.ndarray + gold: np.ndarray + signals: dict[str, np.ndarray] + + @property + def error(self) -> np.ndarray: + return self.pred != self.gold + + @property + def gold_oos(self) -> np.ndarray: + return self.gold == OOS_AGENT + + @property + def pred_oos(self) -> np.ndarray: + return self.pred == OOS_AGENT + + +def top_two(values: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + part = np.partition(values, -2, axis=1) + return part[:, -1], part[:, -2] + + +def signals_of(raw: np.ndarray, cal: np.ndarray, log_cal: np.ndarray) -> dict[str, np.ndarray]: + safe = np.clip(raw, 1e-300, None) + first, second = top_two(log_cal) + return { + "msp": raw.max(axis=1), + "entropy": np.sum(raw * np.log(safe), axis=1), + "margin": first - second, + "msp_t": cal.max(axis=1), + } + + +def route( + split: SplitLogits, temperature: float, aggregation: str, space: LabelSpace, kind: str +) -> Routed: + """8-way predictions and the ``kind``'s signals for one split.""" + z = np.asarray(split.logits, dtype=np.float64) + raw, cal = softmax(z), softmax(z / temperature) + if aggregation == "argmax": + pred = space.agents_of(z.argmax(axis=1)) + signals = signals_of(raw, cal, log_softmax(z / temperature)) + elif aggregation == "summed": + raw8, cal8 = space.aggregate_probs(raw), space.aggregate_probs(cal) + pred = cal8.argmax(axis=1) + signals = signals_of(raw8, cal8, np.log(np.clip(cal8, 1e-300, None))) + else: + raise ValueError(f"unknown aggregation {aggregation!r}; expected one of {AGGREGATIONS}") + kept = {name: signals[name] for name in SIGNALS_BY_KIND[kind]} + return Routed(split.split, pred, space.agents_of(split.labels), kept) + + +def fit_run_temperature(val: SplitLogits, kind: str) -> tuple[float, dict[str, object]]: + """T on validation (none for majority), and a note of how it was obtained.""" + if kind == "majority": + return 1.0, {"temperature": None, "note": "not fitted: every row has the same scores"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", TemperatureBoundWarning) + temperature = fit_temperature(val) + hit_bound = any(issubclass(w.category, TemperatureBoundWarning) for w in caught) + return temperature, {"temperature": temperature, "hit_search_bound": hit_bound} + + +def select_aggregation(candidates: dict[str, Routed]) -> tuple[str, dict[str, float]]: + """The validation-best aggregation (ties: argmax) and each one's validation accuracy.""" + for routed in candidates.values(): + require_validation(routed.split, "the 8-way aggregation") + accuracy = {name: float(np.mean(~r.error)) for name, r in candidates.items()} + best = max(AGGREGATIONS, key=lambda name: (accuracy[name], name == "argmax")) + return best, accuracy + + +def router_metrics( + small: Routed, llm_pred: np.ndarray, llm_cost: np.ndarray, deferred: np.ndarray +) -> dict[str, object]: + """Small model where it keeps the query, the LLM where it defers; all rates over this split.""" + final = np.where(deferred, llm_pred, small.pred) + accepted = ~deferred + recoverable = small.error & (llm_pred == small.gold) + recovered = recoverable & deferred + counts = detection_counts(final == OOS_AGENT, small.gold_oos) + errors, n_recoverable = int(small.error.sum()), int(recoverable.sum()) + return { + "coverage": float(accepted.mean()), + "selective_risk": float(small.error[accepted].mean()) if accepted.any() else None, + "accuracy_8": float(np.mean(final == small.gold)), + "oos_recall": counts["recall"], + "oos_recall_wilson95": list(wilson_interval(counts["tp"], counts["gold_oos"])), + "oos_precision": counts["precision"], + "oos_f1": counts["f1"], + "oos_misroute_rate": 1.0 - counts["recall"], + "high_conf_oos_misroute_rate": high_confidence_oos_misroute( + small.pred_oos, small.gold_oos, accepted + ), + "llm_call_rate": float(deferred.mean()), + "llm_cost_usd_per_1k": float(1000 * llm_cost[deferred].sum() / deferred.size), + "error_recovery_rate": float(recovered.sum() / errors) if errors else None, + "recoverable_caught": float(recovered.sum() / n_recoverable) if n_recoverable else None, + "small_errors": errors, + "recoverable_errors": n_recoverable, + "recovered_errors": int(recovered.sum()), + } + + +def signal_quality(routed: Routed, signal: str) -> dict[str, object]: + """Threshold-free quality of one signal on one split (RQ2 detection, RQ3 selection).""" + confidence = routed.signals[signal] + return { + "aurc": aurc(confidence, routed.error), + "oos_auroc": auroc(-confidence, routed.gold_oos), + "oos_auprc": average_precision(-confidence, routed.gold_oos), + } + + +def select_signal(val: Routed, choices: dict[str, ThresholdChoice]) -> str | None: + """Signal with the largest validation coverage at the target (see module docstring).""" + require_validation(val.split, "the hybrid's signal") + if not choices: + return None + order = {name: i for i, name in enumerate(SIGNALS)} + + def key(name: str) -> tuple[float, float, int]: + return (-choices[name].coverage, aurc(val.signals[name], val.error), order[name]) + + return min(choices, key=key) + + +def choice_record(choice: ThresholdChoice) -> dict[str, object]: + return { + "tau": choice.tau, + "feasible": choice.feasible, + "validation_coverage": choice.coverage, + "validation_selective_risk": choice.risk, + "validation_risk_upper95": choice.risk_upper, + } + + +def fallback( + val: Routed, test: Routed, llm_pred: np.ndarray, llm_cost: np.ndarray +) -> dict[str, object]: + """RQ4: per target risk, every signal's validation-chosen threshold applied to test.""" + out: dict[str, object] = {} + for target in TARGET_RISKS: + choices = { + s: select_threshold(val.split, val.signals[s], val.error, target) for s in val.signals + } + per_signal = {} + for signal, choice in choices.items(): + confidence = test.signals[signal] + deferred = np.ones(confidence.size, dtype=bool) + if choice.tau is not None: + deferred = confidence < choice.tau + test_metrics = router_metrics(test, llm_pred, llm_cost, deferred) + per_signal[signal] = {**choice_record(choice), "test": test_metrics} + chosen = select_signal(val, choices) + out[f"{target:.2f}"] = { + "selected_signal": chosen, + "hybrid": per_signal[chosen] if chosen else None, + "by_signal": per_signal, + } + return out + + +def operating_curve( + val: Routed, test: Routed, llm_pred: np.ndarray, llm_cost: np.ndarray, signal: str +) -> dict[str, list[float | None]]: + """For each validation coverage in the grid, its threshold and what it does on test.""" + taus = coverage_thresholds(val.split, val.signals[signal]) + keys = ( + "coverage", + "selective_risk", + "accuracy_8", + "high_conf_oos_misroute_rate", + "oos_misroute_rate", + "llm_call_rate", + ) + curve: dict[str, list[float | None]] = {"tau": taus, **{k: [] for k in keys}} + for tau in taus: + metrics = router_metrics(test, llm_pred, llm_cost, test.signals[signal] < tau) + for k in keys: + curve[k].append(metrics[k]) + return curve + + +def aggregation_block( + val: Routed, test: Routed, llm_pred: np.ndarray, llm_cost: np.ndarray +) -> tuple[dict[str, object], dict[str, object]]: + """(scalar results, curves) for one aggregation.""" + never = np.zeros(test.pred.size, dtype=bool) + scalars = { + "validation_accuracy_8": float(np.mean(~val.error)), + "small_only": router_metrics(test, llm_pred, llm_cost, never), + "oracle": router_metrics(test, llm_pred, llm_cost, test.error), + "signals": { + s: {"validation": signal_quality(val, s), "test": signal_quality(test, s)} + for s in test.signals + }, + "fallback": fallback(val, test, llm_pred, llm_cost), + } + curves = { + s: { + "risk_coverage": risk_at_coverages(test.signals[s], test.error), + "operating": operating_curve(val, test, llm_pred, llm_cost, s), + } + for s in test.signals + } + return scalars, curves + + +def calibration_block(val: SplitLogits, test: SplitLogits, t: float, space: LabelSpace, kind: str): + """Test ECE before and after temperature scaling, 151-way and 8-way summed.""" + if kind == "majority": + return None + z = np.asarray(test.logits, dtype=np.float64) + gold8 = space.agents_of(test.labels) + out = {} + for name, probs in (("uncalibrated", softmax(z)), ("calibrated", softmax(z / t))): + if kind == "tfidf" and name == "uncalibrated": + out[name] = None + continue + out[name] = { + "ece_151": expected_calibration_error(probs, test.labels), + "ece_8_summed": expected_calibration_error(space.aggregate_probs(probs), gold8), + } + return out + + +def analyze_run( + val: SplitLogits, + test: SplitLogits, + kind: str, + space: LabelSpace, + llm_pred: np.ndarray, + llm_cost: np.ndarray, +) -> tuple[dict[str, object], dict[str, object]]: + """(scalars, curves) for one archive; ``llm_*`` are the test-split Haiku arrays.""" + temperature, t_record = fit_run_temperature(val, kind) + routed = { + a: (route(val, temperature, a, space, kind), route(test, temperature, a, space, kind)) + for a in AGGREGATIONS + } + selected, val_accuracy = select_aggregation({a: v for a, (v, _) in routed.items()}) + scalars: dict[str, object] = { + "calibration": { + **t_record, + "test_ece": calibration_block(val, test, temperature, space, kind), + }, + "selected_aggregation": selected, + "aggregation_validation_accuracy_8": val_accuracy, + "signals_reported": list(SIGNALS_BY_KIND[kind]), + } + curves: dict[str, object] = {} + for a, (v, t) in routed.items(): + scalars[a], curves[a] = aggregation_block(v, t, llm_pred, llm_cost) + scalars["final"] = final_view(scalars[selected]) + return scalars, curves + + +def final_view(block: dict[str, object]) -> dict[str, object]: + """The selected aggregation's router numbers, without the per-signal detail.""" + fallback_block = block["fallback"] + assert isinstance(fallback_block, dict) + return { + "small_only": block["small_only"], + "oracle": block["oracle"], + "signals": block["signals"], + "fallback": { + target: {"selected_signal": v["selected_signal"], "hybrid": v["hybrid"]} + for target, v in fallback_block.items() + }, + } + + +__all__ = ["COVERAGE_GRID", "analyze_run", "route", "router_metrics", "select_aggregation"] diff --git a/src/tinyrouter/analysis_run.py b/src/tinyrouter/analysis_run.py new file mode 100644 index 0000000..4ff45b2 --- /dev/null +++ b/src/tinyrouter/analysis_run.py @@ -0,0 +1,370 @@ +"""``make analysis``: RQ2 to RQ4 over every archived run, written to ``results/analysis/``. + +Inputs (all checked before anything is computed; a missing or changed +input stops the run): + +- the four indexes under ``results/curves/`` (BERT and ModernBERT curves, + the OOS ablation, the baselines), each re-verified with + ``completeness.verify_index``: exactly the expected points, each once, + every archive's SHA-256 equal on disk, in the manifest and in the index; +- 75 distinct archives in total (36 encoder points, of which BERT k=100 + are the three AC2 runs, 3 ablation points, 36 baseline points), a + literal here; +- ``results/llm/haiku-8way.jsonl``, re-verified with ``llm_run.verify`` + (8,600 rows, identity and SHA-256), a literal here. + +Cross-checks: within each split every archive holds the same gold labels, +and Haiku's ``gold_intent`` equals them row by row. + +Outputs (``results/analysis/``, committed): ``summary.json`` (every +scalar, grouped by model and k, mean and sample std over seeds 42/43/44 +with the per-seed values), ``curves.json`` (risk-coverage and operating +curves, mean and std), ``haiku.json`` (the LLM alone, both parsers). +Each is written to a temporary file, read back and checked (the expected +groups, three seeds each, both Haiku splits) before it is moved into +place; only then is ``completed analysis (...)`` printed. There is no +randomness anywhere: no bootstrap, no sampling, no random tie-breaking. +""" + +from __future__ import annotations + +import argparse +import json +import os +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from tinyrouter.analysis import ( + AGGREGATIONS, + SIGNALS, + SIGNALS_BY_KIND, + TARGET_RISKS, + analyze_run, +) +from tinyrouter.archive import MANIFEST_NAME, load_logits +from tinyrouter.completeness import ( + ABLATION_POINTS, + BASELINE_KEY, + BASELINE_POINTS, + CURVE_POINTS, + ENCODER_KEY, + IncompleteError, + verify_index, +) +from tinyrouter.data import sha256_of +from tinyrouter.haiku import HaikuSplit, check_gold, read_records, split_arrays, summarize_split +from tinyrouter.labels import load_label_space +from tinyrouter.llm_run import EXPECTED_ROWS, full_target +from tinyrouter.llm_run import verify as verify_llm +from tinyrouter.selective import COVERAGE_GRID, ONE_SIDED_95_Z + +EXPECTED_ARCHIVES = 75 +EXPECTED_LLM_ROWS = 8600 +EXPECTED_SEEDS = (42, 43, 44) +FORMAT_VERSION = 1 +BASELINE_KINDS = {"majority": "majority", "tfidf-centroid": "tfidf"} + +Log = Callable[[str], None] + + +@dataclass(frozen=True) +class Point: + group: str + model: str + k: int + oos_train: int | None + seed: int + kind: str + run_name: str + logits_file: str + logits_sha256: str + + +@dataclass(frozen=True) +class IndexSpec: + file: str + key: tuple[str, ...] + expected: frozenset + + +INDEXES = ( + IndexSpec("bert.json", ENCODER_KEY, CURVE_POINTS), + IndexSpec("modernbert.json", ENCODER_KEY, CURVE_POINTS), + IndexSpec("oos-ablation.json", ENCODER_KEY, ABLATION_POINTS), + IndexSpec("baselines.json", BASELINE_KEY, BASELINE_POINTS), +) + + +def point_from(index_file: str, entry: dict) -> Point: + if index_file == "baselines.json": + model, kind, oos_train = entry["baseline"], BASELINE_KINDS[entry["baseline"]], None + elif index_file == "oos-ablation.json": + model, kind, oos_train = "modernbert-oos0", "encoder", entry["oos_train"] + else: + model, kind, oos_train = index_file.removesuffix(".json"), "encoder", None + return Point( + group=f"{model}/k{entry['k']}", + model=model, + k=entry["k"], + oos_train=oos_train, + seed=entry["seed"], + kind=kind, + run_name=entry["run_name"], + logits_file=entry["logits_file"], + logits_sha256=entry["logits_sha256"], + ) + + +def collect_points(results_root: Path) -> list[Point]: + """Every point of the four indexes, after each index passes its completion check.""" + points = [] + for spec in INDEXES: + path = results_root / "curves" / spec.file + verify_index(path, spec.key, spec.expected, results_root, spec.file) + entries = json.loads(path.read_text(encoding="utf-8"))["points"] + points.extend(point_from(spec.file, e) for e in entries) + files = {p.logits_file for p in points} + if len(points) != EXPECTED_ARCHIVES or len(files) != EXPECTED_ARCHIVES: + raise IncompleteError( + f"expected {EXPECTED_ARCHIVES} points on {EXPECTED_ARCHIVES} distinct archives, got " + f"{len(points)} points on {len(files)} archives" + ) + return points + + +def load_haiku(results_root: Path) -> dict[str, HaikuSplit]: + target = full_target(results_root) + count = verify_llm(target) + if count != EXPECTED_LLM_ROWS: + raise IncompleteError(f"{target.predictions}: {count} rows, expected {EXPECTED_LLM_ROWS}") + records = read_records(target.predictions) + return {s: split_arrays(records, s, n) for s, n in EXPECTED_ROWS.items()} + + +def combine(values: list, keep_values: bool = True) -> object: + """Seeds merged leaf by leaf: numbers -> mean, sample std (ddof=1) and the values.""" + if all(v is None for v in values): + return None + if all(isinstance(v, dict) for v in values): + keys = set(values[0]) + if any(set(v) != keys for v in values): + raise ValueError("seed results have different keys") + return {k: combine([v[k] for v in values], keep_values) for k in values[0]} + numbers = [v for v in values if isinstance(v, int | float) and not isinstance(v, bool)] + if numbers and len(numbers) + values.count(None) == len(values): + arr = np.array(numbers, dtype=np.float64) + out: dict[str, object] = { + "mean": float(arr.mean()), + "std": float(arr.std(ddof=1)) if arr.size > 1 else 0.0, + } + if len(numbers) < len(values): + out["n"] = len(numbers) + if keep_values: + out["values"] = values + return out + if all(isinstance(v, list) for v in values) and len({len(v) for v in values}) == 1: + return [combine([v[i] for v in values], keep_values) for i in range(len(values[0]))] + return values[0] if all(v == values[0] for v in values) else values + + +def rounded(value: object) -> object: + if isinstance(value, float): + return round(value, 6) + if isinstance(value, dict): + return {k: rounded(v) for k, v in value.items()} + if isinstance(value, list): + return [rounded(v) for v in value] + return value + + +def llm_only(haiku: HaikuSplit) -> dict[str, object]: + summary = summarize_split(haiku)["new_parser"] + assert isinstance(summary, dict) + return { + **summary, + "coverage": 0.0, + "llm_call_rate": 1.0, + "llm_cost_usd_per_1k": float(1000 * haiku.cost_usd.mean()), + "high_conf_oos_misroute_rate": None, + "note": "no confidence score, so no high-confidence misroute; see oos_misroute_rate", + } + + +def analyze_points( + points: list[Point], results_root: Path, haiku: dict[str, HaikuSplit], log: Log +) -> tuple[dict[str, list], dict[str, list]]: + """Per-group lists of per-seed (scalars, curves), with the label cross-checks.""" + space = load_label_space() + reference: dict[str, np.ndarray] = {} + scalars: dict[str, list] = {} + curves: dict[str, list] = {} + test_llm = haiku["test"] + for point in sorted(points, key=lambda p: (p.group, p.seed)): + archive = load_logits(results_root / "logits" / point.logits_file) + for name, split in archive.splits.items(): + ref = reference.setdefault(name, split.labels) + if not np.array_equal(ref, split.labels): + raise IncompleteError(f"{point.logits_file}: {name} labels differ from the others") + check_gold(haiku[name], split.labels, point.logits_file) + s, c = analyze_run( + archive.validation, archive.test, point.kind, space, test_llm.pred, test_llm.cost_usd + ) + scalars.setdefault(point.group, []).append((point, s)) + curves.setdefault(point.group, []).append((point, c)) + log(f"analyzed {point.run_name}") + return scalars, curves + + +def group_body(entries: list, keep_values: bool) -> dict[str, object]: + """One group's seeds merged. Per-seed values are kept for the final router only. + + The ``argmax`` and ``summed`` blocks carry every signal and target for + both aggregations; with per-seed values they would triple the file, + so they keep mean and std. The ``final`` block repeats the selected + aggregation's router numbers with the per-seed values. + """ + points = [p for p, _ in entries] + if tuple(p.seed for p in points) != EXPECTED_SEEDS: + raise IncompleteError(f"{points[0].group}: seeds {[p.seed for p in points]}") + first = points[0] + return { + "model": first.model, + "k": first.k, + "oos_train": first.oos_train, + "kind": first.kind, + "seeds": list(EXPECTED_SEEDS), + "run_names": [p.run_name for p in points], + "logits_sha256": [p.logits_sha256 for p in points], + "result": { + key: combine([r[key] for _, r in entries], keep_values and key not in AGGREGATIONS) + for key in entries[0][1] + }, + } + + +def protocol_record() -> dict[str, object]: + return { + "aggregations": list(AGGREGATIONS), + "aggregation_choice": "higher validation 8-way accuracy; ties go to argmax", + "signals": list(SIGNALS), + "signals_by_kind": {k: list(v) for k, v in SIGNALS_BY_KIND.items()}, + "target_risks": list(TARGET_RISKS), + "threshold_choice": ( + "largest validation coverage whose one-sided 95% Wilson upper bound on selective " + "risk (8-way routing error among kept rows) is <= the target; no such threshold " + "means every query goes to the LLM" + ), + "one_sided_z": ONE_SIDED_95_Z, + "signal_choice": "largest validation coverage at the target, then lower validation AURC", + "oracle": "defers exactly the queries the small model gets wrong (8-way)", + "error_recovery_rate": "small-model errors deferred and fixed by Haiku / small errors", + "recoverable_caught": "errors deferred and fixed by Haiku / errors Haiku would fix", + "high_conf_oos_misroute_rate": ( + "gold OOS kept by the small model (confidence >= tau) and routed to an in-scope " + "agent / gold OOS" + ), + "coverage_grid": list(COVERAGE_GRID), + "operating_curve_tau": "the validation score at each coverage of the grid", + "std": "sample standard deviation (ddof=1) over seeds 42, 43, 44; no other randomness", + "oos_precision_when_none_predicted": 0.0, + } + + +def write_checked( + path: Path, body: dict, check: Callable[[dict], None], indent: int | None = 1 +) -> None: + """Write via a temporary file that is read back and checked before it replaces ``path``. + + ``indent=None`` writes one line (the curves file: long numeric series). + """ + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + text = json.dumps(rounded(body), indent=indent, separators=(",", ":"), allow_nan=False) + tmp.write_text(text + "\n", encoding="utf-8") + try: + check(json.loads(tmp.read_text(encoding="utf-8"))) + except Exception: + tmp.unlink(missing_ok=True) + raise + os.replace(tmp, path) + + +def check_groups(expected: set[str]) -> Callable[[dict], None]: + def check(body: dict) -> None: + groups = body["groups"] + if set(groups) != expected: + raise IncompleteError(f"written groups {sorted(groups)} != expected {sorted(expected)}") + for name, group in groups.items(): + if group["seeds"] != list(EXPECTED_SEEDS) or len(group["run_names"]) != 3: + raise IncompleteError(f"{name}: written with seeds {group['seeds']}") + + return check + + +def check_haiku(body: dict) -> None: + for split, rows in EXPECTED_ROWS.items(): + if body["splits"][split]["new_parser"]["n"] != rows: + raise IncompleteError(f"haiku.json: {split} does not hold {rows} rows") + + +def input_record(results_root: Path, haiku_sha: str) -> dict[str, object]: + indexes = {s.file: sha256_of(results_root / "curves" / s.file) for s in INDEXES} + return { + "archives": EXPECTED_ARCHIVES, + "logits_manifest_sha256": sha256_of(results_root / MANIFEST_NAME), + "curve_index_sha256": indexes, + "haiku_predictions_sha256": haiku_sha, + "llm_rows": EXPECTED_LLM_ROWS, + } + + +def run(results_root: Path, log: Log = print) -> dict[str, int]: + points = collect_points(results_root) + haiku = load_haiku(results_root) + scalars, curves = analyze_points(points, results_root, haiku, log) + haiku_sha = sha256_of(full_target(results_root).predictions) + inputs = input_record(results_root, haiku_sha) + expected = {p.group for p in points} + out = results_root / "analysis" + summary = { + "format_version": FORMAT_VERSION, + "inputs": inputs, + "protocol": protocol_record(), + "llm_only_test": llm_only(haiku["test"]), + "groups": {g: group_body(e, keep_values=True) for g, e in sorted(scalars.items())}, + } + write_checked(out / "summary.json", summary, check_groups(expected)) + curve_body = { + "format_version": FORMAT_VERSION, + "inputs": inputs, + "coverage_grid": list(COVERAGE_GRID), + "groups": {g: group_body(e, keep_values=False) for g, e in sorted(curves.items())}, + } + write_checked(out / "curves.json", curve_body, check_groups(expected), indent=None) + haiku_body = { + "format_version": FORMAT_VERSION, + "haiku_predictions_sha256": haiku_sha, + "splits": {s: summarize_split(h) for s, h in haiku.items()}, + } + write_checked(out / "haiku.json", haiku_body, check_haiku) + return {"archives": len(points), "llm_rows": EXPECTED_LLM_ROWS, "groups": len(expected)} + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results-root", default="results") + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args(argv) + log: Log = (lambda _msg: None) if args.quiet else print + done = run(Path(args.results_root), log) + print( + f"completed analysis ({done['archives']}/{EXPECTED_ARCHIVES} archives, " + f"{done['llm_rows']}/{EXPECTED_LLM_ROWS} llm rows, {done['groups']} groups)" + ) + + +if __name__ == "__main__": + main() diff --git a/src/tinyrouter/haiku.py b/src/tinyrouter/haiku.py new file mode 100644 index 0000000..e2f3929 --- /dev/null +++ b/src/tinyrouter/haiku.py @@ -0,0 +1,143 @@ +"""Stored Haiku predictions as arrays, plus the old project's parsing rule for comparison. + +Reads ``results/llm/haiku-8way.jsonl`` (checked by ``llm_run.verify`` +before this is called) and never calls the API. The replies are +re-parsed two ways: + +- ``new``: ``llm.parse_agent``, the rule the run used and the hybrid uses; +- ``legacy``: cost-aware-hybrid-router ``src/routers/llm_router.py``: + lower-case and strip the reply, then take the first label of + ``VALID_AGENTS`` (a ``set``) that is a substring of it, else ``oos``. + Set iteration order depends on string hashing, so a reply containing two + labels had no fixed answer there. Here it takes the first in ``AGENTS`` + order and the number of such replies is reported as ``order_dependent``. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from tinyrouter.labels import AGENTS, OOS, load_label_space +from tinyrouter.llm import parse_agent +from tinyrouter.metrics import wilson_interval +from tinyrouter.selective import detection_counts + + +class HaikuMismatchError(RuntimeError): + """A stored prediction disagrees with the dataset labels the archives hold.""" + + +def legacy_parse(raw_text: str) -> tuple[str, int]: + """(agent, number of labels found as substrings) under the old project's rule.""" + text = raw_text.strip().lower() + hits = [agent for agent in AGENTS if agent in text] + return (hits[0] if hits else OOS), len(hits) + + +@dataclass(frozen=True) +class HaikuSplit: + """One split's stored predictions, row i = dataset index i.""" + + split: str + gold_intent: np.ndarray + pred: np.ndarray + legacy_pred: np.ndarray + parse_failed: np.ndarray + legacy_hits: np.ndarray + cost_usd: np.ndarray + raw_text: tuple[str, ...] + + +def read_records(path: Path) -> list[dict]: + with path.open(encoding="utf-8") as fh: + return [json.loads(line) for line in fh if line.strip()] + + +def split_arrays(records: list[dict], split: str, rows: int) -> HaikuSplit: + """Arrays for ``split`` ordered by index; indexes must be exactly 0..rows-1.""" + chosen = sorted((r for r in records if r["split"] == split), key=lambda r: r["index"]) + if [r["index"] for r in chosen] != list(range(rows)): + raise HaikuMismatchError(f"{split}: indexes are not exactly 0..{rows - 1}") + space = load_label_space() + for r in chosen: + expected = AGENTS[int(space.intent_to_agent_id[r["gold_intent"]])] + if r["gold_agent"] != expected: + raise HaikuMismatchError( + f"{split}[{r['index']}]: gold_agent {r['gold_agent']} but intent " + f"{r['gold_intent']} maps to {expected}" + ) + if parse_agent(r["raw_text"])[0] != r["agent"]: + raise HaikuMismatchError(f"{split}[{r['index']}]: stored agent is not parse_agent's") + legacy = [legacy_parse(r["raw_text"]) for r in chosen] + return HaikuSplit( + split=split, + gold_intent=np.array([r["gold_intent"] for r in chosen], dtype=np.int64), + pred=np.array([AGENTS.index(r["agent"]) for r in chosen], dtype=np.int64), + legacy_pred=np.array([AGENTS.index(a) for a, _ in legacy], dtype=np.int64), + parse_failed=np.array([r["parse_failed"] for r in chosen], dtype=bool), + legacy_hits=np.array([h for _, h in legacy], dtype=np.int64), + cost_usd=np.array([r["cost_usd"] for r in chosen], dtype=np.float64), + raw_text=tuple(r["raw_text"] for r in chosen), + ) + + +def check_gold(haiku: HaikuSplit, labels: np.ndarray, where: str) -> None: + """Row i of the predictions must carry the gold intent row i of the archive holds.""" + if haiku.gold_intent.shape != labels.shape or not np.array_equal(haiku.gold_intent, labels): + bad = ( + np.flatnonzero(haiku.gold_intent != labels) + if haiku.gold_intent.shape == labels.shape + else [] + ) + raise HaikuMismatchError( + f"{haiku.split}: Haiku gold_intent differs from {where} labels " + f"(shapes {haiku.gold_intent.shape} vs {labels.shape}; first rows {list(bad[:5])})" + ) + + +def routing_summary(pred: np.ndarray, gold_agent: np.ndarray) -> dict[str, object]: + oos = AGENTS.index(OOS) + counts = detection_counts(pred == oos, gold_agent == oos) + low, high = wilson_interval(counts["tp"], counts["gold_oos"]) + return { + "accuracy_8": float(np.mean(pred == gold_agent)), + "oos": counts, + "oos_recall_wilson95": [low, high], + "oos_misroute_rate": 1.0 - float(counts["recall"]), + "n": int(pred.size), + } + + +def summarize_split(haiku: HaikuSplit) -> dict[str, object]: + gold_agent = load_label_space().agents_of(haiku.gold_intent) + differ = np.flatnonzero(haiku.pred != haiku.legacy_pred) + wrong_if_failed = (haiku.pred == gold_agent) & ~haiku.parse_failed + return { + "new_parser": routing_summary(haiku.pred, gold_agent), + "legacy_parser": routing_summary(haiku.legacy_pred, gold_agent), + "parsers_disagree_rows": int(differ.size), + "parsers_disagree_examples": [ + { + "index": int(i), + "raw_text": haiku.raw_text[i], + "new": AGENTS[haiku.pred[i]], + "legacy": AGENTS[haiku.legacy_pred[i]], + } + for i in differ[:10] + ], + "legacy_order_dependent_rows": int(np.sum(haiku.legacy_hits > 1)), + "parse_failed_rows": int(haiku.parse_failed.sum()), + "accuracy_8_parse_failed_as_wrong": float(wrong_if_failed.mean()), + "raw_reply_counts": dict(sorted(_counts(haiku.raw_text).items())), + "cost_usd_total": float(haiku.cost_usd.sum()), + "cost_usd_per_1k_queries": float(1000 * haiku.cost_usd.mean()), + } + + +def _counts(texts: tuple[str, ...]) -> dict[str, int]: + values, counts = np.unique(np.array(texts, dtype=object).astype(str), return_counts=True) + return {str(v): int(c) for v, c in zip(values, counts, strict=True)} diff --git a/src/tinyrouter/selective.py b/src/tinyrouter/selective.py new file mode 100644 index 0000000..5ee9a9d --- /dev/null +++ b/src/tinyrouter/selective.py @@ -0,0 +1,207 @@ +"""Selective prediction and OOS detection metrics (RQ2 to RQ4). Pure numpy. + +Every score here is a confidence: higher means the small model trusts its +decision more. OOS detection flips the sign (low confidence means "this +looks out of scope"). + +Ties are resolved without randomness. The risk-coverage curve and AURC +use the expected value under a uniformly random order within each group +of tied scores, which is what a stable sort would give on average; AUROC +uses average ranks (a tie counts one half); average precision and the +threshold sweep only ever cut between distinct scores, so tied rows are +always accepted or rejected together. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +from tinyrouter.calibrate import FIT_SPLIT, LeakageError + +# One-sided 95% upper confidence bound: the threshold is chosen so that the +# validation selective risk is below the target with 95% confidence. +ONE_SIDED_95_Z = 1.6448536269514722 +COVERAGE_GRID = tuple(round(0.05 * i, 2) for i in range(1, 21)) + + +def require_validation(split: str, what: str) -> None: + if split != FIT_SPLIT: + raise LeakageError( + f"refusing to choose {what} on the '{split}' split; it is chosen on " + f"'{FIT_SPLIT}' only and then applied unchanged to test" + ) + + +def wilson_upper(successes: np.ndarray, n: np.ndarray, z: float) -> np.ndarray: + """Upper end of the Wilson score interval, elementwise; equals ``metrics.wilson_interval``.""" + k, m = np.asarray(successes, dtype=np.float64), np.asarray(n, dtype=np.float64) + if np.any(m < 1) or np.any(k < 0) or np.any(k > m): + raise ValueError("need 0 <= successes <= n and n >= 1") + p, z2 = k / m, z * z + centre = (p + z2 / (2 * m)) / (1 + z2 / m) + half = z * np.sqrt(p * (1 - p) / m + z2 / (4 * m * m)) / (1 + z2 / m) + return np.minimum(1.0, centre + half) + + +def _tie_groups(scores: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """(descending order, start of each tie group in it, size of each group).""" + order = np.argsort(-scores, kind="stable") + ranked = scores[order] + starts = np.flatnonzero(np.r_[True, ranked[1:] != ranked[:-1]]) + sizes = np.diff(np.r_[starts, ranked.size]) + return order, starts, sizes + + +def risk_coverage(confidence: np.ndarray, error: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Coverage i/n and expected selective risk of the i most confident rows, i = 1..n.""" + confidence, error = np.asarray(confidence, dtype=np.float64), np.asarray(error, dtype=bool) + if confidence.shape != error.shape or confidence.ndim != 1 or confidence.size == 0: + raise ValueError(f"need equal non-empty 1-d arrays, got {confidence.shape}, {error.shape}") + order, starts, sizes = _tie_groups(confidence) + group_errors = np.add.reduceat(error[order].astype(np.float64), starts) + before = np.r_[0.0, np.cumsum(group_errors)[:-1]] + group = np.repeat(np.arange(starts.size), sizes) + position = np.arange(confidence.size) - starts[group] + 1 + expected_errors = before[group] + group_errors[group] * position / sizes[group] + taken = np.arange(1, confidence.size + 1) + return taken / confidence.size, expected_errors / taken + + +def aurc(confidence: np.ndarray, error: np.ndarray) -> float: + """Area under the risk-coverage curve: mean selective risk over coverages 1/n .. n/n.""" + return float(risk_coverage(confidence, error)[1].mean()) + + +def risk_at_coverages( + confidence: np.ndarray, error: np.ndarray, grid: tuple[float, ...] = COVERAGE_GRID +) -> list[float]: + """Selective risk when keeping the ceil(c * n) most confident rows, for each c in ``grid``.""" + _, risk = risk_coverage(confidence, error) + n = risk.size + return [float(risk[max(1, int(np.ceil(c * n - 1e-9))) - 1]) for c in grid] + + +def _check_binary(scores: np.ndarray, positive: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + scores, positive = np.asarray(scores, dtype=np.float64), np.asarray(positive, dtype=bool) + if scores.shape != positive.shape or scores.ndim != 1: + raise ValueError(f"need equal 1-d arrays, got {scores.shape} and {positive.shape}") + if positive.all() or not positive.any(): + raise ValueError("need at least one positive and one negative row") + return scores, positive + + +def auroc(scores: np.ndarray, positive: np.ndarray) -> float: + """Probability a random positive outscores a random negative (ties count one half).""" + scores, positive = _check_binary(scores, positive) + _, inverse, counts = np.unique(scores, return_inverse=True, return_counts=True) + upper = np.cumsum(counts) + average_rank = (upper - (counts - 1) / 2.0)[inverse] + n_pos, n_neg = int(positive.sum()), int((~positive).sum()) + return float((average_rank[positive].sum() - n_pos * (n_pos + 1) / 2) / (n_pos * n_neg)) + + +def average_precision(scores: np.ndarray, positive: np.ndarray) -> float: + """Sum over distinct thresholds of (recall gain) x precision; sklearn's definition.""" + scores, positive = _check_binary(scores, positive) + order, starts, sizes = _tie_groups(scores) + hits = np.cumsum(np.add.reduceat(positive[order].astype(np.float64), starts)) + taken = np.cumsum(sizes) + recall = hits / positive.sum() + precision = hits / taken + return float(np.sum(np.diff(np.r_[0.0, recall]) * precision)) + + +def detection_counts(pred_oos: np.ndarray, gold_oos: np.ndarray) -> dict[str, float | int | None]: + """OOS recall, precision and F1 (precision 0 when nothing is predicted oos, as sklearn).""" + pred_oos, gold_oos = np.asarray(pred_oos, dtype=bool), np.asarray(gold_oos, dtype=bool) + tp = int(np.sum(pred_oos & gold_oos)) + positives, predicted = int(gold_oos.sum()), int(pred_oos.sum()) + if positives == 0: + raise ValueError("no gold oos rows") + recall = tp / positives + precision = tp / predicted if predicted else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + return { + "tp": tp, + "gold_oos": positives, + "predicted_oos": predicted, + "recall": recall, + "precision": precision, + "f1": f1, + } + + +def high_confidence_oos_misroute( + pred_oos: np.ndarray, gold_oos: np.ndarray, accepted: np.ndarray +) -> float: + """True OOS sent to an in-scope agent by the small model without asking the LLM, / all OOS.""" + pred_oos, gold_oos = np.asarray(pred_oos, dtype=bool), np.asarray(gold_oos, dtype=bool) + accepted = np.asarray(accepted, dtype=bool) + if not gold_oos.any(): + raise ValueError("no gold oos rows") + return float(np.sum(gold_oos & ~pred_oos & accepted) / gold_oos.sum()) + + +@dataclass(frozen=True) +class ThresholdChoice: + """The chosen confidence threshold and what it did on validation. + + ``tau`` is None when no threshold meets the target; the router then + sends every query to the LLM (coverage 0), which is what a deployed + router with an unreachable risk target would have to do. + """ + + tau: float | None + target_risk: float + coverage: float + risk: float | None + risk_upper: float | None + candidates: int + + @property + def feasible(self) -> bool: + return self.tau is not None + + +def select_threshold( + split: str, + confidence: np.ndarray, + error: np.ndarray, + target_risk: float, + z: float = ONE_SIDED_95_Z, +) -> ThresholdChoice: + """Largest-coverage threshold whose selective-risk Wilson upper bound is <= ``target_risk``. + + Candidates are the distinct validation scores; threshold t accepts + every row with confidence >= t. + """ + require_validation(split, "a deferral threshold") + confidence, error = np.asarray(confidence, dtype=np.float64), np.asarray(error, dtype=bool) + order, starts, sizes = _tie_groups(confidence) + accepted = np.cumsum(sizes) + errors = np.cumsum(np.add.reduceat(error[order].astype(np.float64), starts)) + upper = wilson_upper(errors, accepted, z) + ok = np.flatnonzero(upper <= target_risk) + if ok.size == 0: + return ThresholdChoice(None, target_risk, 0.0, None, None, int(starts.size)) + best = ok[-1] + return ThresholdChoice( + tau=float(confidence[order][starts[best]]), + target_risk=target_risk, + coverage=float(accepted[best] / confidence.size), + risk=float(errors[best] / accepted[best]), + risk_upper=float(upper[best]), + candidates=int(starts.size), + ) + + +def coverage_thresholds( + split: str, confidence: np.ndarray, grid: tuple[float, ...] = COVERAGE_GRID +) -> list[float]: + """For each c in ``grid``, the score of the ceil(c * n)-th most confident validation row.""" + require_validation(split, "operating-curve thresholds") + ranked = np.sort(np.asarray(confidence, dtype=np.float64))[::-1] + n = ranked.size + return [float(ranked[max(1, int(np.ceil(c * n - 1e-9))) - 1]) for c in grid] diff --git a/tests/test_analysis.py b/tests/test_analysis.py new file mode 100644 index 0000000..04e9f14 --- /dev/null +++ b/tests/test_analysis.py @@ -0,0 +1,222 @@ +import numpy as np +import pytest + +from tinyrouter.analysis import ( + AGGREGATIONS, + Routed, + analyze_run, + route, + router_metrics, + select_aggregation, + select_signal, +) +from tinyrouter.calibrate import LeakageError, SplitLogits +from tinyrouter.labels import AGENTS, load_label_space +from tinyrouter.selective import select_threshold + +F, T, OOS = AGENTS.index("finance_agent"), AGENTS.index("travel_agent"), AGENTS.index("oos") +SPACE = load_label_space() + + +def routed(split, pred, gold, signals=None): + return Routed(split, np.array(pred), np.array(gold), signals or {}) + + +def hand_case(): + small = routed("test", [F, T, F, OOS, T], [F, F, OOS, OOS, T]) + llm_pred = np.array([F, F, OOS, F, F]) + cost = np.array([1, 2, 3, 4, 5]) * 1e-3 + return small, llm_pred, cost + + +def test_hybrid_router_metrics_by_hand(): + small, llm_pred, cost = hand_case() + m = router_metrics(small, llm_pred, cost, np.array([0, 1, 0, 1, 0], bool)) + assert m["coverage"] == pytest.approx(3 / 5) + assert m["selective_risk"] == pytest.approx(1 / 3) + assert m["accuracy_8"] == pytest.approx(3 / 5) + assert m["oos_recall"] == 0.0 and m["oos_misroute_rate"] == 1.0 + # Gold oos rows 2 and 3: row 2 kept and routed to finance; row 3 predicted oos. + assert m["high_conf_oos_misroute_rate"] == pytest.approx(1 / 2) + # Small errors: rows 1, 2. Haiku fixes both; only row 1 was deferred. + assert m["error_recovery_rate"] == pytest.approx(1 / 2) + assert m["recoverable_caught"] == pytest.approx(1 / 2) + assert m["llm_call_rate"] == pytest.approx(2 / 5) + assert m["llm_cost_usd_per_1k"] == pytest.approx(1000 * (0.002 + 0.004) / 5) + + +def test_oracle_defers_exactly_the_small_models_errors(): + small, llm_pred, cost = hand_case() + m = router_metrics(small, llm_pred, cost, small.error) + assert m["accuracy_8"] == 1.0 + assert m["llm_call_rate"] == pytest.approx(2 / 5) + assert m["high_conf_oos_misroute_rate"] == 0.0 + assert m["error_recovery_rate"] == 1.0 and m["recoverable_caught"] == 1.0 + + +def test_small_only_misroute_equals_one_minus_oos_recall(): + small, llm_pred, cost = hand_case() + m = router_metrics(small, llm_pred, cost, np.zeros(5, bool)) + assert m["oos_recall"] == pytest.approx(1 / 2) + assert m["oos_misroute_rate"] == pytest.approx(1 / 2) + assert m["high_conf_oos_misroute_rate"] == m["oos_misroute_rate"] + assert m["llm_call_rate"] == 0.0 and m["error_recovery_rate"] == 0.0 + + +def one_row_logits(): + """oos is the top intent; two finance intents together outweigh it.""" + names = SPACE.intent_names + finance = [i for i, n in enumerate(names) if SPACE.intent_to_agent[n] == "finance_agent"] + z = np.full((1, SPACE.num_intents), -20.0) + z[0, SPACE.oos_intent_id] = 2.0 + z[0, finance[:2]] = 1.5 + return SplitLogits("validation", z.astype(np.float32), np.array([SPACE.oos_intent_id])) + + +def test_argmax_and_summed_aggregations_disagree_where_they_should(): + split = one_row_logits() + assert route(split, 1.0, "argmax", SPACE, "encoder").pred.tolist() == [OOS] + assert route(split, 1.0, "summed", SPACE, "encoder").pred.tolist() == [F] + + +def test_argmax_margin_is_the_score_margin_over_t(): + split = one_row_logits() + margin = route(split, 2.0, "argmax", SPACE, "encoder").signals["margin"] + assert margin == pytest.approx([(2.0 - 1.5) / 2.0]) + + +def test_signals_follow_the_plan_by_kind(): + split = one_row_logits() + assert set(route(split, 1.0, "argmax", SPACE, "encoder").signals) == { + "msp", + "entropy", + "margin", + "msp_t", + } + assert set(route(split, 1.0, "argmax", SPACE, "tfidf").signals) == {"margin", "msp_t"} + assert route(split, 1.0, "summed", SPACE, "majority").signals == {} + + +def test_aggregation_is_chosen_on_validation_and_ties_go_to_argmax(): + better = { + "argmax": routed("validation", [F, F], [F, T]), + "summed": routed("validation", [F, T], [F, T]), + } + assert select_aggregation(better)[0] == "summed" + tied = {"argmax": routed("validation", [F], [F]), "summed": routed("validation", [F], [F])} + assert select_aggregation(tied)[0] == "argmax" + + +def test_aggregation_and_signal_choices_refuse_test(): + leaked = {a: routed("test", [F], [F]) for a in AGGREGATIONS} + with pytest.raises(LeakageError, match="test"): + select_aggregation(leaked) + with pytest.raises(LeakageError, match="test"): + select_signal(routed("test", [F], [F], {"msp": np.array([1.0])}), {}) + + +def test_signal_choice_prefers_coverage_then_aurc(): + val = routed( + "validation", + [F, F, F, T], + [F, F, F, F], + {"msp": np.array([4.0, 3, 2, 1]), "entropy": np.array([1.0, 2, 3, 4])}, + ) + choices = { + s: select_threshold("validation", val.signals[s], val.error, 0.8) for s in val.signals + } + # Both keep every row at this target; msp ranks the error last, so its AURC is lower. + assert select_signal(val, choices) == "msp" + + +def synthetic_splits(seed=0, n_val=300, n_test=500): + """Logits that are right about 70% of the time; about 20% of rows are oos.""" + rng = np.random.default_rng(seed) + out = {} + for name, n in (("validation", n_val), ("test", n_test)): + labels = rng.integers(0, SPACE.num_intents, size=n) + labels[rng.random(n) < 0.2] = SPACE.oos_intent_id + z = rng.normal(size=(n, SPACE.num_intents)) + right = rng.random(n) < 0.7 + z[np.arange(n)[right], labels[right]] += 4.0 + out[name] = SplitLogits(name, z.astype(np.float32), labels) + return out + + +def fake_llm(n, seed=1): + rng = np.random.default_rng(seed) + return rng.integers(0, len(AGENTS), size=n), np.full(n, 3.7e-4) + + +def test_analyze_run_reports_both_aggregations_and_a_final_router(): + splits = synthetic_splits() + pred, cost = fake_llm(500) + scalars, curves = analyze_run( + splits["validation"], splits["test"], "encoder", SPACE, pred, cost + ) + assert scalars["selected_aggregation"] in AGGREGATIONS + for a in AGGREGATIONS: + assert set(scalars[a]["signals"]) == {"msp", "entropy", "margin", "msp_t"} + assert set(scalars[a]["fallback"]) == {"0.02", "0.05"} + assert len(curves[a]["msp"]["risk_coverage"]) == 20 + assert scalars["final"]["small_only"] == scalars[scalars["selected_aggregation"]]["small_only"] + + +def test_the_oracle_calls_the_llm_exactly_on_the_small_models_errors(): + splits = synthetic_splits() + pred, cost = fake_llm(500) + scalars, _ = analyze_run(splits["validation"], splits["test"], "encoder", SPACE, pred, cost) + for a in AGGREGATIONS: + small, oracle = scalars[a]["small_only"], scalars[a]["oracle"] + assert oracle["llm_call_rate"] == pytest.approx(1 - small["accuracy_8"]) + assert oracle["recoverable_caught"] == 1.0 + assert oracle["accuracy_8"] == pytest.approx( + small["accuracy_8"] + oracle["recovered_errors"] / 500 + ) + + +def test_majority_has_no_temperature_signals_or_hybrid(): + splits = synthetic_splits() + const = {n: SplitLogits(n, np.zeros_like(s.logits), s.labels) for n, s in splits.items()} + pred, cost = fake_llm(500) + scalars, _ = analyze_run(const["validation"], const["test"], "majority", SPACE, pred, cost) + assert scalars["calibration"]["temperature"] is None + assert scalars["calibration"]["test_ece"] is None + assert scalars["final"]["fallback"]["0.05"]["hybrid"] is None + + +def test_tfidf_reports_no_uncalibrated_ece(): + splits = synthetic_splits() + pred, cost = fake_llm(500) + scalars, _ = analyze_run(splits["validation"], splits["test"], "tfidf", SPACE, pred, cost) + assert scalars["calibration"]["test_ece"]["uncalibrated"] is None + assert scalars["calibration"]["test_ece"]["calibrated"]["ece_151"] >= 0 + + +def test_analyze_run_refuses_test_logits_in_the_validation_slot(): + splits = synthetic_splits() + pred, cost = fake_llm(500) + with pytest.raises(LeakageError): + analyze_run(splits["test"], splits["test"], "encoder", SPACE, pred, cost) + + +def choices_of(scalars): + fallback = { + a: { + t: {s: v["tau"] for s, v in f["by_signal"].items()} + for t, f in scalars[a]["fallback"].items() + } + for a in AGGREGATIONS + } + return scalars["calibration"]["temperature"], scalars["selected_aggregation"], fallback + + +def test_test_labels_do_not_move_any_choice(): + splits = synthetic_splits() + pred, cost = fake_llm(500) + before, _ = analyze_run(splits["validation"], splits["test"], "encoder", SPACE, pred, cost) + shuffled = np.random.default_rng(9).permutation(splits["test"].labels) + test = SplitLogits("test", splits["test"].logits, shuffled) + after, _ = analyze_run(splits["validation"], test, "encoder", SPACE, pred, cost) + assert choices_of(before) == choices_of(after) + assert before["final"]["small_only"] != after["final"]["small_only"] diff --git a/tests/test_analysis_run.py b/tests/test_analysis_run.py new file mode 100644 index 0000000..2ed4830 --- /dev/null +++ b/tests/test_analysis_run.py @@ -0,0 +1,248 @@ +import json + +import numpy as np +import pytest + +from archive_fakes import fake_metadata, fake_splits +from tinyrouter import analysis_run +from tinyrouter.analysis_run import ( + Point, + analyze_points, + collect_points, + combine, + load_haiku, + write_checked, +) +from tinyrouter.archive import MANIFEST_NAME, save_logits, write_manifest +from tinyrouter.calibrate import SplitLogits +from tinyrouter.completeness import IncompleteError +from tinyrouter.data import sha256_of +from tinyrouter.haiku import HaikuMismatchError, HaikuSplit +from tinyrouter.labels import AGENTS, load_label_space +from tinyrouter.llm import identity_sha256 +from tinyrouter.llm_run import EXPECTED_ROWS, Outcome, full_target, write_outputs + +KS, SEEDS = (1, 5, 10, 25, 50, 100), (42, 43, 44) + + +def test_combine_gives_mean_sample_std_and_values(): + out = combine([{"a": 1.0, "b": "x"}, {"a": 2.0, "b": "x"}, {"a": 6.0, "b": "x"}]) + assert out["a"]["mean"] == pytest.approx(3.0) + assert out["a"]["std"] == pytest.approx(np.std([1, 2, 6], ddof=1)) + assert out["a"]["values"] == [1.0, 2.0, 6.0] and out["b"] == "x" + + +def test_combine_keeps_missing_values_visible_and_merges_lists_elementwise(): + assert combine([None, None, None]) is None + partial = combine([0.5, None, 0.7]) + assert partial["mean"] == pytest.approx(0.6) and partial["n"] == 2 + assert [p["mean"] for p in combine([[1.0, 2.0], [3.0, 4.0]], keep_values=False)] == [2.0, 3.0] + assert combine(["argmax", "summed"]) == ["argmax", "summed"] + + +def fake_index_root(tmp_path): + """Four indexes, 75 points, each with its own (dummy) archive listed in the manifest.""" + root = tmp_path / "results" + (root / "logits").mkdir(parents=True) + manifest = {} + + def entry(run_name, **key): + path = root / "logits" / f"{run_name}.npz" + path.write_bytes(run_name.encode()) + manifest[path.name] = {"sha256": sha256_of(path)} + return { + **key, + "run_name": run_name, + "logits_file": path.name, + "logits_sha256": sha256_of(path), + } + + indexes = { + "bert.json": [ + entry(f"b-k{k}-s{s}", k=k, seed=s, oos_train=None) for k in KS for s in SEEDS + ], + "modernbert.json": [ + entry(f"m-k{k}-s{s}", k=k, seed=s, oos_train=None) for k in KS for s in SEEDS + ], + "oos-ablation.json": [entry(f"a-s{s}", k=100, seed=s, oos_train=0) for s in SEEDS], + "baselines.json": [ + entry(f"{b}-k{k}-s{s}", baseline=b, k=k, seed=s) + for b in ("majority", "tfidf-centroid") + for k in KS + for s in SEEDS + ], + } + (root / "curves").mkdir() + for name, points in indexes.items(): + (root / "curves" / name).write_text(json.dumps({"points": points}), encoding="utf-8") + write_manifest(root / MANIFEST_NAME, manifest) + return root + + +def test_collect_points_accepts_the_complete_set(tmp_path): + points = collect_points(fake_index_root(tmp_path)) + assert len(points) == 75 + assert len({p.group for p in points}) == 25 + + +def drop_first_point(root, index): + path = root / "curves" / index + body = json.loads(path.read_text(encoding="utf-8")) + body["points"] = body["points"][1:] + path.write_text(json.dumps(body), encoding="utf-8") + + +@pytest.mark.parametrize( + "index", ["bert.json", "modernbert.json", "oos-ablation.json", "baselines.json"] +) +def test_one_missing_point_in_any_index_stops_the_analysis(tmp_path, index): + root = fake_index_root(tmp_path) + drop_first_point(root, index) + with pytest.raises(IncompleteError): + collect_points(root) + + +def test_a_missing_archive_stops_the_analysis(tmp_path): + root = fake_index_root(tmp_path) + (root / "logits" / "a-s43.npz").unlink() + with pytest.raises(IncompleteError, match="missing"): + collect_points(root) + + +def test_the_archive_count_is_a_literal_not_derived_from_the_indexes(tmp_path, monkeypatch): + monkeypatch.setattr(analysis_run, "EXPECTED_ARCHIVES", 76) + with pytest.raises(IncompleteError, match="76"): + collect_points(fake_index_root(tmp_path)) + + +def haiku_record(split, index, gold_intent): + space = load_label_space() + agent = AGENTS[int(space.intent_to_agent_id[gold_intent])] + return { + "split": split, + "index": index, + "query_sha256": "0" * 64, + "gold_intent": gold_intent, + "gold_agent": agent, + "raw_text": agent, + "agent": agent, + "parse_failed": False, + "input_tokens": 300, + "output_tokens": 5, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cost_usd": 3.25e-4, + "latency_ms": 500, + "attempts": 1, + "request_id": None, + "stop_reason": "end_turn", + "identity_sha256": identity_sha256(), + "created_at": "2026-09-29T00:00:00+00:00", + } + + +def fake_haiku_root(tmp_path): + root = tmp_path / "results" + records = { + (s, i): haiku_record(s, i, i % 151) for s, n in EXPECTED_ROWS.items() for i in range(n) + } + write_outputs(full_target(root), Outcome(records=records, spent_before=0.0), 5.0) + return root + + +def test_load_haiku_reads_all_8600_rows(tmp_path): + haiku = load_haiku(fake_haiku_root(tmp_path)) + assert {s: h.pred.size for s, h in haiku.items()} == {"validation": 3100, "test": 5500} + + +def test_one_missing_haiku_row_stops_the_analysis(tmp_path): + root = fake_haiku_root(tmp_path) + path = full_target(root).predictions + lines = path.read_text(encoding="utf-8").splitlines() + path.write_text("\n".join(lines[:-1]) + "\n", encoding="utf-8") + with pytest.raises(Exception, match="expected 8600 unique rows"): + load_haiku(root) + + +def haiku_for(labels, split): + n = labels.size + return HaikuSplit( + split=split, + gold_intent=labels.copy(), + pred=np.zeros(n, dtype=np.int64), + legacy_pred=np.zeros(n, dtype=np.int64), + parse_failed=np.zeros(n, dtype=bool), + legacy_hits=np.ones(n, dtype=np.int64), + cost_usd=np.full(n, 1e-4), + raw_text=("finance_agent",) * n, + ) + + +def archive_root(tmp_path, n_archives=3, relabel=None): + """``n_archives`` real archives sharing labels (with oos rows); ``relabel`` edits one.""" + root = tmp_path / "results" + splits = fake_splits(seed=0, n_val=40, n_test=60) + oos = load_label_space().oos_intent_id + for s in splits.values(): + s.labels[::5] = oos + points = [] + for i, seed in enumerate(SEEDS[:n_archives]): + mine = { + n: SplitLogits(n, np.roll(s.logits, i, axis=0), s.labels.copy()) + for n, s in splits.items() + } + if relabel is not None and i == 1: + mine[relabel].labels[0] = (mine[relabel].labels[0] + 1) % 151 + name = f"m-k1-s{seed}" + save_logits(root / "logits" / f"{name}.npz", mine, fake_metadata(run_name=name, seed=seed)) + points.append(Point("m/k1", "m", 1, None, seed, "encoder", name, f"{name}.npz", "x")) + haiku = {n: haiku_for(s.labels, n) for n, s in splits.items()} + return root, points, haiku + + +def test_analyze_points_runs_on_archives_that_agree(tmp_path): + root, points, haiku = archive_root(tmp_path) + scalars, curves = analyze_points(points, root, haiku, lambda _m: None) + assert [p.seed for p, _ in scalars["m/k1"]] == list(SEEDS) + body = analysis_run.group_body(scalars["m/k1"], keep_values=True) + assert body["result"]["final"]["small_only"]["accuracy_8"]["values"] + + +@pytest.mark.parametrize("split", ["validation", "test"]) +def test_archives_whose_labels_differ_stop_the_analysis(tmp_path, split): + root, points, haiku = archive_root(tmp_path, relabel=split) + with pytest.raises(IncompleteError, match=f"{split} labels differ"): + analyze_points(points, root, haiku, lambda _m: None) + + +@pytest.mark.parametrize("split", ["validation", "test"]) +def test_haiku_gold_that_differs_from_the_archives_stops_the_analysis(tmp_path, split): + root, points, haiku = archive_root(tmp_path) + haiku[split].gold_intent[3] = (haiku[split].gold_intent[3] + 1) % 151 + with pytest.raises(HaikuMismatchError, match=split): + analyze_points(points, root, haiku, lambda _m: None) + + +def test_a_group_without_all_three_seeds_is_refused(tmp_path): + root, points, haiku = archive_root(tmp_path, n_archives=2) + scalars, _ = analyze_points(points, root, haiku, lambda _m: None) + with pytest.raises(IncompleteError, match="seeds"): + analysis_run.group_body(scalars["m/k1"], keep_values=True) + + +def test_an_output_that_fails_its_read_back_check_is_not_written(tmp_path): + out = tmp_path / "analysis" / "summary.json" + + def refuse(_body): + raise IncompleteError("wrong groups") + + with pytest.raises(IncompleteError): + write_checked(out, {"groups": {}}, refuse) + assert not out.exists() and not list(out.parent.iterdir()) + + +def test_outputs_refuse_nan(tmp_path): + out = tmp_path / "x.json" + with pytest.raises(ValueError): + write_checked(out, {"x": float("nan")}, lambda _b: None) + assert not out.exists() diff --git a/tests/test_haiku.py b/tests/test_haiku.py new file mode 100644 index 0000000..3f7057b --- /dev/null +++ b/tests/test_haiku.py @@ -0,0 +1,83 @@ +import numpy as np +import pytest + +from tinyrouter.haiku import ( + HaikuMismatchError, + check_gold, + legacy_parse, + split_arrays, + summarize_split, +) +from tinyrouter.labels import AGENTS, load_label_space +from tinyrouter.llm import parse_agent + +SPACE = load_label_space() + + +def record(index, raw, gold_intent, split="validation"): + agent, parsed = parse_agent(raw) + return { + "split": split, + "index": index, + "gold_intent": gold_intent, + "gold_agent": AGENTS[int(SPACE.intent_to_agent_id[gold_intent])], + "raw_text": raw, + "agent": agent, + "parse_failed": not parsed, + "cost_usd": 0.001, + } + + +def gold_of(agent): + return int(np.flatnonzero(SPACE.intent_to_agent_id == AGENTS.index(agent))[0]) + + +def test_legacy_parse_takes_the_first_substring_hit_or_oos(): + assert legacy_parse(" Finance_Agent ") == ("finance_agent", 1) + assert legacy_parse("no idea") == ("oos", 0) + # The new rule calls this ambiguous (oos, parse_failed); the old one routed it. + assert legacy_parse("oos (not travel_agent)") == ("travel_agent", 2) + assert parse_agent("oos (not travel_agent)") == ("oos", False) + + +def fake_records(): + return [ + record(0, "finance_agent", gold_of("finance_agent")), + record(1, "oos (not travel_agent)", gold_of("oos")), + record(2, "travel_agent", gold_of("finance_agent")), + ] + + +def test_summary_counts_parser_disagreements_and_parse_failures(): + summary = summarize_split(split_arrays(fake_records(), "validation", 3)) + assert summary["parsers_disagree_rows"] == 1 + assert summary["legacy_order_dependent_rows"] == 1 + assert summary["parse_failed_rows"] == 1 + assert summary["new_parser"]["accuracy_8"] == pytest.approx(2 / 3) + assert summary["legacy_parser"]["accuracy_8"] == pytest.approx(1 / 3) + # Row 1 is right under the new rule only because the failure fell back to oos. + assert summary["accuracy_8_parse_failed_as_wrong"] == pytest.approx(1 / 3) + assert summary["cost_usd_per_1k_queries"] == pytest.approx(1.0) + + +def test_split_arrays_needs_every_index_once(): + records = fake_records() + with pytest.raises(HaikuMismatchError, match="indexes"): + split_arrays(records[:1] + records[2:], "validation", 3) + + +def test_split_arrays_refuses_a_gold_agent_that_the_intent_does_not_map_to(): + records = fake_records() + records[0]["gold_agent"] = "travel_agent" + with pytest.raises(HaikuMismatchError, match="gold_agent"): + split_arrays(records, "validation", 3) + + +def test_gold_must_equal_the_archive_labels_row_by_row(): + haiku = split_arrays(fake_records(), "validation", 3) + check_gold(haiku, haiku.gold_intent.copy(), "fake.npz") + shifted = np.roll(haiku.gold_intent, 1) + with pytest.raises(HaikuMismatchError, match="fake.npz"): + check_gold(haiku, shifted, "fake.npz") + with pytest.raises(HaikuMismatchError): + check_gold(haiku, haiku.gold_intent[:2], "fake.npz") diff --git a/tests/test_selective.py b/tests/test_selective.py new file mode 100644 index 0000000..89addc2 --- /dev/null +++ b/tests/test_selective.py @@ -0,0 +1,149 @@ +import numpy as np +import pytest +from sklearn.metrics import average_precision_score, roc_auc_score + +from tinyrouter.calibrate import LeakageError +from tinyrouter.metrics import wilson_interval +from tinyrouter.selective import ( + ONE_SIDED_95_Z, + aurc, + auroc, + average_precision, + coverage_thresholds, + detection_counts, + high_confidence_oos_misroute, + risk_at_coverages, + risk_coverage, + select_threshold, + wilson_upper, +) + + +def test_aurc_by_hand_for_distinct_scores(): + # Keep 1: risk 0; keep 2: 1/2; keep 3: 1/3. + assert aurc(np.array([0.9, 0.8, 0.7]), np.array([0, 1, 0])) == pytest.approx( + (0 + 1 / 2 + 1 / 3) / 3 + ) + + +def test_aurc_ties_use_the_expected_order_so_a_constant_score_gives_the_error_rate(): + assert aurc(np.ones(3), np.array([0, 1, 0])) == pytest.approx(1 / 3) + + +def test_risk_coverage_spreads_a_tie_groups_errors_evenly(): + coverage, risk = risk_coverage(np.array([1.0, 1.0, 0.0]), np.array([1, 0, 0])) + assert coverage.tolist() == pytest.approx([1 / 3, 2 / 3, 1.0]) + assert risk.tolist() == pytest.approx([0.5, 0.5, 1 / 3]) + + +def test_risk_at_coverages_keeps_the_ceiling_of_c_times_n_rows(): + confidence = np.array([0.9, 0.8, 0.7, 0.6]) + error = np.array([0, 1, 0, 0]) + assert risk_at_coverages(confidence, error, (0.25, 0.5, 0.6, 1.0)) == pytest.approx( + [0.0, 0.5, 1 / 3, 0.25] + ) + + +def test_auroc_and_average_precision_by_hand(): + scores = np.array([0.9, 0.8, 0.7, 0.1]) + positive = np.array([True, False, True, False]) + # Pairs (pos, neg): 0.9>0.8, 0.9>0.1, 0.7<0.8, 0.7>0.1. + assert auroc(scores, positive) == pytest.approx(3 / 4) + # Recall steps 1/2 at precision 1, then 1/2 more at precision 2/3. + assert average_precision(scores, positive) == pytest.approx(0.5 * 1 + 0.5 * 2 / 3) + + +def test_auroc_of_a_constant_score_is_one_half(): + assert auroc(np.zeros(4), np.array([True, False, False, True])) == 0.5 + + +@pytest.mark.parametrize("seed", [0, 1, 2]) +def test_auroc_and_average_precision_match_sklearn_with_ties(seed): + rng = np.random.default_rng(seed) + scores = rng.integers(0, 6, size=300).astype(float) + positive = rng.random(300) < 0.3 + assert auroc(scores, positive) == pytest.approx(roc_auc_score(positive, scores)) + assert average_precision(scores, positive) == pytest.approx( + average_precision_score(positive, scores) + ) + + +def test_detection_needs_both_classes(): + with pytest.raises(ValueError, match="positive and one negative"): + auroc(np.array([0.1, 0.2]), np.array([True, True])) + + +@pytest.mark.parametrize(("k", "n"), [(0, 1), (0, 50), (3, 50), (50, 50), (17, 3100)]) +def test_wilson_upper_equals_the_metrics_interval(k, n): + expected = wilson_interval(k, n, z=ONE_SIDED_95_Z)[1] + assert wilson_upper(np.array([k]), np.array([n]), ONE_SIDED_95_Z)[0] == pytest.approx(expected) + + +def brute_force_threshold(confidence, error, target): + """Try every distinct score as a threshold; keep the feasible one with most rows.""" + best = None + for tau in np.unique(confidence): + kept = confidence >= tau + upper = wilson_interval(int(error[kept].sum()), int(kept.sum()), z=ONE_SIDED_95_Z)[1] + if upper <= target and (best is None or kept.sum() > best[1]): + best = (tau, kept.sum()) + return best + + +@pytest.mark.parametrize("target", [0.02, 0.05, 0.1, 0.3]) +def test_select_threshold_matches_a_brute_force_search(target): + rng = np.random.default_rng(7) + confidence = np.round(rng.random(400), 2) + error = rng.random(400) < 0.3 * (1 - confidence) + choice = select_threshold("validation", confidence, error, target) + expected = brute_force_threshold(confidence, error, target) + if expected is None: + assert not choice.feasible and choice.coverage == 0.0 + else: + assert choice.tau == expected[0] + assert choice.coverage == pytest.approx(expected[1] / 400) + + +def test_select_threshold_accepts_everything_when_the_target_is_loose(): + choice = select_threshold("validation", np.array([3.0, 2.0, 1.0]), np.array([0, 1, 1]), 1.0) + assert choice.tau == 1.0 and choice.coverage == 1.0 and choice.risk == pytest.approx(2 / 3) + + +def test_select_threshold_reports_an_unreachable_target(): + choice = select_threshold("validation", np.array([3.0, 2.0]), np.array([0, 0]), 1e-4) + assert choice.tau is None and not choice.feasible and choice.coverage == 0.0 + + +def test_select_threshold_never_splits_tied_scores(): + # Upper bounds: 1 row, 0 errors 0.73; 2 rows, 1 error 0.88; 3 rows, 1 error 0.746. + # Splitting the tie would keep the error-free first row alone at target 0.74. + choice = select_threshold("validation", np.array([1.0, 1.0, 0.0]), np.array([0, 1, 0]), 0.74) + assert choice.tau is None + + +@pytest.mark.parametrize("split", ["test", "train"]) +def test_threshold_and_curve_choices_refuse_any_split_but_validation(split): + with pytest.raises(LeakageError, match=split): + select_threshold(split, np.array([1.0, 0.0]), np.array([0, 1]), 0.05) + with pytest.raises(LeakageError, match=split): + coverage_thresholds(split, np.array([1.0, 0.0])) + + +def test_coverage_thresholds_are_the_validation_scores_at_each_coverage(): + scores = np.array([0.1, 0.9, 0.5, 0.7]) + assert coverage_thresholds("validation", scores, (0.25, 0.5, 1.0)) == [0.9, 0.7, 0.1] + + +def test_detection_counts_by_hand_and_zero_precision_when_nothing_is_predicted(): + counts = detection_counts(np.array([1, 1, 0, 0], bool), np.array([1, 0, 1, 0], bool)) + assert (counts["recall"], counts["precision"], counts["f1"]) == (0.5, 0.5, 0.5) + none = detection_counts(np.zeros(3, bool), np.array([1, 0, 0], bool)) + assert (none["recall"], none["precision"], none["f1"]) == (0.0, 0.0, 0.0) + + +def test_high_confidence_misroute_counts_only_kept_oos_sent_to_an_agent(): + gold_oos = np.array([1, 1, 1, 1, 0], bool) + pred_oos = np.array([1, 0, 0, 0, 0], bool) + accepted = np.array([1, 1, 0, 1, 1], bool) + # Rows 1 and 3: gold oos, routed to an agent, kept. Row 2 was deferred. + assert high_confidence_oos_misroute(pred_oos, gold_oos, accepted) == pytest.approx(2 / 4) From 63f87ba1af160ed8d1fa960e16e19251320158e8 Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:21:42 +0800 Subject: [PATCH 2/5] Pin the one-sided bound and the validation-only temperature fit in tests Two mutants survived the first mutation pass: a two-sided z for the threshold bound, and a temperature fit that skips the split check while a later guard still catches test data. Each now has a direct test. --- tests/test_analysis.py | 7 +++++++ tests/test_selective.py | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 04e9f14..c947188 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -5,6 +5,7 @@ AGGREGATIONS, Routed, analyze_run, + fit_run_temperature, route, router_metrics, select_aggregation, @@ -200,6 +201,12 @@ def test_analyze_run_refuses_test_logits_in_the_validation_slot(): analyze_run(splits["test"], splits["test"], "encoder", SPACE, pred, cost) +@pytest.mark.parametrize("kind", ["encoder", "tfidf"]) +def test_the_temperature_is_fitted_on_validation_only(kind): + with pytest.raises(LeakageError, match="test"): + fit_run_temperature(synthetic_splits()["test"], kind) + + def choices_of(scalars): fallback = { a: { diff --git a/tests/test_selective.py b/tests/test_selective.py index 89addc2..ea62b87 100644 --- a/tests/test_selective.py +++ b/tests/test_selective.py @@ -147,3 +147,12 @@ def test_high_confidence_misroute_counts_only_kept_oos_sent_to_an_agent(): accepted = np.array([1, 1, 0, 1, 1], bool) # Rows 1 and 3: gold oos, routed to an agent, kept. Row 2 was deferred. assert high_confidence_oos_misroute(pred_oos, gold_oos, accepted) == pytest.approx(2 / 4) + + +def test_the_bound_is_one_sided_95_percent(): + from scipy.stats import norm + + assert ONE_SIDED_95_Z == pytest.approx(norm.ppf(0.95)) + # 60 rows, no error: upper 0.043 one-sided, 0.060 two-sided; the target sits between. + choice = select_threshold("validation", np.arange(60.0), np.zeros(60, bool), 0.05) + assert choice.coverage == 1.0 From a6c949a3a818e442ea3b9994d5e6a64327f08a4d Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:51:49 +0800 Subject: [PATCH 3/5] Close review gaps in the analysis: guards, OOS score, threshold diagnostics Threshold choosers take a Scored built from one routed split, so a split name and another split's arrays cannot be mixed. RQ2 OOS detection uses 1 - max in-scope probability at T; the ablation compares both models with that score and one aggregation. A diagnostics block explains why validation thresholds miss the target on test (OOS share, harder test OOS) and reports the reweighted-validation sensitivity as diagnosis only. Tests cover OOS score direction, noise test logits, signal choice, entropy direction, temperature-scaled summed argmax, ties at tau, single-seed std, and a shared archive across indexes. --- DEVLOG.md | 42 +- README.md | 1 + results/analysis/summary.json | 10747 +++++++++++++++++++++++++++++-- src/tinyrouter/analysis.py | 63 +- src/tinyrouter/analysis_run.py | 59 +- src/tinyrouter/diagnostics.py | 117 + src/tinyrouter/selective.py | 76 +- tests/test_analysis.py | 141 +- tests/test_analysis_run.py | 56 + tests/test_diagnostics.py | 96 + tests/test_selective.py | 55 +- 11 files changed, 10738 insertions(+), 715 deletions(-) create mode 100644 src/tinyrouter/diagnostics.py create mode 100644 tests/test_diagnostics.py diff --git a/DEVLOG.md b/DEVLOG.md index 183c844..6a77ce4 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -4,6 +4,38 @@ --- +## 2026-09-29（深夜，二）：PR #17 審查修正（守門缺口與兩個會誤導的結論） + +### 本次工作 / 執行摘要 +- 審查結論：計算正確（無洩漏、可逐位元重現、獨立重算吻合），退回原因是 9 條突變存活與兩個報告結論。這次一次修完 F1 到 F11。 +- **守門不再只看 split 名稱（F2）**：`select_threshold`、`coverage_thresholds` 改成只收 `Scored`，由 `Routed.scored(signal)` 從同一個 routed split 取出名稱、分數與錯誤，不能再「傳 validation 名稱配 test 陣列」。打亂 labels 的測試之外，新增「test logits 換成亂數」的測試，並把操作曲線的 τ、hybrid 的訊號、(b) 敏感度的 τ 都納入比對。 +- **RQ2 OOS 偵測改用專用分數（F3）**：`1 − max in-scope 機率`（擬合的 T 下；argmax 聚合用 151 類、summed 用 8 類），寫在每個聚合的 `oos_detection`。四個信心訊號只留給 RQ3。`summary.json` 新增 `ablation_comparison`：偵測固定用 argmax 聚合，router 行為用 final router，全部由程式產生。 +- **門檻失效的診斷（F4）**：`diagnostics.py`，對每個 encoder 點的 final hybrid 報 val 與 test 的 selective risk、只看 in-scope 的 risk、被收下 OOS 的誤判率、test 依 validation OOS 比例重新加權後的 risk、比例解釋的缺口比例，另報選項 (b) 的敏感度（validation 依 18.2% OOS 加權後選 τ 再套 test，註明這個比例是看過 test 才知道的，只作診斷）。加權的 Wilson 上界用加權後的計數。 +- **決定（協調者依 Drew 的專案原則拍板）**：主結果採 (a) 照實報，validation-only 協定不變；(b) 只當診斷。 +- 其他：F1 OOS AUROC／AUPRC 方向的呼叫端測試（對 sklearn）；F5 `select_signal` 用手算 choices 測；F6 entropy 方向；F7 summed 用溫度校準機率（T 會改變 argmax 的案例）；F8 分數恰好等於 τ 的列由小模型保留（`deferred_below`，有測試）；F9 可行 seed 少於 2 個時 std 記 null 並保留 n（例：bert/k1 目標 2% 只有 1 個 seed 可行）；F10 兩個 index 指向同一個 archive 會紅；F11 正式程式碼的 `assert isinstance` 改成明確 raise。 + +### 核心發現 / 數據 +（全部取自 `results/analysis/summary.json`，test，百分比，mean ± std） +- **消融改正後**（偵測固定 argmax 聚合、分數 `1 − max in-scope p`）：OOS 250 筆 AUROC 98.33 ± 0.07、AUPRC 94.34 ± 0.25；OOS 0 筆 97.80 ± 0.04、91.52 ± 0.26。有 OOS 訓練資料的模型較好，與前一則的方向相反。 +- **消融的主比較是 router 行為**（目標 2%，OOS 250 對 0）：hybrid OOS recall 62.43 ± 1.33 對 40.60 ± 1.04；高信心 OOS 誤派 34.53 ± 5.53 對 37.67 ± 2.90；LLM 呼叫率 1.27 ± 1.59 對 12.07 ± 0.73。沒有 OOS 訓練資料時，要多交 LLM 近 10 倍的查詢，OOS 仍少攔約 22pp。 +- **門檻失效的歸因**（ModernBERT k=100，目標 2%）：val 1.50 ± 0.13，test 7.34 ± 0.86，test 依 validation OOS 比例加權後 2.29 ± 0.11，比例解釋缺口的 86.66 ± 1.39%。其餘來自 test 的 OOS 較難：被收下 OOS 的誤判率 val 19.91 ± 2.93 對 test 36.48 ± 3.04；in-scope risk 0.90 ± 0.23 對 1.20 ± 0.07。目標 2% 下其他 encoder 點比例解釋約 78% 到 97%（BERT k=5、k=10 超過 100%，加權後反而低於 val）；目標 5% 的 ModernBERT k=1 只有 27%，那一點 coverage 很低、數字不穩。 +- **(b) 敏感度**（同點）：test risk 2.93 ± 0.63（三個 seed 2.58、3.65、2.55），coverage 88.55 ± 1.76。依部署 OOS 比例加權後仍略高於 2%。 +- **結論**：validation 選的門檻在 test 上守不住目標 risk，主要來自 OOS 比例差異（約 86%），其餘是 test OOS 較難；依部署 OOS 比例加權仍不足，部署時應在接近真實流量的資料上重新校準門檻。 +- 重構後 `curves.json` 與改前逐位元組相同（所有 τ 與曲線不變），`summary.json` 除新增欄位外只有 48 處 std 由 0 改成 null（F9）；`make analysis` 連跑兩次輸出相同。 + +### Blockers / 遇到的問題 +- 重新下載 Release 時連線中斷，5 個 archive 下載不完整；`make verify-logits` 以 SHA-256 不符擋下，重下後通過。 + +### Next +- [ ] 步驟 5：README 與圖、成本表（RQ5），數字由 `make report` 從 `results/analysis/*.json` 產生；README 寫明門檻要在接近真實流量的資料上重新校準 + +### Files / Budget +- 新增：`src/tinyrouter/diagnostics.py`、`tests/test_diagnostics.py` +- 修改：`src/tinyrouter/selective.py`、`analysis.py`、`analysis_run.py`；`tests/test_selective.py`、`test_analysis.py`、`test_analysis_run.py`；`results/analysis/summary.json`；`README.md`；`DEVLOG.md` +- API 花費：US$0 + +--- + ## 2026-09-29（深夜）：步驟 4 後半，RQ2 到 RQ4 離線分析 ### 本次工作 / 執行摘要 @@ -20,18 +52,18 @@ （全部取自 `results/analysis/*.json`，test，百分比，mean ± std） - **Haiku 4.5 zero-shot**：8 類準確率 82.1，OOS recall 56.8（Wilson 95% 53.7 到 59.8），每 1K 查詢 US$0.369。8,600 筆回覆全是 8 個標籤之一，parse_failed 0 列，新舊解析規則不一致 0 列，順序相依 0 列。 - **ModernBERT k=100（主模型，validation 選 argmax）**：small-only 91.9 ± 0.1，OOS recall 61.1 ± 0.4；oracle 95.1 ± 0.1、LLM 呼叫 8.1 ± 0.1。 -- **validation 選的門檻在 test 上守不住目標 risk**。目標 2%：ModernBERT k=100 test selective risk 7.3 ± 0.9（coverage 98.7）、k=25 為 5.3 ± 0.5；目標 5% 在 k ≥ 25 時 coverage 幾乎 100%，test selective risk 8.0 到 13.5。原因是 CLINC150 的 validation 只有 100/3,100（3.2%）OOS，test 是 1,000/5,500（18.2%）；小模型的錯誤大多是 OOS，validation 上錯誤率本來就低，Wilson 上界再保守也看不到 test 的 OOS 比例。這不是程式錯，是協定（只用 validation）在這個資料集上的已知代價，需要 Drew 決定要不要處理（見 Blockers）。 +- **validation 選的門檻在 test 上守不住目標 risk**。目標 2%：ModernBERT k=100 test selective risk 7.3 ± 0.9（coverage 98.7）、k=25 為 5.3 ± 0.5；目標 5% 在 k ≥ 25 時 coverage 幾乎 100%，test selective risk 8.0 到 13.5。歸因見下一則（審查修正）的診斷：主要來自 OOS 比例差異，其餘是 test 的 OOS 較難。 - 小資料量時 hybrid 的價值最清楚：ModernBERT k=10 在目標 2% 下 88.0 ± 0.4，LLM 呼叫 23.9 ± 3.8，高於 Haiku 單獨的 82.1 與 small-only 的 81.5 ± 0.7。 -- **OOS 0 筆消融**：small-only OOS recall 0.0（從不預測 oos），但只靠不確定性當 OOS 分數時 AUROC 反而更高：msp_t 98.0 ± 0.1、AUPRC 92.2 ± 0.4，對照同點 OOS 250 筆的 91.9 ± 0.5、72.0 ± 0.7。注意這個比較對有 OOS 訓練的模型不利：它有信心地預測成 oos 的查詢，在「低信心 = OOS」的分數下反而被當成最不像 OOS。以 router 實際行為比較：消融在目標 2% 下 hybrid OOS recall 40.6 ± 1.0、高信心 OOS 誤派 37.7 ± 2.9；目標 5% 時門檻放到全收，高信心 OOS 誤派 99.3 ± 0.2。 +- **OOS 0 筆消融**：small-only OOS recall 0.0（從不預測 oos）。本則原先寫「只靠不確定性當 OOS 分數時 AUROC 反而更高（msp_t 98.0 對 91.9）」，那是分數選擇造成的假象，已在下一則改用 `1 − max in-scope p` 並固定聚合重算，方向反過來。以 router 實際行為比較：消融在目標 2% 下 hybrid OOS recall 40.6 ± 1.0、高信心 OOS 誤派 37.7 ± 2.9；目標 5% 時門檻放到全收，高信心 OOS 誤派 99.3 ± 0.2。 - 聚合方式：ModernBERT k ≤ 10 三個 seed 都選 summed，k ≥ 50 都選 argmax；BERT 在 k=10、50、100 三個 seed 選得不一致，兩者差距都在 1pp 內。兩種的 test 數字都在 JSON。 - 多數類：argmax 聚合永遠猜 oos（k-shot 樣本裡 oos 是單一最大 intent），summed 永遠猜 finance；validation 選 summed，test 8 類 20.7。 ### Blockers / 遇到的問題 -- **待 Drew 決定**：validation 與 test 的 OOS 比例差 5.7 倍，使「只用 validation 選門檻」在 test 上系統性地超出目標 risk。可選：(a) 照實報，當成部署時要重新校準門檻的證據；(b) 另報一個敏感度版本，validation 的 OOS 列依「假設的部署 OOS 比例」加權（比例是事先宣告的參數，不從 test 估），這會動到 PLAN 的協定，所以這次沒做。 -- 「不確定性當 OOS 分數」對有 OOS 訓練的模型不公平（上面的消融段）；若要公平比較，可另加「預測為 oos 或低信心」的組合分數。這次照任務只報四個訊號。 +- 門檻失效要不要處理：已決定（見下一則），主結果照實報，(b) 只當診斷。 +- 「低信心當 OOS 分數」對有 OOS 訓練的模型不公平：已在下一則改用專用的 OOS 偵測分數。 ### Next -- [ ] Drew 決定上面兩點 +- [x] 上面兩點已在下一則處理 - [ ] 步驟 5：README 與圖（risk-coverage、操作曲線、reliability），成本表（RQ5），數字由 `make report` 從 `results/analysis/*.json` 產生 ### Files / Budget diff --git a/README.md b/README.md index 016269a..6a5092e 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ src/tinyrouter/ selective.py AURC, risk-coverage, AUROC, AUPRC, Wilson-bound threshold choice (validation only) haiku.py stored Haiku predictions as arrays; old and new reply parsers analysis.py one archive: 8-way aggregation, OOS, signals, ECE, fallback, oracle + diagnostics.py why validation thresholds miss the target on test (OOS share, reweighting) analysis_run.py `make analysis`: completion checks, label cross-checks, mean and std over seeds report.py results/*.json -> results/summary.md smoke.py end-to-end wiring check diff --git a/results/analysis/summary.json b/results/analysis/summary.json index 04164ab..84cc4c0 100644 --- a/results/analysis/summary.json +++ b/results/analysis/summary.json @@ -45,6 +45,9 @@ "one_sided_z":1.644854, "signal_choice":"largest validation coverage at the target, then lower validation AURC", "oracle":"defers exactly the queries the small model gets wrong (8-way)", + "oos_detection_score":"1 - max in-scope probability at the fitted T (151 intents for argmax, 8 agents for summed); the four confidence signals are for RQ3", + "threshold_result":"main result: thresholds chosen on validation only, reported as they do on test (option a); diagnostics.sensitivity_reweighted_validation is option (b), diagnosis only", + "tie_at_tau":"a row whose confidence equals tau is kept by the small model", "error_recovery_rate":"small-model errors deferred and fixed by Haiku / small errors", "recoverable_caught":"errors deferred and fixed by Haiku / errors Haiku would fix", "high_conf_oos_misroute_rate":"gold OOS kept by the small model (confidence >= tau) and routed to an in-scope agent / gold OOS", @@ -348,6 +351,28 @@ "std":61.147363 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.714808, + "std":0.025399 + }, + "auprc":{ + "mean":0.104562, + "std":0.032461 + } + }, + "test":{ + "auroc":{ + "mean":0.672274, + "std":0.021698 + }, + "auprc":{ + "mean":0.316169, + "std":0.045295 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -1083,7 +1108,7 @@ "entropy":{ "tau":{ "mean":-4.075108, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -1097,12 +1122,12 @@ }, "validation_selective_risk":{ "mean":0.0, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.045315, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -1112,7 +1137,7 @@ }, "selective_risk":{ "mean":0.013514, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -1182,7 +1207,7 @@ "margin":{ "tau":{ "mean":3.709316, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -1196,12 +1221,12 @@ }, "validation_selective_risk":{ "mean":0.019737, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.048357, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -1211,7 +1236,7 @@ }, "selective_risk":{ "mean":0.077491, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -1281,7 +1306,7 @@ "msp_t":{ "tau":{ "mean":0.888534, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -1295,12 +1320,12 @@ }, "validation_selective_risk":{ "mean":0.010526, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.045809, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -1310,7 +1335,7 @@ }, "selective_risk":{ "mean":0.070513, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -1530,6 +1555,28 @@ "std":77.500538 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.752077, + "std":0.020836 + }, + "auprc":{ + "mean":0.126418, + "std":0.013985 + } + }, + "test":{ + "auroc":{ + "mean":0.695794, + "std":0.009686 + }, + "auprc":{ + "mean":0.336892, + "std":0.030114 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -1662,7 +1709,7 @@ "hybrid":{ "tau":{ "mean":0.401006, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -1676,12 +1723,12 @@ }, "validation_selective_risk":{ "mean":0.0, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.01272, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -1691,7 +1738,7 @@ }, "selective_risk":{ "mean":0.023256, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -1762,7 +1809,7 @@ "msp":{ "tau":{ "mean":0.401006, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -1776,12 +1823,12 @@ }, "validation_selective_risk":{ "mean":0.0, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.01272, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -1791,7 +1838,7 @@ }, "selective_risk":{ "mean":0.023256, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -1861,7 +1908,7 @@ "entropy":{ "tau":{ "mean":-1.700069, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -1875,12 +1922,12 @@ }, "validation_selective_risk":{ "mean":0.0, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.017602, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -1890,7 +1937,7 @@ }, "selective_risk":{ "mean":0.027027, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -2922,6 +2969,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.752077, + "std":0.020836, + "values":[ + 0.748983, + 0.73296, + 0.774287 + ] + }, + "auprc":{ + "mean":0.126418, + "std":0.013985, + "values":[ + 0.110309, + 0.135448, + 0.133497 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.695794, + "std":0.009686, + "values":[ + 0.706831, + 0.691837, + 0.688712 + ] + }, + "auprc":{ + "mean":0.336892, + "std":0.030114, + "values":[ + 0.348905, + 0.359145, + 0.302626 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -3174,7 +3263,7 @@ "hybrid":{ "tau":{ "mean":0.401006, - "std":0.0, + "std":null, "n":1, "values":[ null, @@ -3198,7 +3287,7 @@ }, "validation_selective_risk":{ "mean":0.0, - "std":0.0, + "std":null, "n":1, "values":[ null, @@ -3208,7 +3297,7 @@ }, "validation_risk_upper95":{ "mean":0.01272, - "std":0.0, + "std":null, "n":1, "values":[ null, @@ -3228,7 +3317,7 @@ }, "selective_risk":{ "mean":0.023256, - "std":0.0, + "std":null, "n":1, "values":[ null, @@ -3580,6 +3669,629 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":[ + "msp_t", + "msp", + "margin" + ], + "tau":{ + "mean":0.401006, + "std":null, + "n":1, + "values":[ + null, + 0.401006, + null + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.022581, + "std":0.039111, + "values":[ + 0.0, + 0.067742, + 0.0 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":null, + "n":1, + "values":[ + null, + 0.0, + null + ] + }, + "in_scope_risk":{ + "mean":0.0, + "std":null, + "n":1, + "values":[ + null, + 0.0, + null + ] + }, + "kept_oos_error_rate":null, + "kept_oos_share":{ + "mean":0.0, + "std":null, + "n":1, + "values":[ + null, + 0.0, + null + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.015636, + "std":0.027083, + "values":[ + 0.0, + 0.046909, + 0.0 + ] + }, + "selective_risk":{ + "mean":0.023256, + "std":null, + "n":1, + "values":[ + null, + 0.023256, + null + ] + }, + "in_scope_risk":{ + "mean":0.0, + "std":null, + "n":1, + "values":[ + null, + 0.0, + null + ] + }, + "kept_oos_error_rate":{ + "mean":0.6, + "std":null, + "n":1, + "values":[ + null, + 0.6, + null + ] + }, + "kept_oos_share":{ + "mean":0.03876, + "std":null, + "n":1, + "values":[ + null, + 0.03876, + null + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002, + "std":0.003464, + "values":[ + 0.0, + 0.006, + 0.0 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.017885, + "std":0.030978, + "values":[ + 0.0, + 0.053656, + 0.0 + ] + }, + "selective_risk":{ + "mean":0.003607, + "std":null, + "n":1, + "values":[ + null, + 0.003607, + null + ] + }, + "in_scope_risk":{ + "mean":0.0, + "std":null, + "n":1, + "values":[ + null, + 0.0, + null + ] + }, + "kept_oos_error_rate":{ + "mean":0.6, + "std":null, + "n":1, + "values":[ + null, + 0.6, + null + ] + }, + "kept_oos_share":{ + "mean":0.006012, + "std":null, + "n":1, + "values":[ + null, + 0.006012, + null + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002, + "std":0.003464, + "values":[ + 0.0, + 0.006, + 0.0 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.84489, + "std":null, + "n":1, + "values":[ + null, + 0.84489, + null + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.401006, + "std":null, + "n":1, + "values":[ + null, + 0.401006, + null + ] + }, + "feasible":[ + false, + true, + false + ], + "validation_weighted_coverage":{ + "mean":0.019087, + "std":0.033059, + "values":[ + 0.0, + 0.05726, + 0.0 + ] + }, + "validation_weighted_risk":{ + "mean":0.0, + "std":null, + "n":1, + "values":[ + null, + 0.0, + null + ] + }, + "test":{ + "coverage":{ + "mean":0.015636, + "std":0.027083, + "values":[ + 0.0, + 0.046909, + 0.0 + ] + }, + "selective_risk":{ + "mean":0.023256, + "std":null, + "n":1, + "values":[ + null, + 0.023256, + null + ] + }, + "in_scope_risk":{ + "mean":0.0, + "std":null, + "n":1, + "values":[ + null, + 0.0, + null + ] + }, + "kept_oos_error_rate":{ + "mean":0.6, + "std":null, + "n":1, + "values":[ + null, + 0.6, + null + ] + }, + "kept_oos_share":{ + "mean":0.03876, + "std":null, + "n":1, + "values":[ + null, + 0.03876, + null + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.002, + "std":0.003464, + "values":[ + 0.0, + 0.006, + 0.0 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "entropy", + "msp", + "msp" + ], + "tau":{ + "mean":-0.340128, + "std":1.222357, + "values":[ + -1.751573, + 0.370436, + 0.360752 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.101613, + "std":0.024446, + "values":[ + 0.073871, + 0.110968, + 0.12 + ] + }, + "selective_risk":{ + "mean":0.02828, + "std":0.001818, + "values":[ + 0.026201, + 0.02907, + 0.02957 + ] + }, + "in_scope_risk":{ + "mean":0.024092, + "std":0.002584, + "values":[ + 0.02193, + 0.023392, + 0.026954 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.00429, + "std":0.001564, + "values":[ + 0.004367, + 0.005814, + 0.002688 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.013333, + "std":0.005774, + "values":[ + 0.01, + 0.02, + 0.01 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.08503, + "std":0.026823, + "values":[ + 0.055636, + 0.091273, + 0.108182 + ] + }, + "selective_risk":{ + "mean":0.058689, + "std":0.020595, + "values":[ + 0.055556, + 0.039841, + 0.080672 + ] + }, + "in_scope_risk":{ + "mean":0.024462, + "std":0.010429, + "values":[ + 0.023649, + 0.014463, + 0.035273 + ] + }, + "kept_oos_error_rate":{ + "mean":0.907407, + "std":0.160375, + "values":[ + 1.0, + 0.722222, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.038532, + "std":0.007554, + "values":[ + 0.03268, + 0.035857, + 0.047059 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.017, + "std":0.009644, + "values":[ + 0.01, + 0.013, + 0.028 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.097161, + "std":0.030139, + "values":[ + 0.063978, + 0.104667, + 0.122839 + ] + }, + "selective_risk":{ + "mean":0.029776, + "std":0.012034, + "values":[ + 0.028571, + 0.018389, + 0.042367 + ] + }, + "in_scope_risk":{ + "mean":0.024462, + "std":0.010429, + "values":[ + 0.023649, + 0.014463, + 0.035273 + ] + }, + "kept_oos_error_rate":{ + "mean":0.907407, + "std":0.160375, + "values":[ + 1.0, + 0.722222, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.005981, + "std":0.001215, + "values":[ + 0.005042, + 0.005548, + 0.007353 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.017, + "std":0.009644, + "values":[ + 0.01, + 0.013, + 0.028 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":1.220148, + "std":0.673479, + "values":[ + 0.919244, + 1.991621, + 0.74958 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.399272, + "std":0.001517, + "n":2, + "values":[ + null, + 0.3982, + 0.400345 + ] + }, + "feasible":[ + false, + true, + true + ], + "validation_weighted_coverage":{ + "mean":0.029721, + "std":0.028937, + "values":[ + 0.0, + 0.057805, + 0.031357 + ] + }, + "validation_weighted_risk":{ + "mean":0.006706, + "std":0.002813, + "n":2, + "values":[ + null, + 0.004717, + 0.008696 + ] + }, + "test":{ + "coverage":{ + "mean":0.028121, + "std":0.025809, + "values":[ + 0.0, + 0.050727, + 0.033636 + ] + }, + "selective_risk":{ + "mean":0.042275, + "std":0.024303, + "n":2, + "values":[ + null, + 0.02509, + 0.059459 + ] + }, + "in_scope_risk":{ + "mean":0.018525, + "std":0.020942, + "n":2, + "values":[ + null, + 0.003717, + 0.033333 + ] + }, + "kept_oos_error_rate":{ + "mean":0.8, + "std":0.282843, + "n":2, + "values":[ + null, + 0.6, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.031435, + "std":0.006233, + "n":2, + "values":[ + null, + 0.035842, + 0.027027 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.003667, + "std":0.003215, + "values":[ + 0.0, + 0.006, + 0.005 + ] + } + } + } + } } } }, @@ -3838,6 +4550,28 @@ "std":74.446849 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.9046, + "std":0.003276 + }, + "auprc":{ + "mean":0.477856, + "std":0.020971 + } + }, + "test":{ + "auroc":{ + "mean":0.892855, + "std":0.018164 + }, + "auprc":{ + "mean":0.725594, + "std":0.057483 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -5036,6 +5770,28 @@ "std":69.973805 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.90428, + "std":0.01395 + }, + "auprc":{ + "mean":0.507568, + "std":0.050584 + } + }, + "test":{ + "auroc":{ + "mean":0.910829, + "std":0.012737 + }, + "auprc":{ + "mean":0.76365, + "std":0.039589 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -5260,7 +6016,7 @@ "msp":{ "tau":{ "mean":0.482985, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -5274,12 +6030,12 @@ }, "validation_selective_risk":{ "mean":0.003717, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.016489, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -5289,7 +6045,7 @@ }, "selective_risk":{ "mean":0.034483, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -5359,7 +6115,7 @@ "entropy":{ "tau":{ "mean":-1.599557, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -5373,12 +6129,12 @@ }, "validation_selective_risk":{ "mean":0.003817, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.016925, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -5388,7 +6144,7 @@ }, "selective_risk":{ "mean":0.035135, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -6416,6 +7172,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.905987, + "std":0.00327, + "values":[ + 0.907363, + 0.908343, + 0.902253 + ] + }, + "auprc":{ + "mean":0.493539, + "std":0.046069, + "values":[ + 0.544983, + 0.456093, + 0.479541 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.898291, + "std":0.026184, + "values":[ + 0.925535, + 0.896024, + 0.873315 + ] + }, + "auprc":{ + "mean":0.738133, + "std":0.074124, + "values":[ + 0.808891, + 0.744462, + 0.661047 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -7062,6 +7860,600 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":"msp_t", + "tau":{ + "mean":0.81507, + "std":0.075711, + "values":[ + 0.893141, + 0.741964, + 0.810105 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.680968, + "std":0.111005, + "values":[ + 0.798387, + 0.666774, + 0.577742 + ] + }, + "selective_risk":{ + "mean":0.014795, + "std":0.000484, + "values":[ + 0.015354, + 0.014514, + 0.014517 + ] + }, + "in_scope_risk":{ + "mean":0.010146, + "std":0.00102, + "values":[ + 0.008969, + 0.010747, + 0.010722 + ] + }, + "kept_oos_error_rate":{ + "mean":0.498565, + "std":0.198695, + "values":[ + 0.727273, + 0.4, + 0.368421 + ] + }, + "kept_oos_share":{ + "mean":0.009724, + "std":0.000861, + "values":[ + 0.008889, + 0.009676, + 0.010609 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.103333, + "std":0.049329, + "values":[ + 0.16, + 0.08, + 0.07 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.58503, + "std":0.095454, + "values":[ + 0.686545, + 0.571455, + 0.497091 + ] + }, + "selective_risk":{ + "mean":0.037703, + "std":0.00459, + "values":[ + 0.039195, + 0.041362, + 0.032553 + ] + }, + "in_scope_risk":{ + "mean":0.009665, + "std":0.002452, + "values":[ + 0.010547, + 0.011553, + 0.006894 + ] + }, + "kept_oos_error_rate":{ + "mean":0.564358, + "std":0.078713, + "values":[ + 0.635838, + 0.48, + 0.577236 + ] + }, + "kept_oos_share":{ + "mean":0.051479, + "std":0.010534, + "values":[ + 0.045816, + 0.063633, + 0.044989 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.092333, + "std":0.019757, + "values":[ + 0.11, + 0.096, + 0.071 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.661749, + "std":0.109209, + "values":[ + 0.780419, + 0.639355, + 0.565473 + ] + }, + "selective_risk":{ + "mean":0.014065, + "std":0.002816, + "values":[ + 0.015018, + 0.01628, + 0.010896 + ] + }, + "in_scope_risk":{ + "mean":0.009665, + "std":0.002452, + "values":[ + 0.010547, + 0.011553, + 0.006894 + ] + }, + "kept_oos_error_rate":{ + "mean":0.564358, + "std":0.078713, + "values":[ + 0.635838, + 0.48, + 0.577236 + ] + }, + "kept_oos_share":{ + "mean":0.008086, + "std":0.001737, + "values":[ + 0.007151, + 0.010091, + 0.007017 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.092333, + "std":0.019757, + "values":[ + 0.11, + 0.096, + 0.071 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":1.049689, + "std":0.136802, + "values":[ + 1.014069, + 0.93422, + 1.200777 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.952936, + "std":0.038084, + "values":[ + 0.994706, + 0.943961, + 0.920141 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.307911, + "std":0.084763, + "values":[ + 0.216409, + 0.323573, + 0.383751 + ] + }, + "validation_weighted_risk":{ + "mean":0.011627, + "std":0.001428, + "values":[ + 0.01093, + 0.010681, + 0.013269 + ] + }, + "test":{ + "coverage":{ + "mean":0.299758, + "std":0.08497, + "values":[ + 0.206727, + 0.319273, + 0.373273 + ] + }, + "selective_risk":{ + "mean":0.017365, + "std":0.006698, + "values":[ + 0.009675, + 0.020501, + 0.021919 + ] + }, + "in_scope_risk":{ + "mean":0.002425, + "std":0.002809, + "values":[ + 0.0, + 0.001773, + 0.005503 + ] + }, + "kept_oos_error_rate":{ + "mean":0.715085, + "std":0.253242, + "values":[ + 1.0, + 0.515625, + 0.62963 + ] + }, + "kept_oos_share":{ + "mean":0.024141, + "std":0.013516, + "values":[ + 0.009675, + 0.036446, + 0.026303 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.026, + "std":0.013, + "values":[ + 0.011, + 0.033, + 0.034 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "msp_t", + "msp", + "msp" + ], + "tau":{ + "mean":0.20164, + "std":0.286486, + "values":[ + 0.532444, + 0.03688, + 0.035594 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.945699, + "std":0.007443, + "values":[ + 0.954194, + 0.942581, + 0.940323 + ] + }, + "selective_risk":{ + "mean":0.043206, + "std":7.7e-05, + "values":[ + 0.043272, + 0.043121, + 0.043225 + ] + }, + "in_scope_risk":{ + "mean":0.033013, + "std":0.000572, + "values":[ + 0.033069, + 0.032415, + 0.033555 + ] + }, + "kept_oos_error_rate":{ + "mean":0.586672, + "std":0.033805, + "values":[ + 0.581818, + 0.622642, + 0.555556 + ] + }, + "kept_oos_share":{ + "mean":0.018419, + "std":0.000245, + "values":[ + 0.018594, + 0.018138, + 0.018525 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.316667, + "std":0.015275, + "values":[ + 0.32, + 0.33, + 0.3 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.877394, + "std":0.016022, + "values":[ + 0.895818, + 0.866727, + 0.869636 + ] + }, + "selective_risk":{ + "mean":0.105367, + "std":0.00341, + "values":[ + 0.102496, + 0.104468, + 0.109137 + ] + }, + "in_scope_risk":{ + "mean":0.035554, + "std":0.004465, + "values":[ + 0.030561, + 0.039161, + 0.036942 + ] + }, + "kept_oos_error_rate":{ + "mean":0.686049, + "std":0.036549, + "values":[ + 0.646957, + 0.691824, + 0.719368 + ] + }, + "kept_oos_share":{ + "mean":0.107519, + "std":0.008454, + "values":[ + 0.116704, + 0.100063, + 0.105791 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.355333, + "std":0.022301, + "values":[ + 0.372, + 0.33, + 0.364 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.942846, + "std":0.010103, + "values":[ + 0.954462, + 0.937968, + 0.936108 + ] + }, + "selective_risk":{ + "mean":0.047083, + "std":0.003968, + "values":[ + 0.042539, + 0.049868, + 0.048841 + ] + }, + "in_scope_risk":{ + "mean":0.035554, + "std":0.004465, + "values":[ + 0.030561, + 0.039161, + 0.036942 + ] + }, + "kept_oos_error_rate":{ + "mean":0.686049, + "std":0.036549, + "values":[ + 0.646957, + 0.691824, + 0.719368 + ] + }, + "kept_oos_share":{ + "mean":0.017758, + "std":0.00154, + "values":[ + 0.019433, + 0.016405, + 0.017437 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.355333, + "std":0.022301, + "values":[ + 0.372, + 0.33, + 0.364 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.939066, + "std":0.064687, + "values":[ + 1.01238, + 0.890028, + 0.914791 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.3558, + "std":0.51156, + "values":[ + 0.946481, + 0.064359, + 0.05656 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.645562, + "std":0.037423, + "values":[ + 0.602627, + 0.662804, + 0.671257 + ] + }, + "validation_weighted_risk":{ + "mean":0.04122, + "std":0.000482, + "values":[ + 0.040766, + 0.041169, + 0.041727 + ] + }, + "test":{ + "coverage":{ + "mean":0.630242, + "std":0.038068, + "values":[ + 0.587636, + 0.642182, + 0.660909 + ] + }, + "selective_risk":{ + "mean":0.038218, + "std":0.013405, + "values":[ + 0.023824, + 0.040487, + 0.050344 + ] + }, + "in_scope_risk":{ + "mean":0.014287, + "std":0.009329, + "values":[ + 0.004145, + 0.01621, + 0.022504 + ] + }, + "kept_oos_error_rate":{ + "mean":0.640354, + "std":0.023538, + "values":[ + 0.666667, + 0.633094, + 0.621302 + ] + }, + "kept_oos_share":{ + "mean":0.038517, + "std":0.008426, + "values":[ + 0.029703, + 0.039354, + 0.046492 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.085667, + "std":0.020599, + "values":[ + 0.064, + 0.088, + 0.105 + ] + } + } + } + } } } }, @@ -7320,6 +8712,28 @@ "std":13.868429 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.977891, + "std":0.004125 + }, + "auprc":{ + "mean":0.844796, + "std":0.011717 + } + }, + "test":{ + "auroc":{ + "mean":0.976626, + "std":0.000245 + }, + "auprc":{ + "mean":0.928686, + "std":0.00135 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -8518,6 +9932,28 @@ "std":14.843629 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.977837, + "std":0.006695 + }, + "auprc":{ + "mean":0.853635, + "std":0.013132 + } + }, + "test":{ + "auroc":{ + "mean":0.979647, + "std":0.000833 + }, + "auprc":{ + "mean":0.939713, + "std":0.002481 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -9882,6 +11318,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.976844, + "std":0.005901, + "values":[ + 0.979213, + 0.970127, + 0.981193 + ] + }, + "auprc":{ + "mean":0.848616, + "std":0.011157, + "values":[ + 0.836357, + 0.851317, + 0.858174 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.977419, + "std":0.001128, + "values":[ + 0.976768, + 0.978721, + 0.976767 + ] + }, + "auprc":{ + "mean":0.932007, + "std":0.004553, + "values":[ + 0.928848, + 0.937227, + 0.929948 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -10528,6 +12006,600 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":[ + "entropy", + "msp", + "entropy" + ], + "tau":{ + "mean":-1.339347, + "std":1.764824, + "values":[ + -2.465052, + 0.694625, + -2.247615 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.988387, + "std":0.000645, + "values":[ + 0.988387, + 0.987742, + 0.989032 + ] + }, + "selective_risk":{ + "mean":0.015666, + "std":1e-05, + "values":[ + 0.015666, + 0.015676, + 0.015656 + ] + }, + "in_scope_risk":{ + "mean":0.008846, + "std":0.000839, + "values":[ + 0.008728, + 0.008073, + 0.009738 + ] + }, + "kept_oos_error_rate":{ + "mean":0.248132, + "std":0.028427, + "values":[ + 0.258824, + 0.269663, + 0.215909 + ] + }, + "kept_oos_share":{ + "mean":0.028503, + "std":0.000684, + "values":[ + 0.027742, + 0.029066, + 0.028702 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.216667, + "std":0.025166, + "values":[ + 0.22, + 0.24, + 0.19 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.951939, + "std":0.009553, + "values":[ + 0.961273, + 0.942182, + 0.952364 + ] + }, + "selective_risk":{ + "mean":0.059781, + "std":0.001027, + "values":[ + 0.060526, + 0.060208, + 0.05861 + ] + }, + "in_scope_risk":{ + "mean":0.009032, + "std":0.001108, + "values":[ + 0.00849, + 0.0083, + 0.010307 + ] + }, + "kept_oos_error_rate":{ + "mean":0.354776, + "std":0.022381, + "values":[ + 0.347719, + 0.379834, + 0.336774 + ] + }, + "kept_oos_share":{ + "mean":0.147022, + "std":0.006888, + "values":[ + 0.153395, + 0.139714, + 0.147957 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.272667, + "std":0.010693, + "values":[ + 0.282, + 0.275, + 0.261 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.985197, + "std":0.003358, + "values":[ + 0.988742, + 0.982065, + 0.984785 + ] + }, + "selective_risk":{ + "mean":0.017732, + "std":0.000765, + "values":[ + 0.017465, + 0.017135, + 0.018595 + ] + }, + "in_scope_risk":{ + "mean":0.009032, + "std":0.001108, + "values":[ + 0.00849, + 0.0083, + 0.010307 + ] + }, + "kept_oos_error_rate":{ + "mean":0.354776, + "std":0.022381, + "values":[ + 0.347719, + 0.379834, + 0.336774 + ] + }, + "kept_oos_share":{ + "mean":0.025209, + "std":0.001348, + "values":[ + 0.026459, + 0.023781, + 0.025386 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.272667, + "std":0.010693, + "values":[ + 0.282, + 0.275, + 0.261 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.952896, + "std":0.018827, + "values":[ + 0.959883, + 0.967232, + 0.931575 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":-0.097541, + "std":0.939643, + "values":[ + -0.56666, + 0.984293, + -0.710255 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.866839, + "std":0.046672, + "values":[ + 0.888481, + 0.813274, + 0.898761 + ] + }, + "validation_weighted_risk":{ + "mean":0.014253, + "std":0.000584, + "values":[ + 0.013618, + 0.014768, + 0.014372 + ] + }, + "test":{ + "coverage":{ + "mean":0.841576, + "std":0.050338, + "values":[ + 0.863455, + 0.784, + 0.877273 + ] + }, + "selective_risk":{ + "mean":0.021473, + "std":0.008736, + "values":[ + 0.024637, + 0.011596, + 0.028187 + ] + }, + "in_scope_risk":{ + "mean":0.002906, + "std":0.001234, + "values":[ + 0.00351, + 0.001486, + 0.003722 + ] + }, + "kept_oos_error_rate":{ + "mean":0.201153, + "std":0.035767, + "values":[ + 0.214737, + 0.160584, + 0.228137 + ] + }, + "kept_oos_share":{ + "mean":0.09086, + "std":0.02408, + "values":[ + 0.100021, + 0.063544, + 0.109016 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.088667, + "std":0.039716, + "values":[ + 0.102, + 0.044, + 0.12 + ] + } + } + } + }, + "0.05":{ + "signal":"entropy", + "tau":{ + "mean":-3.416748, + "std":1.319133, + "values":[ + -4.129125, + -1.894583, + -4.226536 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.022151, + "std":0.001304, + "values":[ + 0.021935, + 0.023548, + 0.020968 + ] + }, + "in_scope_risk":{ + "mean":0.012889, + "std":0.001018, + "values":[ + 0.012, + 0.014, + 0.012667 + ] + }, + "kept_oos_error_rate":{ + "mean":0.3, + "std":0.026458, + "values":[ + 0.32, + 0.31, + 0.27 + ] + }, + "kept_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.3, + "std":0.026458, + "values":[ + 0.32, + 0.31, + 0.27 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.00042, + "values":[ + 0.999818, + 0.999091, + 0.999818 + ] + }, + "selective_risk":{ + "mean":0.089069, + "std":0.004948, + "values":[ + 0.084015, + 0.093904, + 0.089289 + ] + }, + "in_scope_risk":{ + "mean":0.011704, + "std":0.001446, + "values":[ + 0.010222, + 0.011778, + 0.013111 + ] + }, + "kept_oos_error_rate":{ + "mean":0.438058, + "std":0.024936, + "values":[ + 0.416416, + 0.465327, + 0.432432 + ] + }, + "kept_oos_share":{ + "mean":0.181471, + "std":0.000344, + "values":[ + 0.181669, + 0.181074, + 0.181669 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.437, + "std":0.023896, + "values":[ + 0.416, + 0.463, + 0.432 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.999925, + "std":7.4e-05, + "values":[ + 0.999968, + 0.999839, + 0.999968 + ] + }, + "selective_risk":{ + "mean":0.025425, + "std":0.001835, + "values":[ + 0.023313, + 0.026338, + 0.026625 + ] + }, + "in_scope_risk":{ + "mean":0.011704, + "std":0.001446, + "values":[ + 0.010222, + 0.011778, + 0.013111 + ] + }, + "kept_oos_error_rate":{ + "mean":0.438058, + "std":0.024936, + "values":[ + 0.416416, + 0.465327, + 0.432432 + ] + }, + "kept_oos_share":{ + "mean":0.032185, + "std":7.2e-05, + "values":[ + 0.032227, + 0.032102, + 0.032227 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.437, + "std":0.023896, + "values":[ + 0.416, + 0.463, + 0.432 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.951792, + "std":0.031201, + "values":[ + 0.977817, + 0.960356, + 0.917203 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":-1.602045, + "std":0.64528, + "values":[ + -1.857494, + -0.868148, + -2.080493 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.962093, + "std":0.008978, + "values":[ + 0.953784, + 0.96088, + 0.971616 + ] + }, + "validation_weighted_risk":{ + "mean":0.043121, + "std":0.000325, + "values":[ + 0.043117, + 0.042798, + 0.043448 + ] + }, + "test":{ + "coverage":{ + "mean":0.937818, + "std":0.008092, + "values":[ + 0.938, + 0.929636, + 0.945818 + ] + }, + "selective_risk":{ + "mean":0.051499, + "std":0.003701, + "values":[ + 0.04749, + 0.05222, + 0.054787 + ] + }, + "in_scope_risk":{ + "mean":0.00809, + "std":0.001247, + "values":[ + 0.006746, + 0.008315, + 0.009209 + ] + }, + "kept_oos_error_rate":{ + "mean":0.324736, + "std":0.022477, + "values":[ + 0.301966, + 0.346908, + 0.325333 + ] + }, + "kept_oos_share":{ + "mean":0.137285, + "std":0.00728, + "values":[ + 0.138011, + 0.129669, + 0.144175 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.229667, + "std":0.014503, + "values":[ + 0.215, + 0.23, + 0.244 + ] + } + } + } + } } } }, @@ -10782,6 +12854,28 @@ "std":14.0119 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.944846, + "std":0.009175 + }, + "auprc":{ + "mean":0.659888, + "std":0.061525 + } + }, + "test":{ + "auroc":{ + "mean":0.925392, + "std":0.004929 + }, + "auprc":{ + "mean":0.809191, + "std":0.012645 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -11976,6 +14070,28 @@ "std":22.501852 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.941711, + "std":0.014413 + }, + "auprc":{ + "mean":0.675329, + "std":0.07298 + } + }, + "test":{ + "auroc":{ + "mean":0.942959, + "std":0.005489 + }, + "auprc":{ + "mean":0.840933, + "std":0.020449 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -13336,6 +15452,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.944846, + "std":0.009175, + "values":[ + 0.938303, + 0.9409, + 0.955333 + ] + }, + "auprc":{ + "mean":0.659888, + "std":0.061525, + "values":[ + 0.699873, + 0.58904, + 0.69075 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.925392, + "std":0.004929, + "values":[ + 0.92949, + 0.919922, + 0.926764 + ] + }, + "auprc":{ + "mean":0.809191, + "std":0.012645, + "values":[ + 0.819196, + 0.794978, + 0.813398 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -13978,6 +16136,596 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":"msp", + "tau":{ + "mean":0.113052, + "std":0.020931, + "values":[ + 0.118972, + 0.130385, + 0.089798 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.920753, + "std":0.02376, + "values":[ + 0.917097, + 0.899032, + 0.946129 + ] + }, + "selective_risk":{ + "mean":0.01553, + "std":0.000135, + "values":[ + 0.015477, + 0.015429, + 0.015684 + ] + }, + "in_scope_risk":{ + "mean":0.011287, + "std":0.000646, + "values":[ + 0.011103, + 0.012004, + 0.010753 + ] + }, + "kept_oos_error_rate":{ + "mean":0.272687, + "std":0.024012, + "values":[ + 0.254902, + 0.263158, + 0.3 + ] + }, + "kept_oos_share":{ + "mean":0.016207, + "std":0.002272, + "values":[ + 0.017939, + 0.013635, + 0.017047 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.126667, + "std":0.025166, + "values":[ + 0.13, + 0.1, + 0.15 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.833091, + "std":0.038152, + "values":[ + 0.824182, + 0.800182, + 0.874909 + ] + }, + "selective_risk":{ + "mean":0.047798, + "std":0.010293, + "values":[ + 0.041032, + 0.042718, + 0.059643 + ] + }, + "in_scope_risk":{ + "mean":0.012558, + "std":0.00219, + "values":[ + 0.013072, + 0.014447, + 0.010157 + ] + }, + "kept_oos_error_rate":{ + "mean":0.413849, + "std":0.089147, + "values":[ + 0.328358, + 0.40694, + 0.50625 + ] + }, + "kept_oos_share":{ + "mean":0.086821, + "std":0.013954, + "values":[ + 0.088683, + 0.072029, + 0.099751 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.168, + "std":0.064969, + "values":[ + 0.132, + 0.129, + 0.243 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.912319, + "std":0.030796, + "values":[ + 0.901355, + 0.888505, + 0.947097 + ] + }, + "selective_risk":{ + "mean":0.01828, + "std":0.000678, + "values":[ + 0.017608, + 0.018964, + 0.018267 + ] + }, + "in_scope_risk":{ + "mean":0.012558, + "std":0.00219, + "values":[ + 0.013072, + 0.014447, + 0.010157 + ] + }, + "kept_oos_error_rate":{ + "mean":0.413849, + "std":0.089147, + "values":[ + 0.328358, + 0.40694, + 0.50625 + ] + }, + "kept_oos_share":{ + "mean":0.014082, + "std":0.002434, + "values":[ + 0.014387, + 0.011509, + 0.016349 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.168, + "std":0.064969, + "values":[ + 0.132, + 0.129, + 0.243 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.909427, + "std":0.035923, + "values":[ + 0.916602, + 0.870458, + 0.941221 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.216104, + "std":0.084381, + "values":[ + 0.304938, + 0.20635, + 0.137023 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.569188, + "std":0.247208, + "values":[ + 0.305045, + 0.607541, + 0.794979 + ] + }, + "validation_weighted_risk":{ + "mean":0.013487, + "std":0.001347, + "values":[ + 0.011933, + 0.014227, + 0.014302 + ] + }, + "test":{ + "coverage":{ + "mean":0.561394, + "std":0.241186, + "values":[ + 0.305636, + 0.593818, + 0.784727 + ] + }, + "selective_risk":{ + "mean":0.019632, + "std":0.012344, + "values":[ + 0.008923, + 0.01684, + 0.033133 + ] + }, + "in_scope_risk":{ + "mean":0.003849, + "std":0.003085, + "values":[ + 0.0019, + 0.002242, + 0.007406 + ] + }, + "kept_oos_error_rate":{ + "mean":0.292465, + "std":0.158389, + "values":[ + 0.117647, + 0.333333, + 0.426415 + ] + }, + "kept_oos_share":{ + "mean":0.055389, + "std":0.009792, + "values":[ + 0.060678, + 0.044091, + 0.061399 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.057667, + "std":0.051189, + "values":[ + 0.012, + 0.048, + 0.113 + ] + } + } + } + }, + "0.05":{ + "signal":"msp", + "tau":{ + "mean":0.023213, + "std":0.001684, + "values":[ + 0.025006, + 0.021665, + 0.022969 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.039785, + "std":0.004938, + "values":[ + 0.041613, + 0.043548, + 0.034194 + ] + }, + "in_scope_risk":{ + "mean":0.025222, + "std":0.004526, + "values":[ + 0.028, + 0.027667, + 0.02 + ] + }, + "kept_oos_error_rate":{ + "mean":0.476667, + "std":0.037859, + "values":[ + 0.45, + 0.52, + 0.46 + ] + }, + "kept_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.476667, + "std":0.037859, + "values":[ + 0.45, + 0.52, + 0.46 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.999576, + "std":0.000278, + "values":[ + 0.999273, + 0.999818, + 0.999636 + ] + }, + "selective_risk":{ + "mean":0.13448, + "std":0.006697, + "values":[ + 0.127911, + 0.141298, + 0.134231 + ] + }, + "in_scope_risk":{ + "mean":0.027037, + "std":0.006334, + "values":[ + 0.027111, + 0.033333, + 0.020667 + ] + }, + "kept_oos_error_rate":{ + "mean":0.619085, + "std":0.032337, + "values":[ + 0.583333, + 0.627628, + 0.646293 + ] + }, + "kept_oos_share":{ + "mean":0.181471, + "std":0.000227, + "values":[ + 0.181223, + 0.181669, + 0.181521 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.617667, + "std":0.033005, + "values":[ + 0.581, + 0.627, + 0.645 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.999925, + "std":4.9e-05, + "values":[ + 0.999871, + 0.999968, + 0.999935 + ] + }, + "selective_risk":{ + "mean":0.046093, + "std":0.005917, + "values":[ + 0.044984, + 0.052486, + 0.040809 + ] + }, + "in_scope_risk":{ + "mean":0.027037, + "std":0.006334, + "values":[ + 0.027111, + 0.033333, + 0.020667 + ] + }, + "kept_oos_error_rate":{ + "mean":0.619085, + "std":0.032337, + "values":[ + 0.583333, + 0.627628, + 0.646293 + ] + }, + "kept_oos_share":{ + "mean":0.032185, + "std":4.8e-05, + "values":[ + 0.032133, + 0.032227, + 0.032196 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.617667, + "std":0.033005, + "values":[ + 0.581, + 0.627, + 0.645 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.934458, + "std":0.026186, + "values":[ + 0.960933, + 0.908571, + 0.933869 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.10549, + "std":0.016969, + "values":[ + 0.108044, + 0.121037, + 0.08739 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.866041, + "std":0.022301, + "values":[ + 0.877207, + 0.840363, + 0.880554 + ] + }, + "validation_weighted_risk":{ + "mean":0.042651, + "std":0.000563, + "values":[ + 0.042001, + 0.042974, + 0.042979 + ] + }, + "test":{ + "coverage":{ + "mean":0.848242, + "std":0.032727, + "values":[ + 0.848364, + 0.815455, + 0.880909 + ] + }, + "selective_risk":{ + "mean":0.052173, + "std":0.00864, + "values":[ + 0.047793, + 0.0466, + 0.062126 + ] + }, + "in_scope_risk":{ + "mean":0.013357, + "std":0.002508, + "values":[ + 0.014028, + 0.015463, + 0.010582 + ] + }, + "kept_oos_error_rate":{ + "mean":0.429215, + "std":0.078257, + "values":[ + 0.356522, + 0.419075, + 0.512048 + ] + }, + "kept_oos_share":{ + "mean":0.092839, + "std":0.013752, + "values":[ + 0.098586, + 0.077146, + 0.102786 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.188, + "std":0.058796, + "values":[ + 0.164, + 0.145, + 0.255 + ] + } + } + } + } } } }, @@ -14232,6 +16980,28 @@ "std":116.328558 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.90777, + "std":0.015768 + }, + "auprc":{ + "mean":0.421829, + "std":0.05118 + } + }, + "test":{ + "auroc":{ + "mean":0.886989, + "std":0.019735 + }, + "auprc":{ + "mean":0.670025, + "std":0.06416 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -15430,6 +18200,28 @@ "std":95.212044 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.89945, + "std":0.012074 + }, + "auprc":{ + "mean":0.412981, + "std":0.052366 + } + }, + "test":{ + "auroc":{ + "mean":0.900985, + "std":0.018356 + }, + "auprc":{ + "mean":0.692587, + "std":0.060012 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -15753,7 +18545,7 @@ "entropy":{ "tau":{ "mean":-1.656994, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -15767,12 +18559,12 @@ }, "validation_selective_risk":{ "mean":0.007752, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.019237, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -15782,7 +18574,7 @@ }, "selective_risk":{ "mean":0.043706, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -16810,6 +19602,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.89945, + "std":0.012074, + "values":[ + 0.908087, + 0.885653, + 0.90461 + ] + }, + "auprc":{ + "mean":0.412981, + "std":0.052366, + "values":[ + 0.4683, + 0.406466, + 0.364178 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.900985, + "std":0.018356, + "values":[ + 0.921975, + 0.887943, + 0.893036 + ] + }, + "auprc":{ + "mean":0.692587, + "std":0.060012, + "values":[ + 0.760124, + 0.672254, + 0.645382 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -17456,6 +20290,600 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":[ + "msp_t", + "msp_t", + "margin" + ], + "tau":{ + "mean":1.460889, + "std":0.979203, + "values":[ + 0.879145, + 0.912108, + 2.591415 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.668065, + "std":0.014061, + "values":[ + 0.674516, + 0.651935, + 0.677742 + ] + }, + "selective_risk":{ + "mean":0.014808, + "std":4.7e-05, + "values":[ + 0.014825, + 0.014844, + 0.014755 + ] + }, + "in_scope_risk":{ + "mean":0.009897, + "std":0.000636, + "values":[ + 0.010628, + 0.009472, + 0.009592 + ] + }, + "kept_oos_error_rate":{ + "mean":0.616468, + "std":0.164329, + "values":[ + 0.428571, + 0.733333, + 0.6875 + ] + }, + "kept_oos_share":{ + "mean":0.00836, + "std":0.001461, + "values":[ + 0.010043, + 0.007422, + 0.007615 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.103333, + "std":0.011547, + "values":[ + 0.09, + 0.11, + 0.11 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.581636, + "std":0.008638, + "values":[ + 0.590182, + 0.572909, + 0.581818 + ] + }, + "selective_risk":{ + "mean":0.034648, + "std":0.009893, + "values":[ + 0.024338, + 0.035544, + 0.044062 + ] + }, + "in_scope_risk":{ + "mean":0.010056, + "std":0.001308, + "values":[ + 0.008772, + 0.01001, + 0.011386 + ] + }, + "kept_oos_error_rate":{ + "mean":0.561087, + "std":0.267026, + "values":[ + 0.309524, + 0.532468, + 0.84127 + ] + }, + "kept_oos_share":{ + "mean":0.046668, + "std":0.006478, + "values":[ + 0.051756, + 0.048873, + 0.039375 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.08, + "std":0.027055, + "values":[ + 0.052, + 0.082, + 0.106 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.660659, + "std":0.009742, + "values":[ + 0.667355, + 0.649484, + 0.66514 + ] + }, + "selective_risk":{ + "mean":0.013892, + "std":0.002623, + "values":[ + 0.011214, + 0.014006, + 0.016457 + ] + }, + "in_scope_risk":{ + "mean":0.010056, + "std":0.001308, + "values":[ + 0.008772, + 0.01001, + 0.011386 + ] + }, + "kept_oos_error_rate":{ + "mean":0.561087, + "std":0.267026, + "values":[ + 0.309524, + 0.532468, + 0.84127 + ] + }, + "kept_oos_share":{ + "mean":0.007293, + "std":0.001051, + "values":[ + 0.008121, + 0.007649, + 0.006111 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.08, + "std":0.027055, + "values":[ + 0.052, + 0.082, + 0.106 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":1.120681, + "std":0.229616, + "values":[ + 1.37964, + 1.040482, + 0.941921 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":2.216768, + "std":2.140198, + "values":[ + 0.965718, + 0.996595, + 4.687991 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.2352, + "std":0.119814, + "values":[ + 0.354037, + 0.114431, + 0.237131 + ] + }, + "validation_weighted_risk":{ + "mean":0.007906, + "std":0.006886, + "values":[ + 0.012592, + 0.0, + 0.011125 + ] + }, + "test":{ + "coverage":{ + "mean":0.233515, + "std":0.114225, + "values":[ + 0.341091, + 0.113636, + 0.245818 + ] + }, + "selective_risk":{ + "mean":0.01204, + "std":0.002611, + "values":[ + 0.011727, + 0.0096, + 0.014793 + ] + }, + "in_scope_risk":{ + "mean":0.003034, + "std":0.001426, + "values":[ + 0.004479, + 0.001629, + 0.002994 + ] + }, + "kept_oos_error_rate":{ + "mean":0.5367, + "std":0.428175, + "values":[ + 0.155556, + 0.454545, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.025803, + "std":0.019416, + "values":[ + 0.047974, + 0.0176, + 0.011834 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.011667, + "std":0.005859, + "values":[ + 0.014, + 0.005, + 0.016 + ] + } + } + } + }, + "0.05":{ + "signal":"msp_t", + "tau":{ + "mean":0.600529, + "std":0.015314, + "values":[ + 0.585596, + 0.616197, + 0.599794 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.891075, + "std":0.004701, + "values":[ + 0.895484, + 0.886129, + 0.891613 + ] + }, + "selective_risk":{ + "mean":0.042959, + "std":9.3e-05, + "values":[ + 0.042867, + 0.042956, + 0.043054 + ] + }, + "in_scope_risk":{ + "mean":0.033707, + "std":0.000655, + "values":[ + 0.034091, + 0.032951, + 0.034078 + ] + }, + "kept_oos_error_rate":{ + "mean":0.63832, + "std":0.100827, + "values":[ + 0.541667, + 0.630435, + 0.742857 + ] + }, + "kept_oos_share":{ + "mean":0.015566, + "std":0.002529, + "values":[ + 0.017291, + 0.016746, + 0.012663 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.27, + "std":0.017321, + "values":[ + 0.26, + 0.29, + 0.26 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.814606, + "std":0.010941, + "values":[ + 0.826, + 0.813636, + 0.804182 + ] + }, + "selective_risk":{ + "mean":0.101138, + "std":0.012195, + "values":[ + 0.087167, + 0.106592, + 0.109654 + ] + }, + "in_scope_risk":{ + "mean":0.037519, + "std":0.002933, + "values":[ + 0.03434, + 0.038098, + 0.040119 + ] + }, + "kept_oos_error_rate":{ + "mean":0.709456, + "std":0.127756, + "values":[ + 0.583524, + 0.705882, + 0.838961 + ] + }, + "kept_oos_share":{ + "mean":0.095269, + "std":0.007803, + "values":[ + 0.096192, + 0.10257, + 0.087045 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.300667, + "std":0.039552, + "values":[ + 0.255, + 0.324, + 0.323 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.885459, + "std":0.010156, + "values":[ + 0.897108, + 0.878462, + 0.880806 + ] + }, + "selective_risk":{ + "mean":0.047902, + "std":0.00439, + "values":[ + 0.04297, + 0.049353, + 0.051383 + ] + }, + "in_scope_risk":{ + "mean":0.037519, + "std":0.002933, + "values":[ + 0.03434, + 0.038098, + 0.040119 + ] + }, + "kept_oos_error_rate":{ + "mean":0.709456, + "std":0.127756, + "values":[ + 0.583524, + 0.705882, + 0.838961 + ] + }, + "kept_oos_share":{ + "mean":0.015556, + "std":0.001384, + "values":[ + 0.015714, + 0.016855, + 0.0141 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.300667, + "std":0.039552, + "values":[ + 0.255, + 0.324, + 0.323 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.924036, + "std":0.064958, + "values":[ + 0.997693, + 0.899473, + 0.874941 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.859032, + "std":0.062143, + "values":[ + 0.8147, + 0.930063, + 0.832334 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.620538, + "std":0.07115, + "values":[ + 0.678632, + 0.541181, + 0.641802 + ] + }, + "validation_weighted_risk":{ + "mean":0.04098, + "std":0.000257, + "values":[ + 0.041152, + 0.040684, + 0.041103 + ] + }, + "test":{ + "coverage":{ + "mean":0.611152, + "std":0.064828, + "values":[ + 0.665636, + 0.539455, + 0.628364 + ] + }, + "selective_risk":{ + "mean":0.036818, + "std":0.008678, + "values":[ + 0.033871, + 0.029997, + 0.046586 + ] + }, + "in_scope_risk":{ + "mean":0.011664, + "std":0.003688, + "values":[ + 0.012192, + 0.007741, + 0.01506 + ] + }, + "kept_oos_error_rate":{ + "mean":0.577269, + "std":0.22118, + "values":[ + 0.37963, + 0.536, + 0.816176 + ] + }, + "kept_oos_share":{ + "mean":0.046827, + "std":0.010633, + "values":[ + 0.059, + 0.04213, + 0.039352 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.086667, + "std":0.022368, + "values":[ + 0.082, + 0.067, + 0.111 + ] + } + } + } + } } } }, @@ -17714,6 +21142,28 @@ "std":10.503968 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.97016, + "std":0.005392 + }, + "auprc":{ + "mean":0.768718, + "std":0.011253 + } + }, + "test":{ + "auroc":{ + "mean":0.962126, + "std":0.002305 + }, + "auprc":{ + "mean":0.896537, + "std":0.003028 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -18912,6 +22362,28 @@ "std":7.81025 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.966179, + "std":0.010153 + }, + "auprc":{ + "mean":0.769589, + "std":0.010021 + } + }, + "test":{ + "auroc":{ + "mean":0.970209, + "std":0.001938 + }, + "auprc":{ + "mean":0.915883, + "std":0.00234 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -20280,6 +23752,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.966259, + "std":0.010244, + "values":[ + 0.970017, + 0.954667, + 0.974093 + ] + }, + "auprc":{ + "mean":0.772972, + "std":0.011861, + "values":[ + 0.759289, + 0.779307, + 0.780321 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.967819, + "std":0.002954, + "values":[ + 0.970625, + 0.968096, + 0.964736 + ] + }, + "auprc":{ + "mean":0.908901, + "std":0.010064, + "values":[ + 0.913614, + 0.915745, + 0.897345 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -20930,6 +24444,604 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":[ + "entropy", + "msp_t", + "entropy" + ], + "tau":{ + "mean":-1.18306, + "std":1.810917, + "values":[ + -1.318183, + 0.691634, + -2.922631 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.975054, + "std":0.002266, + "values":[ + 0.974839, + 0.977419, + 0.972903 + ] + }, + "selective_risk":{ + "mean":0.015549, + "std":3.6e-05, + "values":[ + 0.015553, + 0.015512, + 0.015584 + ] + }, + "in_scope_risk":{ + "mean":0.008244, + "std":0.001427, + "values":[ + 0.007786, + 0.007102, + 0.009844 + ] + }, + "kept_oos_error_rate":{ + "mean":0.322083, + "std":0.056263, + "values":[ + 0.352941, + 0.356164, + 0.257143 + ] + }, + "kept_oos_share":{ + "mean":0.023268, + "std":0.000797, + "values":[ + 0.022502, + 0.024092, + 0.02321 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.226667, + "std":0.041633, + "values":[ + 0.24, + 0.26, + 0.18 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.914848, + "std":0.015874, + "values":[ + 0.907273, + 0.933091, + 0.904182 + ] + }, + "selective_risk":{ + "mean":0.058597, + "std":0.008491, + "values":[ + 0.055511, + 0.0682, + 0.052081 + ] + }, + "in_scope_risk":{ + "mean":0.008502, + "std":0.000479, + "values":[ + 0.008786, + 0.008772, + 0.007949 + ] + }, + "kept_oos_error_rate":{ + "mean":0.426092, + "std":0.030607, + "values":[ + 0.431942, + 0.453353, + 0.392982 + ] + }, + "kept_oos_share":{ + "mean":0.11957, + "std":0.012391, + "values":[ + 0.110421, + 0.133671, + 0.114619 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.257667, + "std":0.046715, + "values":[ + 0.238, + 0.311, + 0.224 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.971975, + "std":0.006505, + "values":[ + 0.972398, + 0.978258, + 0.965269 + ] + }, + "selective_risk":{ + "mean":0.016878, + "std":0.001799, + "values":[ + 0.016521, + 0.018829, + 0.015284 + ] + }, + "in_scope_risk":{ + "mean":0.008502, + "std":0.000479, + "values":[ + 0.008786, + 0.008772, + 0.007949 + ] + }, + "kept_oos_error_rate":{ + "mean":0.426092, + "std":0.030607, + "values":[ + 0.431942, + 0.453353, + 0.392982 + ] + }, + "kept_oos_share":{ + "mean":0.019983, + "std":0.002317, + "values":[ + 0.018279, + 0.022621, + 0.019049 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.257667, + "std":0.046715, + "values":[ + 0.238, + 0.311, + 0.224 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.97368, + "std":0.035636, + "values":[ + 0.975777, + 0.937041, + 1.008221 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":-0.444952, + "std":1.42457, + "values":[ + -0.469773, + 0.991866, + -1.856949 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.712088, + "std":0.147527, + "values":[ + 0.733533, + 0.555012, + 0.847719 + ] + }, + "validation_weighted_risk":{ + "mean":0.014132, + "std":0.000953, + "values":[ + 0.015008, + 0.013117, + 0.014273 + ] + }, + "test":{ + "coverage":{ + "mean":0.690242, + "std":0.153996, + "values":[ + 0.713818, + 0.525818, + 0.831091 + ] + }, + "selective_risk":{ + "mean":0.015332, + "std":0.010066, + "values":[ + 0.010443, + 0.008645, + 0.026909 + ] + }, + "in_scope_risk":{ + "mean":0.001993, + "std":0.00136, + "values":[ + 0.001343, + 0.001081, + 0.003556 + ] + }, + "kept_oos_error_rate":{ + "mean":0.224607, + "std":0.070676, + "values":[ + 0.178218, + 0.189655, + 0.305949 + ] + }, + "kept_oos_share":{ + "mean":0.056263, + "std":0.01902, + "values":[ + 0.051452, + 0.040111, + 0.077226 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.055333, + "std":0.046145, + "values":[ + 0.036, + 0.022, + 0.108 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "tau":{ + "mean":-0.537914, + "std":1.219665, + "values":[ + -1.944545, + 0.225611, + 0.105192 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.029032, + "std":0.001613, + "values":[ + 0.029032, + 0.027419, + 0.030645 + ] + }, + "in_scope_risk":{ + "mean":0.015111, + "std":0.001925, + "values":[ + 0.014, + 0.014, + 0.017333 + ] + }, + "kept_oos_error_rate":{ + "mean":0.446667, + "std":0.028868, + "values":[ + 0.48, + 0.43, + 0.43 + ] + }, + "kept_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.446667, + "std":0.028868, + "values":[ + 0.48, + 0.43, + 0.43 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.997697, + "std":0.002044, + "values":[ + 0.995455, + 0.999455, + 0.998182 + ] + }, + "selective_risk":{ + "mean":0.110196, + "std":0.002169, + "values":[ + 0.112694, + 0.108787, + 0.109107 + ] + }, + "in_scope_risk":{ + "mean":0.013632, + "std":0.000338, + "values":[ + 0.014, + 0.013559, + 0.013336 + ] + }, + "kept_oos_error_rate":{ + "mean":0.550059, + "std":0.015982, + "values":[ + 0.568205, + 0.538076, + 0.543895 + ] + }, + "kept_oos_share":{ + "mean":0.180049, + "std":0.001781, + "values":[ + 0.178082, + 0.181554, + 0.18051 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.543333, + "std":0.009292, + "values":[ + 0.554, + 0.537, + 0.539 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.99947, + "std":0.000264, + "values":[ + 0.999194, + 0.99972, + 0.999495 + ] + }, + "selective_risk":{ + "mean":0.030733, + "std":0.00062, + "values":[ + 0.031445, + 0.030449, + 0.030306 + ] + }, + "in_scope_risk":{ + "mean":0.013632, + "std":0.000338, + "values":[ + 0.014, + 0.013559, + 0.013336 + ] + }, + "kept_oos_error_rate":{ + "mean":0.550059, + "std":0.015982, + "values":[ + 0.568205, + 0.538076, + 0.543895 + ] + }, + "kept_oos_share":{ + "mean":0.031888, + "std":0.000372, + "values":[ + 0.031477, + 0.032203, + 0.031984 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.543333, + "std":0.009292, + "values":[ + 0.554, + 0.537, + 0.539 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.979418, + "std":0.021978, + "values":[ + 0.971164, + 0.962761, + 1.004327 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.100221, + "std":1.05829, + "values":[ + -1.108145, + 0.862094, + 0.546715 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.911481, + "std":0.009812, + "values":[ + 0.903655, + 0.908297, + 0.922489 + ] + }, + "validation_weighted_risk":{ + "mean":0.042493, + "std":0.000527, + "values":[ + 0.043095, + 0.042274, + 0.042111 + ] + }, + "test":{ + "coverage":{ + "mean":0.897273, + "std":0.016461, + "values":[ + 0.885091, + 0.890727, + 0.916 + ] + }, + "selective_risk":{ + "mean":0.050902, + "std":0.00775, + "values":[ + 0.044577, + 0.048581, + 0.059547 + ] + }, + "in_scope_risk":{ + "mean":0.006758, + "std":0.001448, + "values":[ + 0.005912, + 0.005933, + 0.00843 + ] + }, + "kept_oos_error_rate":{ + "mean":0.407227, + "std":0.002518, + "values":[ + 0.406383, + 0.410058, + 0.405239 + ] + }, + "kept_oos_share":{ + "mean":0.110301, + "std":0.016656, + "values":[ + 0.096549, + 0.105532, + 0.128821 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.222, + "std":0.037027, + "values":[ + 0.191, + 0.212, + 0.263 + ] + } + } + } + } } } }, @@ -21130,6 +25242,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -21293,6 +25409,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -21622,6 +25742,10 @@ ] } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -21833,6 +25957,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -21996,6 +26124,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -22325,6 +26457,10 @@ ] } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -22536,6 +26672,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -22699,6 +26839,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -23028,6 +27172,10 @@ ] } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -23239,6 +27387,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -23402,6 +27554,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -23731,6 +27887,10 @@ ] } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -23942,6 +28102,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -24105,6 +28269,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -24434,6 +28602,10 @@ ] } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -24645,6 +28817,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -24808,6 +28984,10 @@ "std":0.0 } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -25137,6 +29317,10 @@ ] } }, + "oos_detection":{ + "validation":null, + "test":null + }, "signals":{}, "fallback":{ "0.02":{ @@ -25406,6 +29590,28 @@ "std":1.154701 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.977906, + "std":0.003772 + }, + "auprc":{ + "mean":0.737454, + "std":0.051905 + } + }, + "test":{ + "auroc":{ + "mean":0.977988, + "std":0.00044 + }, + "auprc":{ + "mean":0.915221, + "std":0.002638 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -26608,6 +30814,28 @@ "std":2.516611 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.980386, + "std":0.002221 + }, + "auprc":{ + "mean":0.769788, + "std":0.026091 + } + }, + "test":{ + "auroc":{ + "mean":0.980798, + "std":0.000585 + }, + "auprc":{ + "mean":0.925321, + "std":0.003309 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -27972,6 +32200,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.979147, + "std":0.003792, + "values":[ + 0.98295, + 0.979123, + 0.975367 + ] + }, + "auprc":{ + "mean":0.751084, + "std":0.05669, + "values":[ + 0.795516, + 0.7705, + 0.687237 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.979811, + "std":0.001238, + "values":[ + 0.980798, + 0.980212, + 0.978422 + ] + }, + "auprc":{ + "mean":0.922028, + "std":0.004238, + "values":[ + 0.926467, + 0.921591, + 0.918025 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -28618,6 +32888,600 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":"entropy", + "tau":{ + "mean":-1.1938, + "std":0.440804, + "values":[ + -0.959934, + -0.919213, + -1.702253 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.971613, + "std":0.002115, + "values":[ + 0.973548, + 0.971935, + 0.969355 + ] + }, + "selective_risk":{ + "mean":0.015604, + "std":3.4e-05, + "values":[ + 0.015573, + 0.015599, + 0.015641 + ] + }, + "in_scope_risk":{ + "mean":0.0047, + "std":0.000341, + "values":[ + 0.00469, + 0.004364, + 0.005045 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.010956, + "std":0.000318, + "values":[ + 0.010934, + 0.011284, + 0.010649 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.33, + "std":0.01, + "values":[ + 0.33, + 0.34, + 0.32 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.879273, + "std":0.007264, + "values":[ + 0.885455, + 0.881091, + 0.871273 + ] + }, + "selective_risk":{ + "mean":0.08475, + "std":0.005611, + "values":[ + 0.089733, + 0.085844, + 0.078673 + ] + }, + "in_scope_risk":{ + "mean":0.007474, + "std":0.000326, + "values":[ + 0.007833, + 0.007394, + 0.007196 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.077858, + "std":0.005373, + "values":[ + 0.082546, + 0.079034, + 0.071995 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.376667, + "std":0.029023, + "values":[ + 0.402, + 0.383, + 0.345 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.971147, + "std":0.003292, + "values":[ + 0.973828, + 0.97214, + 0.967473 + ] + }, + "selective_risk":{ + "mean":0.01989, + "std":0.001219, + "values":[ + 0.021045, + 0.020009, + 0.018616 + ] + }, + "in_scope_risk":{ + "mean":0.007474, + "std":0.000326, + "values":[ + 0.007833, + 0.007394, + 0.007196 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.012509, + "std":0.000923, + "values":[ + 0.013316, + 0.012709, + 0.011503 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.376667, + "std":0.029023, + "values":[ + 0.402, + 0.383, + 0.345 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.93874, + "std":0.013355, + "values":[ + 0.926211, + 0.93722, + 0.952791 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":-0.136809, + "std":0.011912, + "values":[ + -0.150062, + -0.13337, + -0.126994 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.785596, + "std":0.020965, + "values":[ + 0.797836, + 0.797563, + 0.761387 + ] + }, + "validation_weighted_risk":{ + "mean":0.014038, + "std":0.000892, + "values":[ + 0.014712, + 0.014375, + 0.013026 + ] + }, + "test":{ + "coverage":{ + "mean":0.789455, + "std":0.024618, + "values":[ + 0.805273, + 0.802, + 0.761091 + ] + }, + "selective_risk":{ + "mean":0.021834, + "std":0.005878, + "values":[ + 0.025062, + 0.025391, + 0.01505 + ] + }, + "in_scope_risk":{ + "mean":0.001793, + "std":0.000552, + "values":[ + 0.002311, + 0.001857, + 0.001211 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.020079, + "std":0.005403, + "values":[ + 0.022804, + 0.023577, + 0.013856 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.087667, + "std":0.025736, + "values":[ + 0.101, + 0.104, + 0.058 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "entropy", + "msp_t", + "msp_t" + ], + "tau":{ + "mean":-0.534049, + "std":1.151601, + "values":[ + -1.861188, + 0.201693, + 0.05735 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.039462, + "std":0.000931, + "values":[ + 0.04, + 0.038387, + 0.04 + ] + }, + "in_scope_risk":{ + "mean":0.007444, + "std":0.000962, + "values":[ + 0.008, + 0.006333, + 0.008 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.998788, + "std":0.00042, + "values":[ + 0.999273, + 0.998545, + 0.998545 + ] + }, + "selective_risk":{ + "mean":0.189684, + "std":0.00066, + "values":[ + 0.19032, + 0.189002, + 0.189731 + ] + }, + "in_scope_risk":{ + "mean":0.010815, + "std":0.000513, + "values":[ + 0.011111, + 0.010222, + 0.011111 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.180825, + "std":0.000344, + "values":[ + 0.181223, + 0.180626, + 0.180626 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.993333, + "std":0.002309, + "values":[ + 0.996, + 0.992, + 0.992 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.999785, + "std":7.4e-05, + "values":[ + 0.999871, + 0.999742, + 0.999742 + ] + }, + "selective_risk":{ + "mean":0.042518, + "std":0.000536, + "values":[ + 0.042887, + 0.041903, + 0.042764 + ] + }, + "in_scope_risk":{ + "mean":0.010815, + "std":0.000513, + "values":[ + 0.011111, + 0.010222, + 0.011111 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.03205, + "std":7.2e-05, + "values":[ + 0.032133, + 0.032008, + 0.032008 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.993333, + "std":0.002309, + "values":[ + 0.996, + 0.992, + 0.992 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.979663, + "std":0.002632, + "values":[ + 0.980793, + 0.976654, + 0.981542 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.16363, + "std":0.630569, + "values":[ + -0.557901, + 0.609019, + 0.439773 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.838399, + "std":0.002474, + "values":[ + 0.838763, + 0.840671, + 0.835763 + ] + }, + "validation_weighted_risk":{ + "mean":0.04211, + "std":0.000674, + "values":[ + 0.041333, + 0.042537, + 0.04246 + ] + }, + "test":{ + "coverage":{ + "mean":0.847333, + "std":0.005033, + "values":[ + 0.848, + 0.852, + 0.842 + ] + }, + "selective_risk":{ + "mean":0.056494, + "std":0.002956, + "values":[ + 0.055746, + 0.059752, + 0.053984 + ] + }, + "in_scope_risk":{ + "mean":0.004978, + "std":0.000437, + "values":[ + 0.004971, + 0.005418, + 0.004544 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.051775, + "std":0.002565, + "values":[ + 0.051029, + 0.054631, + 0.049665 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.241333, + "std":0.013317, + "values":[ + 0.238, + 0.256, + 0.23 + ] + } + } + } + } } } }, @@ -28872,6 +33736,28 @@ "std":32.65476 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.601017, + "std":0.045967 + }, + "auprc":{ + "mean":0.049707, + "std":0.015099 + } + }, + "test":{ + "auroc":{ + "mean":0.600622, + "std":0.01993 + }, + "auprc":{ + "mean":0.234677, + "std":0.021154 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -29591,7 +34477,7 @@ "entropy":{ "tau":{ "mean":-0.350296, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -29605,12 +34491,12 @@ }, "validation_selective_risk":{ "mean":0.0, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.049456, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -29620,7 +34506,7 @@ }, "selective_risk":{ "mean":0.029412, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -29781,7 +34667,7 @@ "msp_t":{ "tau":{ "mean":0.714544, - "std":0.0, + "std":null, "n":1 }, "feasible":[ @@ -29795,12 +34681,12 @@ }, "validation_selective_risk":{ "mean":0.0, - "std":0.0, + "std":null, "n":1 }, "validation_risk_upper95":{ "mean":0.048569, - "std":0.0, + "std":null, "n":1 }, "test":{ @@ -29810,7 +34696,7 @@ }, "selective_risk":{ "mean":0.028571, - "std":0.0, + "std":null, "n":1 }, "accuracy_8":{ @@ -30030,6 +34916,28 @@ "std":116.413058 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.654303, + "std":0.019714 + }, + "auprc":{ + "mean":0.060924, + "std":0.01297 + } + }, + "test":{ + "auroc":{ + "mean":0.630211, + "std":0.025757 + }, + "auprc":{ + "mean":0.260362, + "std":0.029626 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -31358,6 +36266,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.654303, + "std":0.019714, + "values":[ + 0.665347, + 0.66602, + 0.631543 + ] + }, + "auprc":{ + "mean":0.060924, + "std":0.01297, + "values":[ + 0.073013, + 0.062534, + 0.047224 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.630211, + "std":0.025757, + "values":[ + 0.656317, + 0.604817, + 0.6295 + ] + }, + "auprc":{ + "mean":0.260362, + "std":0.029626, + "values":[ + 0.291098, + 0.231986, + 0.258002 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -31972,6 +36922,444 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":"msp_t", + "tau":null, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "selective_risk":null, + "in_scope_risk":null, + "kept_oos_error_rate":null, + "kept_oos_share":null, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "selective_risk":null, + "in_scope_risk":null, + "kept_oos_error_rate":null, + "kept_oos_share":null, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "selective_risk":null, + "in_scope_risk":null, + "kept_oos_error_rate":null, + "kept_oos_share":null, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + } + }, + "share_of_gap_explained_by_oos_share":null, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":null, + "feasible":false, + "validation_weighted_coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "validation_weighted_risk":null, + "test":{ + "coverage":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "selective_risk":null, + "in_scope_risk":null, + "kept_oos_error_rate":null, + "kept_oos_share":null, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "msp", + "msp_t", + "margin" + ], + "tau":{ + "mean":1.080174, + "std":0.555253, + "values":[ + 0.962084, + 0.593466, + 1.684973 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.036022, + "std":0.016707, + "values":[ + 0.018387, + 0.051613, + 0.038065 + ] + }, + "selective_risk":{ + "mean":0.0119, + "std":0.010345, + "values":[ + 0.0, + 0.01875, + 0.016949 + ] + }, + "in_scope_risk":{ + "mean":0.0119, + "std":0.010345, + "values":[ + 0.0, + 0.01875, + 0.016949 + ] + }, + "kept_oos_error_rate":null, + "kept_oos_share":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.028485, + "std":0.016547, + "values":[ + 0.011818, + 0.044909, + 0.028727 + ] + }, + "selective_risk":{ + "mean":0.077161, + "std":0.040823, + "values":[ + 0.030769, + 0.093117, + 0.107595 + ] + }, + "in_scope_risk":{ + "mean":0.050616, + "std":0.018104, + "values":[ + 0.030769, + 0.054852, + 0.066225 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "n":2, + "values":[ + null, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.028263, + "std":0.024551, + "values":[ + 0.0, + 0.040486, + 0.044304 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.005667, + "std":0.005132, + "values":[ + 0.0, + 0.01, + 0.007 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.032656, + "std":0.018656, + "values":[ + 0.013978, + 0.05129, + 0.032699 + ] + }, + "selective_risk":{ + "mean":0.054746, + "std":0.021597, + "values":[ + 0.030769, + 0.060797, + 0.072673 + ] + }, + "in_scope_risk":{ + "mean":0.050616, + "std":0.018104, + "values":[ + 0.030769, + 0.054852, + 0.066225 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "n":2, + "values":[ + null, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.004398, + "std":0.003821, + "values":[ + 0.0, + 0.006289, + 0.006906 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.005667, + "std":0.005132, + "values":[ + 0.0, + 0.01, + 0.007 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.273287, + "std":0.237957, + "values":[ + 0.0, + 0.434609, + 0.385252 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":1.162692, + "std":0.805007, + "n":2, + "values":[ + null, + 0.593466, + 1.731918 + ] + }, + "feasible":[ + false, + true, + true + ], + "validation_weighted_coverage":{ + "mean":0.024358, + "std":0.022254, + "values":[ + 0.0, + 0.043627, + 0.029448 + ] + }, + "validation_weighted_risk":{ + "mean":0.014005, + "std":0.006711, + "n":2, + "values":[ + null, + 0.01875, + 0.009259 + ] + }, + "test":{ + "coverage":{ + "mean":0.023818, + "std":0.022578, + "values":[ + 0.0, + 0.044909, + 0.026545 + ] + }, + "selective_risk":{ + "mean":0.094504, + "std":0.001961, + "n":2, + "values":[ + null, + 0.093117, + 0.09589 + ] + }, + "in_scope_risk":{ + "mean":0.052606, + "std":0.003177, + "n":2, + "values":[ + null, + 0.054852, + 0.05036 + ] + }, + "kept_oos_error_rate":{ + "mean":1.0, + "std":0.0, + "n":2, + "values":[ + null, + 1.0, + 1.0 + ] + }, + "kept_oos_share":{ + "mean":0.044216, + "std":0.005275, + "n":2, + "values":[ + null, + 0.040486, + 0.047945 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.005667, + "std":0.005132, + "values":[ + 0.0, + 0.01, + 0.007 + ] + } + } + } + } } } }, @@ -32226,6 +37614,28 @@ "std":45.90207 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.935153, + "std":0.016966 + }, + "auprc":{ + "mean":0.463139, + "std":0.039752 + } + }, + "test":{ + "auroc":{ + "mean":0.9251, + "std":0.006766 + }, + "auprc":{ + "mean":0.746059, + "std":0.023413 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -33424,6 +38834,28 @@ "std":28.005952 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.949641, + "std":0.010213 + }, + "auprc":{ + "mean":0.527879, + "std":0.045362 + } + }, + "test":{ + "auroc":{ + "mean":0.93973, + "std":0.006363 + }, + "auprc":{ + "mean":0.787927, + "std":0.020805 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -34792,6 +40224,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.949641, + "std":0.010213, + "values":[ + 0.956993, + 0.95395, + 0.93798 + ] + }, + "auprc":{ + "mean":0.527879, + "std":0.045362, + "values":[ + 0.563157, + 0.54377, + 0.47671 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.93973, + "std":0.006363, + "values":[ + 0.944674, + 0.941965, + 0.93255 + ] + }, + "auprc":{ + "mean":0.787927, + "std":0.020805, + "values":[ + 0.810199, + 0.78459, + 0.768993 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -35442,6 +40916,604 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":[ + "msp_t", + "margin", + "msp" + ], + "tau":{ + "mean":0.961059, + "std":0.490732, + "values":[ + 0.680271, + 1.5277, + 0.675207 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.860323, + "std":0.029334, + "values":[ + 0.877097, + 0.877419, + 0.826452 + ] + }, + "selective_risk":{ + "mean":0.01537, + "std":0.000128, + "values":[ + 0.015447, + 0.015441, + 0.015222 + ] + }, + "in_scope_risk":{ + "mean":0.01136, + "std":0.000288, + "values":[ + 0.011528, + 0.011524, + 0.011028 + ] + }, + "kept_oos_error_rate":{ + "mean":0.403865, + "std":0.064429, + "values":[ + 0.366667, + 0.366667, + 0.478261 + ] + }, + "kept_oos_share":{ + "mean":0.010347, + "std":0.001186, + "values":[ + 0.011033, + 0.011029, + 0.008977 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.11, + "std":0.0, + "values":[ + 0.11, + 0.11, + 0.11 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.761212, + "std":0.037625, + "values":[ + 0.775455, + 0.789636, + 0.718545 + ] + }, + "selective_risk":{ + "mean":0.05093, + "std":0.011219, + "values":[ + 0.05041, + 0.062399, + 0.03998 + ] + }, + "in_scope_risk":{ + "mean":0.015436, + "std":0.002874, + "values":[ + 0.017728, + 0.016369, + 0.012211 + ] + }, + "kept_oos_error_rate":{ + "mean":0.606139, + "std":0.052663, + "values":[ + 0.553846, + 0.659164, + 0.605405 + ] + }, + "kept_oos_share":{ + "mean":0.059794, + "std":0.01244, + "values":[ + 0.060961, + 0.071609, + 0.046812 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.153667, + "std":0.047248, + "values":[ + 0.144, + 0.205, + 0.112 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.854294, + "std":0.033307, + "values":[ + 0.869677, + 0.877129, + 0.816075 + ] + }, + "selective_risk":{ + "mean":0.021056, + "std":0.003925, + "values":[ + 0.022898, + 0.023721, + 0.016549 + ] + }, + "in_scope_risk":{ + "mean":0.015436, + "std":0.002874, + "values":[ + 0.017728, + 0.016369, + 0.012211 + ] + }, + "kept_oos_error_rate":{ + "mean":0.606139, + "std":0.052663, + "values":[ + 0.553846, + 0.659164, + 0.605405 + ] + }, + "kept_oos_share":{ + "mean":0.009465, + "std":0.002068, + "values":[ + 0.009644, + 0.011438, + 0.007313 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.153667, + "std":0.047248, + "values":[ + 0.144, + 0.205, + 0.112 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.852324, + "std":0.083533, + "values":[ + 0.786884, + 0.823675, + 0.946412 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":1.836352, + "std":1.65303, + "values":[ + 0.932785, + 3.74422, + 0.83205 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.61152, + "std":0.013195, + "values":[ + 0.613793, + 0.597337, + 0.623431 + ] + }, + "validation_weighted_risk":{ + "mean":0.012785, + "std":0.000682, + "values":[ + 0.012449, + 0.012336, + 0.013569 + ] + }, + "test":{ + "coverage":{ + "mean":0.610121, + "std":0.004853, + "values":[ + 0.609455, + 0.605636, + 0.615273 + ] + }, + "selective_risk":{ + "mean":0.016379, + "std":0.002116, + "values":[ + 0.01611, + 0.01441, + 0.018617 + ] + }, + "in_scope_risk":{ + "mean":0.004772, + "std":0.000352, + "values":[ + 0.005178, + 0.004586, + 0.004552 + ] + }, + "kept_oos_error_rate":{ + "mean":0.541853, + "std":0.007223, + "values":[ + 0.536232, + 0.55, + 0.539326 + ] + }, + "kept_oos_share":{ + "mean":0.021633, + "std":0.004242, + "values":[ + 0.020585, + 0.018013, + 0.0263 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.039333, + "std":0.007767, + "values":[ + 0.037, + 0.033, + 0.048 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "margin", + "msp", + "msp" + ], + "tau":{ + "mean":0.461076, + "std":0.067367, + "values":[ + 0.530907, + 0.396479, + 0.455842 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.940215, + "std":0.022466, + "values":[ + 0.95, + 0.956129, + 0.914516 + ] + }, + "selective_risk":{ + "mean":0.043114, + "std":7.6e-05, + "values":[ + 0.043124, + 0.043185, + 0.043034 + ] + }, + "in_scope_risk":{ + "mean":0.032659, + "std":0.001528, + "values":[ + 0.031586, + 0.031981, + 0.034409 + ] + }, + "kept_oos_error_rate":{ + "mean":0.588426, + "std":0.032582, + "values":[ + 0.5625, + 0.625, + 0.577778 + ] + }, + "kept_oos_share":{ + "mean":0.018833, + "std":0.00293, + "values":[ + 0.021732, + 0.018893, + 0.015873 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.323333, + "std":0.055076, + "values":[ + 0.36, + 0.35, + 0.26 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.882485, + "std":0.032791, + "values":[ + 0.903818, + 0.898909, + 0.844727 + ] + }, + "selective_risk":{ + "mean":0.112, + "std":0.0149, + "values":[ + 0.117481, + 0.123382, + 0.095136 + ] + }, + "in_scope_risk":{ + "mean":0.033657, + "std":0.003167, + "values":[ + 0.03382, + 0.036739, + 0.030411 + ] + }, + "kept_oos_error_rate":{ + "mean":0.717422, + "std":0.047151, + "values":[ + 0.669725, + 0.764007, + 0.718535 + ] + }, + "kept_oos_share":{ + "mean":0.114919, + "std":0.019104, + "values":[ + 0.131563, + 0.119134, + 0.094059 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.400667, + "std":0.075295, + "values":[ + 0.438, + 0.45, + 0.314 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.941434, + "std":0.019443, + "values":[ + 0.949484, + 0.955559, + 0.919258 + ] + }, + "selective_risk":{ + "mean":0.046704, + "std":0.005231, + "values":[ + 0.047949, + 0.0512, + 0.040963 + ] + }, + "in_scope_risk":{ + "mean":0.033657, + "std":0.003167, + "values":[ + 0.03382, + 0.036739, + 0.030411 + ] + }, + "kept_oos_error_rate":{ + "mean":0.717422, + "std":0.047151, + "values":[ + 0.669725, + 0.764007, + 0.718535 + ] + }, + "kept_oos_share":{ + "mean":0.019146, + "std":0.003501, + "values":[ + 0.022219, + 0.019884, + 0.015335 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.400667, + "std":0.075295, + "values":[ + 0.438, + 0.45, + 0.314 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.958299, + "std":0.072669, + "values":[ + 0.935109, + 0.900056, + 1.039732 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.948894, + "std":0.561235, + "values":[ + 1.594872, + 0.580977, + 0.670832 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.771798, + "std":0.029052, + "values":[ + 0.785442, + 0.791516, + 0.738435 + ] + }, + "validation_weighted_risk":{ + "mean":0.041769, + "std":0.001308, + "values":[ + 0.042273, + 0.04275, + 0.040284 + ] + }, + "test":{ + "coverage":{ + "mean":0.764848, + "std":0.039766, + "values":[ + 0.775273, + 0.798364, + 0.720909 + ] + }, + "selective_risk":{ + "mean":0.052865, + "std":0.011082, + "values":[ + 0.055816, + 0.062173, + 0.040605 + ] + }, + "in_scope_risk":{ + "mean":0.016668, + "std":0.004022, + "values":[ + 0.017884, + 0.019942, + 0.012179 + ] + }, + "kept_oos_error_rate":{ + "mean":0.621439, + "std":0.058887, + "values":[ + 0.568027, + 0.684588, + 0.611702 + ] + }, + "kept_oos_share":{ + "mean":0.059968, + "std":0.011203, + "values":[ + 0.068949, + 0.063539, + 0.047415 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.157667, + "std":0.03885, + "values":[ + 0.167, + 0.191, + 0.115 + ] + } + } + } + } } } }, @@ -35696,6 +41768,28 @@ "std":9.539392 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.986162, + "std":0.003975 + }, + "auprc":{ + "mean":0.892032, + "std":0.015007 + } + }, + "test":{ + "auroc":{ + "mean":0.983263, + "std":0.000662 + }, + "auprc":{ + "mean":0.943354, + "std":0.002453 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -36898,6 +42992,28 @@ "std":2.645751 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.986842, + "std":0.003719 + }, + "auprc":{ + "mean":0.904913, + "std":0.014698 + } + }, + "test":{ + "auroc":{ + "mean":0.986057, + "std":0.00071 + }, + "auprc":{ + "mean":0.953116, + "std":0.002191 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -38266,6 +44382,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.986162, + "std":0.003975, + "values":[ + 0.989797, + 0.986773, + 0.981917 + ] + }, + "auprc":{ + "mean":0.892032, + "std":0.015007, + "values":[ + 0.874796, + 0.902207, + 0.899092 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.983263, + "std":0.000662, + "values":[ + 0.983097, + 0.982699, + 0.983991 + ] + }, + "auprc":{ + "mean":0.943354, + "std":0.002453, + "values":[ + 0.943151, + 0.941008, + 0.945902 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -38916,6 +45074,604 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":[ + "msp_t", + "entropy", + "margin" + ], + "tau":{ + "mean":-0.969815, + "std":2.014441, + "values":[ + 0.296954, + -3.292709, + 0.08631 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.997204, + "std":0.003539, + "values":[ + 0.993226, + 1.0, + 0.998387 + ] + }, + "selective_risk":{ + "mean":0.01499, + "std":0.001254, + "values":[ + 0.015589, + 0.013548, + 0.015832 + ] + }, + "in_scope_risk":{ + "mean":0.009019, + "std":0.002348, + "values":[ + 0.010044, + 0.006333, + 0.010681 + ] + }, + "kept_oos_error_rate":{ + "mean":0.199123, + "std":0.029296, + "values":[ + 0.195652, + 0.23, + 0.171717 + ] + }, + "kept_oos_share":{ + "mean":0.031375, + "std":0.001302, + "values":[ + 0.02988, + 0.032258, + 0.031987 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.193333, + "std":0.032146, + "values":[ + 0.18, + 0.23, + 0.17 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.987273, + "std":0.015872, + "values":[ + 0.969091, + 0.998364, + 0.994364 + ] + }, + "selective_risk":{ + "mean":0.07345, + "std":0.00857, + "values":[ + 0.063602, + 0.079221, + 0.077528 + ] + }, + "in_scope_risk":{ + "mean":0.012034, + "std":0.000699, + "values":[ + 0.012735, + 0.011336, + 0.012032 + ] + }, + "kept_oos_error_rate":{ + "mean":0.364825, + "std":0.030385, + "values":[ + 0.330211, + 0.387097, + 0.377166 + ] + }, + "kept_oos_share":{ + "mean":0.17342, + "std":0.011445, + "values":[ + 0.160225, + 0.180659, + 0.179375 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.345333, + "std":0.055293, + "values":[ + 0.282, + 0.384, + 0.37 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.995487, + "std":0.004836, + "values":[ + 0.990129, + 0.999527, + 0.996806 + ] + }, + "selective_risk":{ + "mean":0.022852, + "std":0.00112, + "values":[ + 0.021568, + 0.023366, + 0.023624 + ] + }, + "in_scope_risk":{ + "mean":0.012034, + "std":0.000699, + "values":[ + 0.012735, + 0.011336, + 0.012032 + ] + }, + "kept_oos_error_rate":{ + "mean":0.364825, + "std":0.030385, + "values":[ + 0.330211, + 0.387097, + 0.377166 + ] + }, + "kept_oos_share":{ + "mean":0.030528, + "std":0.002347, + "values":[ + 0.027823, + 0.032015, + 0.031747 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.345333, + "std":0.055293, + "values":[ + 0.282, + 0.384, + 0.37 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.866566, + "std":0.013936, + "values":[ + 0.875486, + 0.850507, + 0.873706 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.634354, + "std":1.549989, + "values":[ + 0.742111, + -0.966702, + 2.127651 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.912401, + "std":0.022449, + "values":[ + 0.892749, + 0.936865, + 0.907588 + ] + }, + "validation_weighted_risk":{ + "mean":0.015589, + "std":4.8e-05, + "values":[ + 0.015592, + 0.015636, + 0.015539 + ] + }, + "test":{ + "coverage":{ + "mean":0.885455, + "std":0.017603, + "values":[ + 0.873273, + 0.905636, + 0.877455 + ] + }, + "selective_risk":{ + "mean":0.029281, + "std":0.006288, + "values":[ + 0.025817, + 0.036539, + 0.025487 + ] + }, + "in_scope_risk":{ + "mean":0.00583, + "std":0.001422, + "values":[ + 0.005776, + 0.007278, + 0.004436 + ] + }, + "kept_oos_error_rate":{ + "mean":0.218933, + "std":0.033905, + "values":[ + 0.208421, + 0.256849, + 0.191529 + ] + }, + "kept_oos_share":{ + "mean":0.109553, + "std":0.009527, + "values":[ + 0.098897, + 0.117246, + 0.112516 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.117667, + "std":0.028113, + "values":[ + 0.099, + 0.15, + 0.104 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "entropy", + "entropy", + "margin" + ], + "tau":{ + "mean":-2.199503, + "std":1.914925, + "values":[ + -3.317417, + -3.292709, + 0.011616 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.016559, + "std":0.002781, + "values":[ + 0.019032, + 0.013548, + 0.017097 + ] + }, + "in_scope_risk":{ + "mean":0.009889, + "std":0.003079, + "values":[ + 0.011667, + 0.006333, + 0.011667 + ] + }, + "kept_oos_error_rate":{ + "mean":0.216667, + "std":0.032146, + "values":[ + 0.24, + 0.23, + 0.18 + ] + }, + "kept_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.216667, + "std":0.032146, + "values":[ + 0.24, + 0.23, + 0.18 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.998424, + "std":0.000639, + "values":[ + 0.997818, + 0.998364, + 0.999091 + ] + }, + "selective_risk":{ + "mean":0.080066, + "std":0.000842, + "values":[ + 0.080904, + 0.079221, + 0.080073 + ] + }, + "in_scope_risk":{ + "mean":0.012597, + "std":0.001339, + "values":[ + 0.014003, + 0.011336, + 0.012453 + ] + }, + "kept_oos_error_rate":{ + "mean":0.385701, + "std":0.001231, + "values":[ + 0.385238, + 0.387097, + 0.38477 + ] + }, + "kept_oos_share":{ + "mean":0.18083, + "std":0.00072, + "values":[ + 0.180211, + 0.180659, + 0.18162 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.383, + "std":0.001732, + "values":[ + 0.381, + 0.384, + 0.384 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.999416, + "std":0.000119, + "values":[ + 0.99943, + 0.999527, + 0.99929 + ] + }, + "selective_risk":{ + "mean":0.024556, + "std":0.001247, + "values":[ + 0.025853, + 0.023366, + 0.024447 + ] + }, + "in_scope_risk":{ + "mean":0.012597, + "std":0.001339, + "values":[ + 0.014003, + 0.011336, + 0.012453 + ] + }, + "kept_oos_error_rate":{ + "mean":0.385701, + "std":0.001231, + "values":[ + 0.385238, + 0.387097, + 0.38477 + ] + }, + "kept_oos_share":{ + "mean":0.032051, + "std":0.000151, + "values":[ + 0.031921, + 0.032015, + 0.032216 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.383, + "std":0.001732, + "values":[ + 0.381, + 0.384, + 0.384 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.874513, + "std":0.02104, + "values":[ + 0.889752, + 0.850507, + 0.883278 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":-1.643991, + "std":1.495057, + "values":[ + -2.048214, + -2.895375, + 0.011616 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.991175, + "std":0.012057, + "values":[ + 0.977437, + 0.996087, + 1.0 + ] + }, + "validation_weighted_risk":{ + "mean":0.042495, + "std":0.000725, + "values":[ + 0.041885, + 0.043297, + 0.042303 + ] + }, + "test":{ + "coverage":{ + "mean":0.985091, + "std":0.018589, + "values":[ + 0.964, + 0.992182, + 0.999091 + ] + }, + "selective_risk":{ + "mean":0.072545, + "std":0.009102, + "values":[ + 0.062429, + 0.075133, + 0.080073 + ] + }, + "in_scope_risk":{ + "mean":0.012249, + "std":0.000827, + "values":[ + 0.012955, + 0.011338, + 0.012453 + ] + }, + "kept_oos_error_rate":{ + "mean":0.363342, + "std":0.028567, + "values":[ + 0.330909, + 0.374348, + 0.38477 + ] + }, + "kept_oos_share":{ + "mean":0.170986, + "std":0.013644, + "values":[ + 0.155602, + 0.175738, + 0.18162 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.338667, + "std":0.058227, + "values":[ + 0.273, + 0.359, + 0.384 + ] + } + } + } + } } } }, @@ -39174,6 +45930,28 @@ "std":29.022979 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.967709, + "std":0.004586 + }, + "auprc":{ + "mean":0.659696, + "std":0.02793 + } + }, + "test":{ + "auroc":{ + "mean":0.958338, + "std":0.00445 + }, + "auprc":{ + "mean":0.857856, + "std":0.018183 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -40376,6 +47154,28 @@ "std":35.697806 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.974374, + "std":0.004489 + }, + "auprc":{ + "mean":0.703019, + "std":0.022743 + } + }, + "test":{ + "auroc":{ + "mean":0.967132, + "std":0.004157 + }, + "auprc":{ + "mean":0.882811, + "std":0.016865 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -41574,170 +48374,212 @@ ] }, "recovered_errors":{ - "mean":0.0, - "std":0.0, + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.862727, + "std":0.005216, + "values":[ + 0.868727, + 0.859273, + 0.860182 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.927212, + "std":0.001794, + "values":[ + 0.929273, + 0.926364, + 0.926 + ] + }, + "oos_recall":{ + "mean":0.647667, + "std":0.01464, + "values":[ + 0.661, + 0.65, + 0.632 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.617555, + "std":0.01486, + "values":[ + 0.631094, + 0.619914, + 0.601657 + ] + }, + { + "mean":0.676648, + "std":0.014309, + "values":[ + 0.689674, + 0.678938, + 0.661333 + ] + } + ], + "oos_precision":{ + "mean":0.974979, + "std":0.003944, + "values":[ + 0.970631, + 0.975976, + 0.978328 + ] + }, + "oos_f1":{ + "mean":0.778224, + "std":0.009432, + "values":[ + 0.786437, + 0.780312, + 0.767922 + ] + }, + "oos_misroute_rate":{ + "mean":0.352333, + "std":0.01464, + "values":[ + 0.339, + 0.35, + 0.368 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.137273, + "std":0.005216, + "values":[ + 0.131273, + 0.140727, + 0.139818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.05058, + "std":0.001922, + "values":[ + 0.048369, + 0.05185, + 0.051522 + ] + }, + "error_recovery_rate":{ + "mean":0.469568, + "std":0.007829, + "values":[ + 0.461219, + 0.476744, + 0.470741 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":755.0, + "std":28.687977, + "values":[ + 722, + 774, + 769 + ] + }, + "recoverable_errors":{ + "mean":354.666667, + "std":19.087518, + "values":[ + 333, + 369, + 362 + ] + }, + "recovered_errors":{ + "mean":354.666667, + "std":19.087518, "values":[ - 0, - 0, - 0 + 333, + 369, + 362 ] } }, - "oracle":{ - "coverage":{ - "mean":0.862727, - "std":0.005216, - "values":[ - 0.868727, - 0.859273, - 0.860182 - ] - }, - "selective_risk":{ - "mean":0.0, - "std":0.0, - "values":[ - 0.0, - 0.0, - 0.0 - ] - }, - "accuracy_8":{ - "mean":0.927212, - "std":0.001794, - "values":[ - 0.929273, - 0.926364, - 0.926 - ] - }, - "oos_recall":{ - "mean":0.647667, - "std":0.01464, - "values":[ - 0.661, - 0.65, - 0.632 - ] - }, - "oos_recall_wilson95":[ - { - "mean":0.617555, - "std":0.01486, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.972151, + "std":0.008338, "values":[ - 0.631094, - 0.619914, - 0.601657 + 0.976747, + 0.962527, + 0.97718 ] }, - { - "mean":0.676648, - "std":0.014309, + "auprc":{ + "mean":0.686921, + "std":0.050195, "values":[ - 0.689674, - 0.678938, - 0.661333 + 0.709801, + 0.629362, + 0.721599 ] } - ], - "oos_precision":{ - "mean":0.974979, - "std":0.003944, - "values":[ - 0.970631, - 0.975976, - 0.978328 - ] - }, - "oos_f1":{ - "mean":0.778224, - "std":0.009432, - "values":[ - 0.786437, - 0.780312, - 0.767922 - ] - }, - "oos_misroute_rate":{ - "mean":0.352333, - "std":0.01464, - "values":[ - 0.339, - 0.35, - 0.368 - ] - }, - "high_conf_oos_misroute_rate":{ - "mean":0.0, - "std":0.0, - "values":[ - 0.0, - 0.0, - 0.0 - ] - }, - "llm_call_rate":{ - "mean":0.137273, - "std":0.005216, - "values":[ - 0.131273, - 0.140727, - 0.139818 - ] }, - "llm_cost_usd_per_1k":{ - "mean":0.05058, - "std":0.001922, - "values":[ - 0.048369, - 0.05185, - 0.051522 - ] - }, - "error_recovery_rate":{ - "mean":0.469568, - "std":0.007829, - "values":[ - 0.461219, - 0.476744, - 0.470741 - ] - }, - "recoverable_caught":{ - "mean":1.0, - "std":0.0, - "values":[ - 1.0, - 1.0, - 1.0 - ] - }, - "small_errors":{ - "mean":755.0, - "std":28.687977, - "values":[ - 722, - 774, - 769 - ] - }, - "recoverable_errors":{ - "mean":354.666667, - "std":19.087518, - "values":[ - 333, - 369, - 362 - ] - }, - "recovered_errors":{ - "mean":354.666667, - "std":19.087518, - "values":[ - 333, - 369, - 362 - ] + "test":{ + "auroc":{ + "mean":0.96425, + "std":0.009139, + "values":[ + 0.969926, + 0.953708, + 0.969117 + ] + }, + "auprc":{ + "mean":0.874778, + "std":0.030686, + "values":[ + 0.895034, + 0.839472, + 0.889827 + ] + } } }, "signals":{ @@ -42390,6 +49232,604 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":[ + "msp_t", + "entropy", + "msp_t" + ], + "tau":{ + "mean":-0.205695, + "std":1.552599, + "values":[ + 0.703851, + -1.998417, + 0.677481 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.941935, + "std":0.011179, + "values":[ + 0.948065, + 0.929032, + 0.94871 + ] + }, + "selective_risk":{ + "mean":0.015639, + "std":1.3e-05, + "values":[ + 0.015652, + 0.015625, + 0.015641 + ] + }, + "in_scope_risk":{ + "mean":0.010893, + "std":0.000511, + "values":[ + 0.011391, + 0.010919, + 0.01037 + ] + }, + "kept_oos_error_rate":{ + "mean":0.328107, + "std":0.016599, + "values":[ + 0.309524, + 0.341463, + 0.333333 + ] + }, + "kept_oos_share":{ + "mean":0.014949, + "std":0.001188, + "values":[ + 0.014291, + 0.014236, + 0.016321 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.143333, + "std":0.015275, + "values":[ + 0.13, + 0.14, + 0.16 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.856606, + "std":0.012531, + "values":[ + 0.86, + 0.842727, + 0.867091 + ] + }, + "selective_risk":{ + "mean":0.052779, + "std":0.005235, + "values":[ + 0.046934, + 0.054369, + 0.057035 + ] + }, + "in_scope_risk":{ + "mean":0.010949, + "std":0.004234, + "values":[ + 0.007859, + 0.015776, + 0.009212 + ] + }, + "kept_oos_error_rate":{ + "mean":0.495159, + "std":0.042105, + "values":[ + 0.465347, + 0.476804, + 0.543326 + ] + }, + "kept_oos_share":{ + "mean":0.08622, + "std":0.002996, + "values":[ + 0.085412, + 0.083711, + 0.089537 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.201667, + "std":0.026312, + "values":[ + 0.188, + 0.185, + 0.232 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.938914, + "std":0.011506, + "values":[ + 0.943355, + 0.925849, + 0.947538 + ] + }, + "selective_risk":{ + "mean":0.017721, + "std":0.003967, + "values":[ + 0.01418, + 0.022008, + 0.016977 + ] + }, + "in_scope_risk":{ + "mean":0.010949, + "std":0.004234, + "values":[ + 0.007859, + 0.015776, + 0.009212 + ] + }, + "kept_oos_error_rate":{ + "mean":0.495159, + "std":0.042105, + "values":[ + 0.465347, + 0.476804, + 0.543326 + ] + }, + "kept_oos_share":{ + "mean":0.013957, + "std":0.000524, + "values":[ + 0.013815, + 0.013519, + 0.014537 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.201667, + "std":0.026312, + "values":[ + 0.188, + 0.185, + 0.232 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.950011, + "std":0.107012, + "values":[ + 1.047056, + 0.835245, + 0.967732 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.291703, + "std":1.114023, + "values":[ + 0.93586, + -0.994659, + 0.933909 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.754238, + "std":0.013729, + "values":[ + 0.761141, + 0.738428, + 0.763145 + ] + }, + "validation_weighted_risk":{ + "mean":0.015116, + "std":7.9e-05, + "values":[ + 0.01518, + 0.015028, + 0.01514 + ] + }, + "test":{ + "coverage":{ + "mean":0.744606, + "std":0.013828, + "values":[ + 0.751091, + 0.728727, + 0.754 + ] + }, + "selective_risk":{ + "mean":0.019235, + "std":0.002146, + "values":[ + 0.018155, + 0.021707, + 0.017844 + ] + }, + "in_scope_risk":{ + "mean":0.003913, + "std":0.002015, + "values":[ + 0.002513, + 0.006222, + 0.003002 + ] + }, + "kept_oos_error_rate":{ + "mean":0.419394, + "std":0.007393, + "values":[ + 0.427632, + 0.417219, + 0.413333 + ] + }, + "kept_oos_share":{ + "mean":0.03688, + "std":0.000756, + "values":[ + 0.036795, + 0.037675, + 0.036171 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.063333, + "std":0.001528, + "values":[ + 0.065, + 0.063, + 0.062 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "msp_t", + "msp", + "margin" + ], + "tau":{ + "mean":0.112426, + "std":0.104565, + "values":[ + 0.213919, + 0.11832, + 0.005039 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.999247, + "std":0.001304, + "values":[ + 1.0, + 0.997742, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.041, + "std":0.002168, + "values":[ + 0.039032, + 0.043324, + 0.040645 + ] + }, + "in_scope_risk":{ + "mean":0.025228, + "std":0.001272, + "values":[ + 0.024667, + 0.026684, + 0.024333 + ] + }, + "kept_oos_error_rate":{ + "mean":0.522807, + "std":0.049603, + "values":[ + 0.47, + 0.568421, + 0.53 + ] + }, + "kept_oos_share":{ + "mean":0.031744, + "std":0.000891, + "values":[ + 0.032258, + 0.030715, + 0.032258 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.513333, + "std":0.037859, + "values":[ + 0.47, + 0.54, + 0.53 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.99697, + "std":0.00447, + "values":[ + 0.999273, + 0.991818, + 0.999818 + ] + }, + "selective_risk":{ + "mean":0.135135, + "std":0.004605, + "values":[ + 0.13064, + 0.134922, + 0.139844 + ] + }, + "in_scope_risk":{ + "mean":0.026309, + "std":0.003844, + "values":[ + 0.025561, + 0.030472, + 0.022894 + ] + }, + "kept_oos_error_rate":{ + "mean":0.631808, + "std":0.031222, + "values":[ + 0.604814, + 0.624609, + 0.666 + ] + }, + "kept_oos_share":{ + "mean":0.179686, + "std":0.003371, + "values":[ + 0.181405, + 0.175802, + 0.181851 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.622667, + "std":0.037581, + "values":[ + 0.603, + 0.599, + 0.666 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.999097, + "std":0.001109, + "values":[ + 0.999688, + 0.997817, + 0.999785 + ] + }, + "selective_risk":{ + "mean":0.045577, + "std":0.002884, + "values":[ + 0.044197, + 0.048892, + 0.043644 + ] + }, + "in_scope_risk":{ + "mean":0.026309, + "std":0.003844, + "values":[ + 0.025561, + 0.030472, + 0.022894 + ] + }, + "kept_oos_error_rate":{ + "mean":0.631808, + "std":0.031222, + "values":[ + 0.604814, + 0.624609, + 0.666 + ] + }, + "kept_oos_share":{ + "mean":0.031813, + "std":0.000703, + "values":[ + 0.032171, + 0.031003, + 0.032265 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.622667, + "std":0.037581, + "values":[ + 0.603, + 0.599, + 0.666 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.95087, + "std":0.016517, + "values":[ + 0.943626, + 0.939213, + 0.969771 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":1.001867, + "std":0.745569, + "values":[ + 0.682157, + 0.469472, + 1.853974 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.861729, + "std":0.014265, + "values":[ + 0.878182, + 0.854187, + 0.852817 + ] + }, + "validation_weighted_risk":{ + "mean":0.042969, + "std":7.5e-05, + "values":[ + 0.042886, + 0.043029, + 0.042994 + ] + }, + "test":{ + "coverage":{ + "mean":0.854848, + "std":0.009999, + "values":[ + 0.866182, + 0.851091, + 0.847273 + ] + }, + "selective_risk":{ + "mean":0.05233, + "std":0.005423, + "values":[ + 0.049958, + 0.058535, + 0.048498 + ] + }, + "in_scope_risk":{ + "mean":0.010271, + "std":0.004741, + "values":[ + 0.008984, + 0.015522, + 0.006305 + ] + }, + "kept_oos_error_rate":{ + "mean":0.493918, + "std":0.029084, + "values":[ + 0.470449, + 0.484848, + 0.526455 + ] + }, + "kept_oos_share":{ + "mean":0.087185, + "std":0.005446, + "values":[ + 0.088791, + 0.091647, + 0.081116 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.202, + "std":0.005196, + "values":[ + 0.199, + 0.208, + 0.199 + ] + } + } + } + } } } }, @@ -42644,6 +50084,28 @@ "std":163.685674 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.846247, + "std":0.022211 + }, + "auprc":{ + "mean":0.253549, + "std":0.045434 + } + }, + "test":{ + "auroc":{ + "mean":0.827879, + "std":0.026987 + }, + "auprc":{ + "mean":0.508474, + "std":0.06374 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -43846,6 +51308,28 @@ "std":130.936371 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.884641, + "std":0.022125 + }, + "auprc":{ + "mean":0.301445, + "std":0.065157 + } + }, + "test":{ + "auroc":{ + "mean":0.853225, + "std":0.025929 + }, + "auprc":{ + "mean":0.558398, + "std":0.065821 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -45214,6 +52698,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.884641, + "std":0.022125, + "values":[ + 0.898023, + 0.859103, + 0.896797 + ] + }, + "auprc":{ + "mean":0.301445, + "std":0.065157, + "values":[ + 0.316291, + 0.230145, + 0.357898 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.853225, + "std":0.025929, + "values":[ + 0.880519, + 0.828919, + 0.850238 + ] + }, + "auprc":{ + "mean":0.558398, + "std":0.065821, + "values":[ + 0.627209, + 0.496042, + 0.551944 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -45864,6 +53390,604 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":[ + "margin", + "margin", + "msp_t" + ], + "tau":{ + "mean":1.99402, + "std":0.962075, + "values":[ + 2.537916, + 2.560953, + 0.88319 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.469677, + "std":0.078016, + "values":[ + 0.55, + 0.394194, + 0.464839 + ] + }, + "selective_risk":{ + "mean":0.013683, + "std":0.00052, + "values":[ + 0.014076, + 0.013093, + 0.013879 + ] + }, + "in_scope_risk":{ + "mean":0.013316, + "std":0.000214, + "values":[ + 0.013553, + 0.013136, + 0.013259 + ] + }, + "kept_oos_error_rate":{ + "mean":0.083333, + "std":0.072169, + "values":[ + 0.125, + 0.0, + 0.125 + ] + }, + "kept_oos_share":{ + "mean":0.004506, + "std":0.001151, + "values":[ + 0.004692, + 0.003273, + 0.005552 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.006667, + "std":0.005774, + "values":[ + 0.01, + 0.0, + 0.01 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.400424, + "std":0.067239, + "values":[ + 0.466, + 0.331636, + 0.403636 + ] + }, + "selective_risk":{ + "mean":0.033054, + "std":0.002248, + "values":[ + 0.031994, + 0.035636, + 0.031532 + ] + }, + "in_scope_risk":{ + "mean":0.015241, + "std":0.001308, + "values":[ + 0.013776, + 0.016292, + 0.015654 + ] + }, + "kept_oos_error_rate":{ + "mean":0.691148, + "std":0.164551, + "values":[ + 0.505263, + 0.818182, + 0.75 + ] + }, + "kept_oos_share":{ + "mean":0.027603, + "std":0.00829, + "values":[ + 0.037066, + 0.024123, + 0.021622 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.04, + "std":0.006928, + "values":[ + 0.048, + 0.036, + 0.036 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.462226, + "std":0.075007, + "values":[ + 0.533817, + 0.384215, + 0.468645 + ] + }, + "selective_risk":{ + "mean":0.017977, + "std":0.001331, + "values":[ + 0.016598, + 0.019254, + 0.01808 + ] + }, + "in_scope_risk":{ + "mean":0.015241, + "std":0.001308, + "values":[ + 0.013776, + 0.016292, + 0.015654 + ] + }, + "kept_oos_error_rate":{ + "mean":0.691148, + "std":0.164551, + "values":[ + 0.505263, + 0.818182, + 0.75 + ] + }, + "kept_oos_share":{ + "mean":0.004246, + "std":0.001309, + "values":[ + 0.005741, + 0.003694, + 0.003304 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.04, + "std":0.006928, + "values":[ + 0.048, + 0.036, + 0.036 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.782661, + "std":0.068655, + "values":[ + 0.859266, + 0.726689, + 0.762026 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":2.099428, + "std":1.044613, + "values":[ + 2.833788, + 2.560953, + 0.903542 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.384048, + "std":0.048556, + "values":[ + 0.435735, + 0.339388, + 0.377023 + ] + }, + "validation_weighted_risk":{ + "mean":0.012181, + "std":0.001815, + "values":[ + 0.013563, + 0.012855, + 0.010125 + ] + }, + "test":{ + "coverage":{ + "mean":0.379212, + "std":0.048288, + "values":[ + 0.428182, + 0.331636, + 0.377818 + ] + }, + "selective_risk":{ + "mean":0.029213, + "std":0.005643, + "values":[ + 0.025053, + 0.035636, + 0.026949 + ] + }, + "in_scope_risk":{ + "mean":0.013493, + "std":0.002682, + "values":[ + 0.010946, + 0.016292, + 0.013242 + ] + }, + "kept_oos_error_rate":{ + "mean":0.680215, + "std":0.178311, + "values":[ + 0.478873, + 0.818182, + 0.74359 + ] + }, + "kept_oos_share":{ + "mean":0.024346, + "std":0.005694, + "values":[ + 0.030149, + 0.024123, + 0.018768 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.033, + "std":0.003606, + "values":[ + 0.034, + 0.036, + 0.029 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "msp", + "margin", + "margin" + ], + "tau":{ + "mean":1.234831, + "std":0.600639, + "values":[ + 0.54394, + 1.6329, + 1.527653 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.665376, + "std":0.099866, + "values":[ + 0.756129, + 0.558387, + 0.681613 + ] + }, + "selective_risk":{ + "mean":0.041791, + "std":0.000673, + "values":[ + 0.042235, + 0.041017, + 0.04212 + ] + }, + "in_scope_risk":{ + "mean":0.0368, + "std":0.000446, + "values":[ + 0.03667, + 0.037296, + 0.036433 + ] + }, + "kept_oos_error_rate":{ + "mean":0.495537, + "std":0.037905, + "values":[ + 0.538462, + 0.466667, + 0.481481 + ] + }, + "kept_oos_share":{ + "mean":0.010845, + "std":0.002067, + "values":[ + 0.011092, + 0.008666, + 0.012778 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.113333, + "std":0.037859, + "values":[ + 0.14, + 0.07, + 0.13 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.584364, + "std":0.090743, + "values":[ + 0.668727, + 0.488364, + 0.596 + ] + }, + "selective_risk":{ + "mean":0.082761, + "std":0.007935, + "values":[ + 0.079391, + 0.077066, + 0.091824 + ] + }, + "in_scope_risk":{ + "mean":0.038959, + "std":0.000295, + "values":[ + 0.038852, + 0.038732, + 0.039293 + ] + }, + "kept_oos_error_rate":{ + "mean":0.776254, + "std":0.075595, + "values":[ + 0.689956, + 0.830769, + 0.808036 + ] + }, + "kept_oos_share":{ + "mean":0.059665, + "std":0.010218, + "values":[ + 0.062262, + 0.048399, + 0.068334 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.149, + "std":0.037323, + "values":[ + 0.158, + 0.108, + 0.181 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.655659, + "std":0.097885, + "values":[ + 0.749108, + 0.553871, + 0.664 + ] + }, + "selective_risk":{ + "mean":0.045887, + "std":0.001558, + "values":[ + 0.045273, + 0.044729, + 0.047658 + ] + }, + "in_scope_risk":{ + "mean":0.038959, + "std":0.000295, + "values":[ + 0.038852, + 0.038732, + 0.039293 + ] + }, + "kept_oos_error_rate":{ + "mean":0.776254, + "std":0.075595, + "values":[ + 0.689956, + 0.830769, + 0.808036 + ] + }, + "kept_oos_share":{ + "mean":0.009438, + "std":0.001695, + "values":[ + 0.009861, + 0.007571, + 0.010882 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.149, + "std":0.037323, + "values":[ + 0.158, + 0.108, + 0.181 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.901286, + "std":0.015296, + "values":[ + 0.918262, + 0.897019, + 0.888577 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":1.567414, + "std":0.814044, + "values":[ + 0.634666, + 1.933021, + 2.134554 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.519945, + "std":0.086301, + "values":[ + 0.6099, + 0.437834, + 0.512101 + ] + }, + "validation_weighted_risk":{ + "mean":0.040406, + "std":0.001167, + "values":[ + 0.041745, + 0.039872, + 0.0396 + ] + }, + "test":{ + "coverage":{ + "mean":0.506667, + "std":0.082701, + "values":[ + 0.594545, + 0.430364, + 0.495091 + ] + }, + "selective_risk":{ + "mean":0.056078, + "std":0.003225, + "values":[ + 0.053211, + 0.059569, + 0.055454 + ] + }, + "in_scope_risk":{ + "mean":0.027552, + "std":0.003283, + "values":[ + 0.027876, + 0.030661, + 0.024119 + ] + }, + "kept_oos_error_rate":{ + "mean":0.740641, + "std":0.138258, + "values":[ + 0.583893, + 0.845238, + 0.792793 + ] + }, + "kept_oos_share":{ + "mean":0.040606, + "std":0.005041, + "values":[ + 0.045566, + 0.035488, + 0.040764 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.082, + "std":0.009539, + "values":[ + 0.087, + 0.071, + 0.088 + ] + } + } + } + } } } }, @@ -46118,6 +54242,28 @@ "std":15.620499 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.98479, + "std":0.003799 + }, + "auprc":{ + "mean":0.849991, + "std":0.026715 + } + }, + "test":{ + "auroc":{ + "mean":0.978376, + "std":0.001909 + }, + "auprc":{ + "mean":0.923522, + "std":0.004787 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -47316,6 +55462,28 @@ "std":10.969655 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.985706, + "std":0.002137 + }, + "auprc":{ + "mean":0.867989, + "std":0.022486 + } + }, + "test":{ + "auroc":{ + "mean":0.982345, + "std":0.001008 + }, + "auprc":{ + "mean":0.937779, + "std":0.001723 + } + } + }, "signals":{ "msp":{ "validation":{ @@ -48680,6 +56848,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.98479, + "std":0.003799, + "values":[ + 0.981253, + 0.98431, + 0.988807 + ] + }, + "auprc":{ + "mean":0.849991, + "std":0.026715, + "values":[ + 0.821683, + 0.85353, + 0.87476 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.978376, + "std":0.001909, + "values":[ + 0.979479, + 0.976171, + 0.979477 + ] + }, + "auprc":{ + "mean":0.923522, + "std":0.004787, + "values":[ + 0.926174, + 0.917996, + 0.926397 + ] + } + } + }, "signals":{ "msp":{ "validation":{ @@ -49326,6 +57536,600 @@ } } } + }, + "diagnostics":{ + "0.02":{ + "signal":"msp", + "tau":{ + "mean":0.350617, + "std":0.073413, + "values":[ + 0.433208, + 0.325855, + 0.292786 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":0.987204, + "std":0.005439, + "values":[ + 0.980968, + 0.989677, + 0.990968 + ] + }, + "selective_risk":{ + "mean":0.015685, + "std":8.7e-05, + "values":[ + 0.015784, + 0.015645, + 0.015625 + ] + }, + "in_scope_risk":{ + "mean":0.009868, + "std":0.000835, + "values":[ + 0.009801, + 0.00907, + 0.010735 + ] + }, + "kept_oos_error_rate":{ + "mean":0.212767, + "std":0.031997, + "values":[ + 0.231707, + 0.230769, + 0.175824 + ] + }, + "kept_oos_share":{ + "mean":0.028749, + "std":0.001546, + "values":[ + 0.026965, + 0.029661, + 0.029622 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.186667, + "std":0.025166, + "values":[ + 0.19, + 0.21, + 0.16 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.961455, + "std":0.016545, + "values":[ + 0.942545, + 0.968545, + 0.973273 + ] + }, + "selective_risk":{ + "mean":0.072151, + "std":0.013156, + "values":[ + 0.057099, + 0.077905, + 0.08145 + ] + }, + "in_scope_risk":{ + "mean":0.012008, + "std":0.001185, + "values":[ + 0.010794, + 0.01207, + 0.013161 + ] + }, + "kept_oos_error_rate":{ + "mean":0.397682, + "std":0.053227, + "values":[ + 0.336499, + 0.423212, + 0.433333 + ] + }, + "kept_oos_share":{ + "mean":0.154941, + "std":0.011126, + "values":[ + 0.142168, + 0.160128, + 0.162526 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.328667, + "std":0.070316, + "values":[ + 0.248, + 0.361, + 0.377 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.987312, + "std":0.006352, + "values":[ + 0.980118, + 0.989667, + 0.992151 + ] + }, + "selective_risk":{ + "mean":0.022414, + "std":0.003313, + "values":[ + 0.018694, + 0.023501, + 0.025046 + ] + }, + "in_scope_risk":{ + "mean":0.012008, + "std":0.001185, + "values":[ + 0.010794, + 0.01207, + 0.013161 + ] + }, + "kept_oos_error_rate":{ + "mean":0.397682, + "std":0.053227, + "values":[ + 0.336499, + 0.423212, + 0.433333 + ] + }, + "kept_oos_share":{ + "mean":0.026782, + "std":0.002201, + "values":[ + 0.024256, + 0.027803, + 0.028287 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.328667, + "std":0.070316, + "values":[ + 0.248, + 0.361, + 0.377 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.886756, + "std":0.03803, + "values":[ + 0.929565, + 0.873826, + 0.856876 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":0.783936, + "std":0.105887, + "values":[ + 0.856062, + 0.833374, + 0.662372 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.87678, + "std":0.029431, + "values":[ + 0.856832, + 0.862926, + 0.910581 + ] + }, + "validation_weighted_risk":{ + "mean":0.01486, + "std":0.001011, + "values":[ + 0.01529, + 0.013705, + 0.015586 + ] + }, + "test":{ + "coverage":{ + "mean":0.851758, + "std":0.030085, + "values":[ + 0.831818, + 0.837091, + 0.886364 + ] + }, + "selective_risk":{ + "mean":0.031132, + "std":0.010948, + "values":[ + 0.021202, + 0.029322, + 0.042872 + ] + }, + "in_scope_risk":{ + "mean":0.005557, + "std":0.001588, + "values":[ + 0.004763, + 0.004523, + 0.007385 + ] + }, + "kept_oos_error_rate":{ + "mean":0.273066, + "std":0.062221, + "values":[ + 0.204787, + 0.287841, + 0.326568 + ] + }, + "kept_oos_share":{ + "mean":0.093633, + "std":0.015429, + "values":[ + 0.082186, + 0.087533, + 0.111179 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.123333, + "std":0.050402, + "values":[ + 0.077, + 0.116, + 0.177 + ] + } + } + } + }, + "0.05":{ + "signal":[ + "entropy", + "entropy", + "margin" + ], + "tau":{ + "mean":-2.411483, + "std":2.10235, + "values":[ + -3.457853, + -3.785324, + 0.008728 + ] + }, + "validation_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "test_oos_share":{ + "mean":0.181818, + "std":0.0, + "values":[ + 0.181818, + 0.181818, + 0.181818 + ] + }, + "validation":{ + "coverage":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "selective_risk":{ + "mean":0.021183, + "std":0.001624, + "values":[ + 0.022903, + 0.019677, + 0.020968 + ] + }, + "in_scope_risk":{ + "mean":0.013222, + "std":0.001388, + "values":[ + 0.013667, + 0.011667, + 0.014333 + ] + }, + "kept_oos_error_rate":{ + "mean":0.26, + "std":0.04, + "values":[ + 0.3, + 0.26, + 0.22 + ] + }, + "kept_oos_share":{ + "mean":0.032258, + "std":0.0, + "values":[ + 0.032258, + 0.032258, + 0.032258 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.26, + "std":0.04, + "values":[ + 0.3, + 0.26, + 0.22 + ] + } + }, + "test":{ + "coverage":{ + "mean":0.998848, + "std":0.001183, + "values":[ + 0.997636, + 1.0, + 0.998909 + ] + }, + "selective_risk":{ + "mean":0.095259, + "std":0.004495, + "values":[ + 0.090213, + 0.096727, + 0.098835 + ] + }, + "in_scope_risk":{ + "mean":0.014818, + "std":0.001002, + "values":[ + 0.015778, + 0.013778, + 0.014899 + ] + }, + "kept_oos_error_rate":{ + "mean":0.459006, + "std":0.025749, + "values":[ + 0.429585, + 0.47, + 0.477432 + ] + }, + "kept_oos_share":{ + "mean":0.181056, + "std":0.001034, + "values":[ + 0.17988, + 0.181818, + 0.181471 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.456667, + "std":0.028449, + "values":[ + 0.424, + 0.47, + 0.476 + ] + } + }, + "test_reweighted_to_validation_oos_share":{ + "coverage":{ + "mean":0.999613, + "std":0.000372, + "values":[ + 0.999581, + 1.0, + 0.999258 + ] + }, + "selective_risk":{ + "mean":0.02908, + "std":0.000654, + "values":[ + 0.028958, + 0.028495, + 0.029786 + ] + }, + "in_scope_risk":{ + "mean":0.014818, + "std":0.001002, + "values":[ + 0.015778, + 0.013778, + 0.014899 + ] + }, + "kept_oos_error_rate":{ + "mean":0.459006, + "std":0.025749, + "values":[ + 0.429585, + 0.47, + 0.477432 + ] + }, + "kept_oos_share":{ + "mean":0.032098, + "std":0.000216, + "values":[ + 0.031852, + 0.032258, + 0.032185 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.456667, + "std":0.028449, + "values":[ + 0.424, + 0.47, + 0.476 + ] + } + }, + "share_of_gap_explained_by_oos_share":{ + "mean":0.894121, + "std":0.013799, + "values":[ + 0.910041, + 0.885565, + 0.886759 + ] + }, + "sensitivity_reweighted_validation":{ + "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", + "tau":{ + "mean":-1.382348, + "std":1.398516, + "values":[ + -2.053999, + -2.318337, + 0.225291 + ] + }, + "feasible":true, + "validation_weighted_coverage":{ + "mean":0.961548, + "std":0.018973, + "values":[ + 0.942775, + 0.961153, + 0.980716 + ] + }, + "validation_weighted_risk":{ + "mean":0.043017, + "std":0.000467, + "values":[ + 0.043136, + 0.042502, + 0.043413 + ] + }, + "test":{ + "coverage":{ + "mean":0.95503, + "std":0.021507, + "values":[ + 0.936, + 0.950727, + 0.978364 + ] + }, + "selective_risk":{ + "mean":0.069762, + "std":0.015615, + "values":[ + 0.055167, + 0.067891, + 0.086229 + ] + }, + "in_scope_risk":{ + "mean":0.011735, + "std":0.001123, + "values":[ + 0.010801, + 0.011422, + 0.012981 + ] + }, + "kept_oos_error_rate":{ + "mean":0.392607, + "std":0.054922, + "values":[ + 0.335227, + 0.397906, + 0.444688 + ] + }, + "kept_oos_share":{ + "mean":0.150844, + "std":0.016963, + "values":[ + 0.136752, + 0.146108, + 0.169671 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.315333, + "std":0.085565, + "values":[ + 0.236, + 0.304, + 0.406 + ] + } + } + } + } } } }, @@ -49559,6 +58363,28 @@ "std":70.14984 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.647698, + "std":0.028063 + }, + "auprc":{ + "mean":0.060512, + "std":0.014318 + } + }, + "test":{ + "auroc":{ + "mean":0.62508, + "std":0.009563 + }, + "auprc":{ + "mean":0.247802, + "std":0.006316 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -50301,6 +59127,28 @@ "std":61.32971 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.648124, + "std":0.028677 + }, + "auprc":{ + "mean":0.06421, + "std":0.024765 + } + }, + "test":{ + "auroc":{ + "mean":0.634548, + "std":0.011313 + }, + "auprc":{ + "mean":0.251264, + "std":0.011063 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -51039,170 +59887,212 @@ ] }, "recovered_errors":{ - "mean":0.0, - "std":0.0, + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.498, + "std":0.011789, + "values":[ + 0.485273, + 0.500182, + 0.508545 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.869152, + "std":0.003456, + "values":[ + 0.869273, + 0.865636, + 0.872545 + ] + }, + "oos_recall":{ + "mean":0.568667, + "std":0.001155, + "values":[ + 0.568, + 0.57, + 0.568 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.537765, + "std":0.00116, + "values":[ + 0.537095, + 0.539105, + 0.537095 + ] + }, + { + "mean":0.599043, + "std":0.00114, + "values":[ + 0.598384, + 0.60036, + 0.598384 + ] + } + ], + "oos_precision":{ + "mean":0.845467, + "std":0.010443, + "values":[ + 0.847761, + 0.854573, + 0.834068 + ] + }, + "oos_f1":{ + "mean":0.679964, + "std":0.004045, + "values":[ + 0.68024, + 0.683863, + 0.675788 + ] + }, + "oos_misroute_rate":{ + "mean":0.431333, + "std":0.001155, + "values":[ + 0.432, + 0.43, + 0.432 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.502, + "std":0.011789, + "values":[ + 0.514727, + 0.499818, + 0.491455 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.185303, + "std":0.004398, + "values":[ + 0.190043, + 0.184515, + 0.181352 + ] + }, + "error_recovery_rate":{ + "mean":0.739287, + "std":0.00752, + "values":[ + 0.746026, + 0.731175, + 0.740659 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":2761.0, + "std":64.83826, + "values":[ + 2831, + 2749, + 2703 + ] + }, + "recoverable_errors":{ + "mean":2041.333333, + "std":61.32971, + "values":[ + 2112, + 2010, + 2002 + ] + }, + "recovered_errors":{ + "mean":2041.333333, + "std":61.32971, "values":[ - 0, - 0, - 0 + 2112, + 2010, + 2002 ] } }, - "oracle":{ - "coverage":{ - "mean":0.498, - "std":0.011789, - "values":[ - 0.485273, - 0.500182, - 0.508545 - ] - }, - "selective_risk":{ - "mean":0.0, - "std":0.0, - "values":[ - 0.0, - 0.0, - 0.0 - ] - }, - "accuracy_8":{ - "mean":0.869152, - "std":0.003456, - "values":[ - 0.869273, - 0.865636, - 0.872545 - ] - }, - "oos_recall":{ - "mean":0.568667, - "std":0.001155, - "values":[ - 0.568, - 0.57, - 0.568 - ] - }, - "oos_recall_wilson95":[ - { - "mean":0.537765, - "std":0.00116, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.648124, + "std":0.028677, "values":[ - 0.537095, - 0.539105, - 0.537095 + 0.645295, + 0.678112, + 0.620967 ] }, - { - "mean":0.599043, - "std":0.00114, + "auprc":{ + "mean":0.06421, + "std":0.024765, "values":[ - 0.598384, - 0.60036, - 0.598384 + 0.049071, + 0.092789, + 0.050771 ] } - ], - "oos_precision":{ - "mean":0.845467, - "std":0.010443, - "values":[ - 0.847761, - 0.854573, - 0.834068 - ] - }, - "oos_f1":{ - "mean":0.679964, - "std":0.004045, - "values":[ - 0.68024, - 0.683863, - 0.675788 - ] - }, - "oos_misroute_rate":{ - "mean":0.431333, - "std":0.001155, - "values":[ - 0.432, - 0.43, - 0.432 - ] - }, - "high_conf_oos_misroute_rate":{ - "mean":0.0, - "std":0.0, - "values":[ - 0.0, - 0.0, - 0.0 - ] - }, - "llm_call_rate":{ - "mean":0.502, - "std":0.011789, - "values":[ - 0.514727, - 0.499818, - 0.491455 - ] - }, - "llm_cost_usd_per_1k":{ - "mean":0.185303, - "std":0.004398, - "values":[ - 0.190043, - 0.184515, - 0.181352 - ] - }, - "error_recovery_rate":{ - "mean":0.739287, - "std":0.00752, - "values":[ - 0.746026, - 0.731175, - 0.740659 - ] - }, - "recoverable_caught":{ - "mean":1.0, - "std":0.0, - "values":[ - 1.0, - 1.0, - 1.0 - ] - }, - "small_errors":{ - "mean":2761.0, - "std":64.83826, - "values":[ - 2831, - 2749, - 2703 - ] - }, - "recoverable_errors":{ - "mean":2041.333333, - "std":61.32971, - "values":[ - 2112, - 2010, - 2002 - ] }, - "recovered_errors":{ - "mean":2041.333333, - "std":61.32971, - "values":[ - 2112, - 2010, - 2002 - ] + "test":{ + "auroc":{ + "mean":0.634548, + "std":0.011313, + "values":[ + 0.639109, + 0.642868, + 0.621667 + ] + }, + "auprc":{ + "mean":0.251264, + "std":0.011063, + "values":[ + 0.251379, + 0.262268, + 0.240143 + ] + } } }, "signals":{ @@ -51936,6 +60826,28 @@ "std":18.009257 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.813943, + "std":0.021869 + }, + "auprc":{ + "mean":0.130947, + "std":0.016434 + } + }, + "test":{ + "auroc":{ + "mean":0.813111, + "std":0.014313 + }, + "auprc":{ + "mean":0.461756, + "std":0.014934 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -52706,6 +61618,28 @@ "std":17.097758 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.823955, + "std":0.01844 + }, + "auprc":{ + "mean":0.135805, + "std":0.013668 + } + }, + "test":{ + "auroc":{ + "mean":0.81995, + "std":0.016252 + }, + "auprc":{ + "mean":0.466491, + "std":0.019912 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -53476,170 +62410,212 @@ ] }, "recovered_errors":{ - "mean":0.0, - "std":0.0, + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.730424, + "std":0.00564, + "values":[ + 0.724909, + 0.736182, + 0.730182 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.901576, + "std":0.001794, + "values":[ + 0.900364, + 0.903636, + 0.900727 + ] + }, + "oos_recall":{ + "mean":0.573333, + "std":0.00611, + "values":[ + 0.568, + 0.58, + 0.572 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.542457, + "std":0.006143, + "values":[ + 0.537095, + 0.54916, + 0.541115 + ] + }, + { + "mean":0.603649, + "std":0.00603, + "values":[ + 0.598384, + 0.610228, + 0.602334 + ] + } + ], + "oos_precision":{ + "mean":0.923726, + "std":0.005139, + "values":[ + 0.922078, + 0.929487, + 0.919614 + ] + }, + "oos_f1":{ + "mean":0.707519, + "std":0.005975, + "values":[ + 0.70297, + 0.714286, + 0.705302 + ] + }, + "oos_misroute_rate":{ + "mean":0.426667, + "std":0.00611, + "values":[ + 0.432, + 0.42, + 0.428 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.269576, + "std":0.00564, + "values":[ + 0.275091, + 0.263818, + 0.269818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.099261, + "std":0.002065, + "values":[ + 0.101283, + 0.097156, + 0.099344 + ] + }, + "error_recovery_rate":{ + "mean":0.634872, + "std":0.002868, + "values":[ + 0.637806, + 0.634735, + 0.632075 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1482.666667, + "std":31.021498, + "values":[ + 1513, + 1451, + 1484 + ] + }, + "recoverable_errors":{ + "mean":941.333333, + "std":22.188586, + "values":[ + 965, + 921, + 938 + ] + }, + "recovered_errors":{ + "mean":941.333333, + "std":22.188586, "values":[ - 0, - 0, - 0 + 965, + 921, + 938 ] } }, - "oracle":{ - "coverage":{ - "mean":0.730424, - "std":0.00564, - "values":[ - 0.724909, - 0.736182, - 0.730182 - ] - }, - "selective_risk":{ - "mean":0.0, - "std":0.0, - "values":[ - 0.0, - 0.0, - 0.0 - ] - }, - "accuracy_8":{ - "mean":0.901576, - "std":0.001794, - "values":[ - 0.900364, - 0.903636, - 0.900727 - ] - }, - "oos_recall":{ - "mean":0.573333, - "std":0.00611, - "values":[ - 0.568, - 0.58, - 0.572 - ] - }, - "oos_recall_wilson95":[ - { - "mean":0.542457, - "std":0.006143, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.818494, + "std":0.017615, "values":[ - 0.537095, - 0.54916, - 0.541115 + 0.810742, + 0.838655, + 0.806085 ] }, - { - "mean":0.603649, - "std":0.00603, + "auprc":{ + "mean":0.136921, + "std":0.016723, "values":[ - 0.598384, - 0.610228, - 0.602334 + 0.143431, + 0.14941, + 0.117922 ] } - ], - "oos_precision":{ - "mean":0.923726, - "std":0.005139, - "values":[ - 0.922078, - 0.929487, - 0.919614 - ] - }, - "oos_f1":{ - "mean":0.707519, - "std":0.005975, - "values":[ - 0.70297, - 0.714286, - 0.705302 - ] - }, - "oos_misroute_rate":{ - "mean":0.426667, - "std":0.00611, - "values":[ - 0.432, - 0.42, - 0.428 - ] - }, - "high_conf_oos_misroute_rate":{ - "mean":0.0, - "std":0.0, - "values":[ - 0.0, - 0.0, - 0.0 - ] - }, - "llm_call_rate":{ - "mean":0.269576, - "std":0.00564, - "values":[ - 0.275091, - 0.263818, - 0.269818 - ] - }, - "llm_cost_usd_per_1k":{ - "mean":0.099261, - "std":0.002065, - "values":[ - 0.101283, - 0.097156, - 0.099344 - ] - }, - "error_recovery_rate":{ - "mean":0.634872, - "std":0.002868, - "values":[ - 0.637806, - 0.634735, - 0.632075 - ] - }, - "recoverable_caught":{ - "mean":1.0, - "std":0.0, - "values":[ - 1.0, - 1.0, - 1.0 - ] - }, - "small_errors":{ - "mean":1482.666667, - "std":31.021498, - "values":[ - 1513, - 1451, - 1484 - ] }, - "recoverable_errors":{ - "mean":941.333333, - "std":22.188586, - "values":[ - 965, - 921, - 938 - ] - }, - "recovered_errors":{ - "mean":941.333333, - "std":22.188586, - "values":[ - 965, - 921, - 938 - ] + "test":{ + "auroc":{ + "mean":0.816019, + "std":0.012614, + "values":[ + 0.813686, + 0.829637, + 0.804734 + ] + }, + "auprc":{ + "mean":0.463049, + "std":0.013983, + "values":[ + 0.457256, + 0.478998, + 0.452893 + ] + } } }, "signals":{ @@ -54401,6 +63377,28 @@ "std":0.0 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.851095, + "std":0.0 + }, + "auprc":{ + "mean":0.200923, + "std":0.0 + } + }, + "test":{ + "auroc":{ + "mean":0.865454, + "std":0.0 + }, + "auprc":{ + "mean":0.580019, + "std":0.0 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -55171,6 +64169,28 @@ "std":0.0 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.872335, + "std":0.0 + }, + "auprc":{ + "mean":0.294041, + "std":0.0 + } + }, + "test":{ + "auroc":{ + "mean":0.874847, + "std":0.0 + }, + "auprc":{ + "mean":0.58504, + "std":0.0 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -55941,170 +64961,212 @@ ] }, "recovered_errors":{ - "mean":0.0, + "mean":0.0, + "std":0.0, + "values":[ + 0, + 0, + 0 + ] + } + }, + "oracle":{ + "coverage":{ + "mean":0.776182, + "std":0.0, + "values":[ + 0.776182, + 0.776182, + 0.776182 + ] + }, + "selective_risk":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "accuracy_8":{ + "mean":0.910545, + "std":0.0, + "values":[ + 0.910545, + 0.910545, + 0.910545 + ] + }, + "oos_recall":{ + "mean":0.588, + "std":0.0, + "values":[ + 0.588, + 0.588, + 0.588 + ] + }, + "oos_recall_wilson95":[ + { + "mean":0.557213, + "std":0.0, + "values":[ + 0.557213, + 0.557213, + 0.557213 + ] + }, + { + "mean":0.618113, + "std":0.0, + "values":[ + 0.618113, + 0.618113, + 0.618113 + ] + } + ], + "oos_precision":{ + "mean":0.945338, + "std":0.0, + "values":[ + 0.945338, + 0.945338, + 0.945338 + ] + }, + "oos_f1":{ + "mean":0.725031, + "std":0.0, + "values":[ + 0.725031, + 0.725031, + 0.725031 + ] + }, + "oos_misroute_rate":{ + "mean":0.412, + "std":0.0, + "values":[ + 0.412, + 0.412, + 0.412 + ] + }, + "high_conf_oos_misroute_rate":{ + "mean":0.0, + "std":0.0, + "values":[ + 0.0, + 0.0, + 0.0 + ] + }, + "llm_call_rate":{ + "mean":0.223818, + "std":0.0, + "values":[ + 0.223818, + 0.223818, + 0.223818 + ] + }, + "llm_cost_usd_per_1k":{ + "mean":0.082427, + "std":0.0, + "values":[ + 0.082427, + 0.082427, + 0.082427 + ] + }, + "error_recovery_rate":{ + "mean":0.600325, + "std":0.0, + "values":[ + 0.600325, + 0.600325, + 0.600325 + ] + }, + "recoverable_caught":{ + "mean":1.0, + "std":0.0, + "values":[ + 1.0, + 1.0, + 1.0 + ] + }, + "small_errors":{ + "mean":1231.0, + "std":0.0, + "values":[ + 1231, + 1231, + 1231 + ] + }, + "recoverable_errors":{ + "mean":739.0, + "std":0.0, + "values":[ + 739, + 739, + 739 + ] + }, + "recovered_errors":{ + "mean":739.0, "std":0.0, "values":[ - 0, - 0, - 0 + 739, + 739, + 739 ] } }, - "oracle":{ - "coverage":{ - "mean":0.776182, - "std":0.0, - "values":[ - 0.776182, - 0.776182, - 0.776182 - ] - }, - "selective_risk":{ - "mean":0.0, - "std":0.0, - "values":[ - 0.0, - 0.0, - 0.0 - ] - }, - "accuracy_8":{ - "mean":0.910545, - "std":0.0, - "values":[ - 0.910545, - 0.910545, - 0.910545 - ] - }, - "oos_recall":{ - "mean":0.588, - "std":0.0, - "values":[ - 0.588, - 0.588, - 0.588 - ] - }, - "oos_recall_wilson95":[ - { - "mean":0.557213, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.851095, "std":0.0, "values":[ - 0.557213, - 0.557213, - 0.557213 + 0.851095, + 0.851095, + 0.851095 ] }, - { - "mean":0.618113, + "auprc":{ + "mean":0.200923, "std":0.0, "values":[ - 0.618113, - 0.618113, - 0.618113 + 0.200923, + 0.200923, + 0.200923 ] } - ], - "oos_precision":{ - "mean":0.945338, - "std":0.0, - "values":[ - 0.945338, - 0.945338, - 0.945338 - ] - }, - "oos_f1":{ - "mean":0.725031, - "std":0.0, - "values":[ - 0.725031, - 0.725031, - 0.725031 - ] - }, - "oos_misroute_rate":{ - "mean":0.412, - "std":0.0, - "values":[ - 0.412, - 0.412, - 0.412 - ] - }, - "high_conf_oos_misroute_rate":{ - "mean":0.0, - "std":0.0, - "values":[ - 0.0, - 0.0, - 0.0 - ] - }, - "llm_call_rate":{ - "mean":0.223818, - "std":0.0, - "values":[ - 0.223818, - 0.223818, - 0.223818 - ] - }, - "llm_cost_usd_per_1k":{ - "mean":0.082427, - "std":0.0, - "values":[ - 0.082427, - 0.082427, - 0.082427 - ] - }, - "error_recovery_rate":{ - "mean":0.600325, - "std":0.0, - "values":[ - 0.600325, - 0.600325, - 0.600325 - ] - }, - "recoverable_caught":{ - "mean":1.0, - "std":0.0, - "values":[ - 1.0, - 1.0, - 1.0 - ] - }, - "small_errors":{ - "mean":1231.0, - "std":0.0, - "values":[ - 1231, - 1231, - 1231 - ] - }, - "recoverable_errors":{ - "mean":739.0, - "std":0.0, - "values":[ - 739, - 739, - 739 - ] }, - "recovered_errors":{ - "mean":739.0, - "std":0.0, - "values":[ - 739, - 739, - 739 - ] + "test":{ + "auroc":{ + "mean":0.865454, + "std":0.0, + "values":[ + 0.865454, + 0.865454, + 0.865454 + ] + }, + "auprc":{ + "mean":0.580019, + "std":0.0, + "values":[ + 0.580019, + 0.580019, + 0.580019 + ] + } } }, "signals":{ @@ -56866,6 +65928,28 @@ "std":40.447497 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.836389, + "std":0.010252 + }, + "auprc":{ + "mean":0.162329, + "std":0.014391 + } + }, + "test":{ + "auroc":{ + "mean":0.845126, + "std":0.008984 + }, + "auprc":{ + "mean":0.521826, + "std":0.024413 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -57640,6 +66724,28 @@ "std":26.501572 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.856416, + "std":0.011046 + }, + "auprc":{ + "mean":0.207669, + "std":0.058603 + } + }, + "test":{ + "auroc":{ + "mean":0.853316, + "std":0.010315 + }, + "auprc":{ + "mean":0.52119, + "std":0.040485 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -58576,6 +67682,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.840592, + "std":0.014968, + "values":[ + 0.854228, + 0.842972, + 0.824577 + ] + }, + "auprc":{ + "mean":0.169955, + "std":0.019965, + "values":[ + 0.185951, + 0.176334, + 0.147581 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.847369, + "std":0.00909, + "values":[ + 0.849894, + 0.854929, + 0.837284 + ] + }, + "auprc":{ + "mean":0.519449, + "std":0.026264, + "values":[ + 0.503918, + 0.549773, + 0.504657 + ] + } + } + }, "signals":{ "margin":{ "validation":{ @@ -59343,6 +68491,28 @@ "std":28.930952 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.777147, + "std":0.020783 + }, + "auprc":{ + "mean":0.097822, + "std":0.009835 + } + }, + "test":{ + "auroc":{ + "mean":0.769799, + "std":0.010664 + }, + "auprc":{ + "mean":0.38047, + "std":0.017713 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -60113,6 +69283,28 @@ "std":24.583192 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.800224, + "std":0.020828 + }, + "auprc":{ + "mean":0.124751, + "std":0.019955 + } + }, + "test":{ + "auroc":{ + "mean":0.782198, + "std":0.005997 + }, + "auprc":{ + "mean":0.390537, + "std":0.007046 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -61049,6 +70241,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.792571, + "std":0.033963, + "values":[ + 0.753573, + 0.808478, + 0.81566 + ] + }, + "auprc":{ + "mean":0.116308, + "std":0.028858, + "values":[ + 0.090706, + 0.147581, + 0.110637 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.777237, + "std":0.014088, + "values":[ + 0.761357, + 0.78212, + 0.788235 + ] + }, + "auprc":{ + "mean":0.384102, + "std":0.014778, + "values":[ + 0.368926, + 0.384932, + 0.398447 + ] + } + } + }, "signals":{ "margin":{ "validation":{ @@ -61808,6 +71042,28 @@ "std":14.571662 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.840429, + "std":0.003065 + }, + "auprc":{ + "mean":0.179597, + "std":0.003938 + } + }, + "test":{ + "auroc":{ + "mean":0.85541, + "std":0.004257 + }, + "auprc":{ + "mean":0.557197, + "std":0.007557 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -62582,6 +71838,28 @@ "std":12.342339 } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.859182, + "std":0.003913 + }, + "auprc":{ + "mean":0.25185, + "std":0.006557 + } + }, + "test":{ + "auroc":{ + "mean":0.863748, + "std":0.005356 + }, + "auprc":{ + "mean":0.562495, + "std":0.013928 + } + } + }, "signals":{ "margin":{ "validation":{ @@ -63518,6 +72796,48 @@ ] } }, + "oos_detection":{ + "validation":{ + "auroc":{ + "mean":0.840429, + "std":0.003065, + "values":[ + 0.843697, + 0.837618, + 0.839972 + ] + }, + "auprc":{ + "mean":0.179597, + "std":0.003938, + "values":[ + 0.17553, + 0.17987, + 0.183391 + ] + } + }, + "test":{ + "auroc":{ + "mean":0.85541, + "std":0.004257, + "values":[ + 0.860222, + 0.852136, + 0.853871 + ] + }, + "auprc":{ + "mean":0.557197, + "std":0.007557, + "values":[ + 0.55856, + 0.549052, + 0.56398 + ] + } + } + }, "signals":{ "margin":{ "validation":{ @@ -64047,5 +73367,118 @@ } } } + }, + "ablation_comparison":{ + "detection_aggregation":"argmax", + "detection_score":"1 - max in-scope probability at the fitted T", + "routers":"final router (validation-selected aggregation and signal)", + "oos_250":{ + "oos_detection_test":{ + "auroc":{ + "mean":0.983263, + "std":0.000662 + }, + "auprc":{ + "mean":0.943354, + "std":0.002453 + } + }, + "small_only_oos_recall":{ + "mean":0.610667, + "std":0.003786 + }, + "hybrid_test":{ + "0.02":{ + "oos_recall":{ + "mean":0.624333, + "std":0.013317 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.345333, + "std":0.055293 + }, + "llm_call_rate":{ + "mean":0.012727, + "std":0.015872 + }, + "accuracy_8":{ + "mean":0.921273, + "std":0.00142 + } + }, + "0.05":{ + "oos_recall":{ + "mean":0.613667, + "std":0.002082 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.383, + "std":0.001732 + }, + "llm_call_rate":{ + "mean":0.001576, + "std":0.000639 + }, + "accuracy_8":{ + "mean":0.919273, + "std":0.001311 + } + } + } + }, + "oos_0":{ + "oos_detection_test":{ + "auroc":{ + "mean":0.977988, + "std":0.00044 + }, + "auprc":{ + "mean":0.915221, + "std":0.002638 + } + }, + "small_only_oos_recall":{ + "mean":0.0, + "std":0.0 + }, + "hybrid_test":{ + "0.02":{ + "oos_recall":{ + "mean":0.406, + "std":0.01044 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.376667, + "std":0.029023 + }, + "llm_call_rate":{ + "mean":0.120727, + "std":0.007264 + }, + "accuracy_8":{ + "mean":0.883152, + "std":0.001794 + } + }, + "0.05":{ + "oos_recall":{ + "mean":0.005333, + "std":0.002082 + }, + "high_conf_oos_misroute_rate":{ + "mean":0.993333, + "std":0.002309 + }, + "llm_call_rate":{ + "mean":0.001212, + "std":0.00042 + }, + "accuracy_8":{ + "mean":0.810303, + "std":0.000735 + } + } + } + } } } diff --git a/src/tinyrouter/analysis.py b/src/tinyrouter/analysis.py index 88c2a5e..177ed58 100644 --- a/src/tinyrouter/analysis.py +++ b/src/tinyrouter/analysis.py @@ -25,6 +25,19 @@ ``(z1 - z2) / T``, which ranks rows exactly as the raw score margin does, so it is the "score margin" PLAN section 4.1 asks for on TF-IDF. +The threshold choosers (``selective.select_threshold`` and +``coverage_thresholds``) take a ``Scored`` built by ``Routed.scored``, so +the split name they check and the scores they use always come from the +same routed split; there is no way to pass a validation name with test +arrays short of building a ``Scored`` by hand. A test also replaces the +test logits with noise and checks that no choice moves. + +RQ2 OOS detection uses its own score, ``1 - max in-scope probability`` +at the fitted T (151 intents for argmax, 8 agents for summed); the four +confidence signals are for RQ3 selective prediction. Ranking OOS by low +confidence would count a confident ``oos`` prediction as the least +OOS-like row, which penalises exactly the models trained with OOS data. + Which signals a run gets (PLAN section 4.1): encoders all four; TF-IDF only ``msp_t`` and ``margin`` (its 100x cosine scale distorts the T = 1 signals and the raw ECE); majority none, and no temperature (every row @@ -39,15 +52,18 @@ import numpy as np from tinyrouter.calibrate import SplitLogits, TemperatureBoundWarning, fit_temperature +from tinyrouter.diagnostics import threshold_diagnostics from tinyrouter.labels import AGENTS, OOS, LabelSpace from tinyrouter.metrics import expected_calibration_error, log_softmax, softmax, wilson_interval from tinyrouter.selective import ( COVERAGE_GRID, + Scored, ThresholdChoice, aurc, auroc, average_precision, coverage_thresholds, + deferred_below, detection_counts, high_confidence_oos_misroute, require_validation, @@ -74,11 +90,16 @@ class Routed: pred: np.ndarray gold: np.ndarray signals: dict[str, np.ndarray] + oos_score: np.ndarray | None = None @property def error(self) -> np.ndarray: return self.pred != self.gold + def scored(self, signal: str, weight: np.ndarray | None = None) -> Scored: + """This split's ``signal`` and errors, the only input the threshold choosers accept.""" + return Scored(self.split, self.signals[signal], self.error, weight) + @property def gold_oos(self) -> np.ndarray: return self.gold == OOS_AGENT @@ -113,14 +134,17 @@ def route( if aggregation == "argmax": pred = space.agents_of(z.argmax(axis=1)) signals = signals_of(raw, cal, log_softmax(z / temperature)) + in_scope = np.delete(cal, space.oos_intent_id, axis=1) elif aggregation == "summed": raw8, cal8 = space.aggregate_probs(raw), space.aggregate_probs(cal) pred = cal8.argmax(axis=1) signals = signals_of(raw8, cal8, np.log(np.clip(cal8, 1e-300, None))) + in_scope = np.delete(cal8, OOS_AGENT, axis=1) else: raise ValueError(f"unknown aggregation {aggregation!r}; expected one of {AGGREGATIONS}") kept = {name: signals[name] for name in SIGNALS_BY_KIND[kind]} - return Routed(split.split, pred, space.agents_of(split.labels), kept) + oos_score = None if kind == "majority" else 1.0 - in_scope.max(axis=1) + return Routed(split.split, pred, space.agents_of(split.labels), kept, oos_score) def fit_run_temperature(val: SplitLogits, kind: str) -> tuple[float, dict[str, object]]: @@ -185,6 +209,21 @@ def signal_quality(routed: Routed, signal: str) -> dict[str, object]: } +def oos_detection(routed: Routed) -> dict[str, float] | None: + """RQ2 OOS detection with ``1 - max in-scope probability`` at T (None for majority). + + Unlike a confidence signal, this score is high when the model puts its + mass on oos, so a model trained with OOS rows is not penalised for + confidently predicting oos. + """ + if routed.oos_score is None: + return None + return { + "auroc": auroc(routed.oos_score, routed.gold_oos), + "auprc": average_precision(routed.oos_score, routed.gold_oos), + } + + def select_signal(val: Routed, choices: dict[str, ThresholdChoice]) -> str | None: """Signal with the largest validation coverage at the target (see module docstring).""" require_validation(val.split, "the hybrid's signal") @@ -214,15 +253,10 @@ def fallback( """RQ4: per target risk, every signal's validation-chosen threshold applied to test.""" out: dict[str, object] = {} for target in TARGET_RISKS: - choices = { - s: select_threshold(val.split, val.signals[s], val.error, target) for s in val.signals - } + choices = {s: select_threshold(val.scored(s), target) for s in val.signals} per_signal = {} for signal, choice in choices.items(): - confidence = test.signals[signal] - deferred = np.ones(confidence.size, dtype=bool) - if choice.tau is not None: - deferred = confidence < choice.tau + deferred = deferred_below(test.signals[signal], choice.tau) test_metrics = router_metrics(test, llm_pred, llm_cost, deferred) per_signal[signal] = {**choice_record(choice), "test": test_metrics} chosen = select_signal(val, choices) @@ -238,7 +272,7 @@ def operating_curve( val: Routed, test: Routed, llm_pred: np.ndarray, llm_cost: np.ndarray, signal: str ) -> dict[str, list[float | None]]: """For each validation coverage in the grid, its threshold and what it does on test.""" - taus = coverage_thresholds(val.split, val.signals[signal]) + taus = coverage_thresholds(val.scored(signal)) keys = ( "coverage", "selective_risk", @@ -249,7 +283,8 @@ def operating_curve( ) curve: dict[str, list[float | None]] = {"tau": taus, **{k: [] for k in keys}} for tau in taus: - metrics = router_metrics(test, llm_pred, llm_cost, test.signals[signal] < tau) + deferred = deferred_below(test.signals[signal], tau) + metrics = router_metrics(test, llm_pred, llm_cost, deferred) for k in keys: curve[k].append(metrics[k]) return curve @@ -264,6 +299,7 @@ def aggregation_block( "validation_accuracy_8": float(np.mean(~val.error)), "small_only": router_metrics(test, llm_pred, llm_cost, never), "oracle": router_metrics(test, llm_pred, llm_cost, test.error), + "oos_detection": {"validation": oos_detection(val), "test": oos_detection(test)}, "signals": { s: {"validation": signal_quality(val, s), "test": signal_quality(test, s)} for s in test.signals @@ -326,16 +362,21 @@ def analyze_run( for a, (v, t) in routed.items(): scalars[a], curves[a] = aggregation_block(v, t, llm_pred, llm_cost) scalars["final"] = final_view(scalars[selected]) + if kind == "encoder": + val_sel, test_sel = routed[selected] + scalars["diagnostics"] = threshold_diagnostics(val_sel, test_sel, scalars[selected]) return scalars, curves def final_view(block: dict[str, object]) -> dict[str, object]: """The selected aggregation's router numbers, without the per-signal detail.""" fallback_block = block["fallback"] - assert isinstance(fallback_block, dict) + if not isinstance(fallback_block, dict): + raise TypeError(f"fallback block is {type(fallback_block).__name__}, expected dict") return { "small_only": block["small_only"], "oracle": block["oracle"], + "oos_detection": block["oos_detection"], "signals": block["signals"], "fallback": { target: {"selected_signal": v["selected_signal"], "hybrid": v["hybrid"]} diff --git a/src/tinyrouter/analysis_run.py b/src/tinyrouter/analysis_run.py index 4ff45b2..81dd308 100644 --- a/src/tinyrouter/analysis_run.py +++ b/src/tinyrouter/analysis_run.py @@ -145,7 +145,12 @@ def load_haiku(results_root: Path) -> dict[str, HaikuSplit]: def combine(values: list, keep_values: bool = True) -> object: - """Seeds merged leaf by leaf: numbers -> mean, sample std (ddof=1) and the values.""" + """Seeds merged leaf by leaf: numbers -> mean, sample std (ddof=1) and the values. + + Seeds whose value is None (e.g. no feasible threshold) are left out of + the mean and counted in ``n``; with fewer than two numbers the std is + None, not 0. + """ if all(v is None for v in values): return None if all(isinstance(v, dict) for v in values): @@ -158,7 +163,7 @@ def combine(values: list, keep_values: bool = True) -> object: arr = np.array(numbers, dtype=np.float64) out: dict[str, object] = { "mean": float(arr.mean()), - "std": float(arr.std(ddof=1)) if arr.size > 1 else 0.0, + "std": float(arr.std(ddof=1)) if arr.size > 1 else None, } if len(numbers) < len(values): out["n"] = len(numbers) @@ -182,7 +187,8 @@ def rounded(value: object) -> object: def llm_only(haiku: HaikuSplit) -> dict[str, object]: summary = summarize_split(haiku)["new_parser"] - assert isinstance(summary, dict) + if not isinstance(summary, dict): + raise TypeError(f"Haiku summary is {type(summary).__name__}, expected dict") return { **summary, "coverage": 0.0, @@ -260,6 +266,16 @@ def protocol_record() -> dict[str, object]: "one_sided_z": ONE_SIDED_95_Z, "signal_choice": "largest validation coverage at the target, then lower validation AURC", "oracle": "defers exactly the queries the small model gets wrong (8-way)", + "oos_detection_score": ( + "1 - max in-scope probability at the fitted T (151 intents for argmax, 8 agents " + "for summed); the four confidence signals are for RQ3" + ), + "threshold_result": ( + "main result: thresholds chosen on validation only, reported as they do on test " + "(option a); diagnostics.sensitivity_reweighted_validation is option (b), " + "diagnosis only" + ), + "tie_at_tau": "a row whose confidence equals tau is kept by the small model", "error_recovery_rate": "small-model errors deferred and fixed by Haiku / small errors", "recoverable_caught": "errors deferred and fixed by Haiku / errors Haiku would fix", "high_conf_oos_misroute_rate": ( @@ -321,6 +337,42 @@ def input_record(results_root: Path, haiku_sha: str) -> dict[str, object]: } +ABLATION_PAIR = {"oos_250": "modernbert/k100", "oos_0": "modernbert-oos0/k100"} +ABLATION_AGGREGATION = "argmax" + + +def mean_std(value: dict | None) -> dict[str, object] | None: + return None if value is None else {"mean": value["mean"], "std": value["std"]} + + +def ablation_side(group: dict) -> dict[str, object]: + result = group["result"] + fixed = result[ABLATION_AGGREGATION]["oos_detection"]["test"] + final = result["final"] + routers = {} + for target, entry in final["fallback"].items(): + hybrid = entry["hybrid"]["test"] + routers[target] = { + k: mean_std(hybrid[k]) + for k in ("oos_recall", "high_conf_oos_misroute_rate", "llm_call_rate", "accuracy_8") + } + return { + "oos_detection_test": {k: mean_std(fixed[k]) for k in ("auroc", "auprc")}, + "small_only_oos_recall": mean_std(final["small_only"]["oos_recall"]), + "hybrid_test": routers, + } + + +def ablation_comparison(groups: dict[str, dict]) -> dict[str, object]: + """ModernBERT k=100 with 250 vs 0 OOS training rows: same score, same aggregation.""" + return { + "detection_aggregation": ABLATION_AGGREGATION, + "detection_score": "1 - max in-scope probability at the fitted T", + "routers": "final router (validation-selected aggregation and signal)", + **{side: ablation_side(groups[name]) for side, name in ABLATION_PAIR.items()}, + } + + def run(results_root: Path, log: Log = print) -> dict[str, int]: points = collect_points(results_root) haiku = load_haiku(results_root) @@ -336,6 +388,7 @@ def run(results_root: Path, log: Log = print) -> dict[str, int]: "llm_only_test": llm_only(haiku["test"]), "groups": {g: group_body(e, keep_values=True) for g, e in sorted(scalars.items())}, } + summary["ablation_comparison"] = ablation_comparison(summary["groups"]) write_checked(out / "summary.json", summary, check_groups(expected)) curve_body = { "format_version": FORMAT_VERSION, diff --git a/src/tinyrouter/diagnostics.py b/src/tinyrouter/diagnostics.py new file mode 100644 index 0000000..30eeb0a --- /dev/null +++ b/src/tinyrouter/diagnostics.py @@ -0,0 +1,117 @@ +"""Why validation-chosen thresholds miss the target risk on test (diagnosis, not method). + +CLINC150's validation split is 3.2% OOS (100 of 3,100) and test is 18.2% +(1,000 of 5,500). For each encoder point's final hybrid, this reports the +selective risk on validation and on test, split into in-scope rows and +kept OOS rows, and the test risk after reweighting test rows to the +validation OOS share. The share of the gap that reweighting closes is how +much the OOS proportion explains; the rest is test OOS being harder. + +``sensitivity_reweighted_validation`` is PLAN option (b): validation rows +reweighted to ``DECLARED_OOS_SHARE`` before choosing the threshold, then +applied to test. That share is the test split's, known only because we +looked at test, so this is a diagnostic sensitivity check and never the +reported method (the main result stays validation-only). + +Routed splits are used through their attributes only (``signals``, +``error``, ``gold_oos``, ``pred_oos``, ``scored``), so this module does +not import ``analysis``. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from tinyrouter.selective import deferred_below, select_threshold + +DECLARED_OOS_SHARE = 0.182 +SENSITIVITY_NOTE = ( + "validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking " + "at test) before choosing tau; diagnostic only, not the reported method" +) + + +def prior_weights(gold_oos: np.ndarray, share: float) -> np.ndarray: + """Per-row weights that make the weighted OOS share ``share``; they average to 1.""" + gold_oos = np.asarray(gold_oos, dtype=bool) + p = float(gold_oos.mean()) + if not 0 < p < 1 or not 0 < share < 1: + raise ValueError(f"need both classes and 0 < share < 1, got OOS share {p} and {share}") + return np.where(gold_oos, share / p, (1 - share) / (1 - p)) + + +def kept_rates(routed: Any, signal: str, tau: float | None, weight: np.ndarray | None = None): + """Coverage and error rates of the rows the small model keeps at ``tau`` (weighted).""" + kept = ~deferred_below(routed.signals[signal], tau) + w = np.ones(kept.size) if weight is None else np.asarray(weight, dtype=np.float64) + error, gold_oos = routed.error, routed.gold_oos + + def rate(mask: np.ndarray, hit: np.ndarray) -> float | None: + denominator = float(w[mask].sum()) + return float(w[mask & hit].sum() / denominator) if denominator > 0 else None + + return { + "coverage": float(w[kept].sum() / w.sum()), + "selective_risk": rate(kept, error), + "in_scope_risk": rate(kept & ~gold_oos, error), + "kept_oos_error_rate": rate(kept & gold_oos, error), + "kept_oos_share": rate(kept, gold_oos), + "high_conf_oos_misroute_rate": rate(gold_oos, kept & ~routed.pred_oos), + } + + +def gap_explained(val_risk: float | None, test_risk: float | None, reweighted: float | None): + """(test - reweighted) / (test - validation); None when undefined.""" + if None in (val_risk, test_risk, reweighted) or test_risk <= val_risk: + return None + return float((test_risk - reweighted) / (test_risk - val_risk)) + + +def sensitivity(val: Any, test: Any, signal: str, target: float) -> dict[str, object]: + weight = prior_weights(val.gold_oos, DECLARED_OOS_SHARE) + choice = select_threshold(val.scored(signal, weight), target) + return { + "note": SENSITIVITY_NOTE, + "tau": choice.tau, + "feasible": choice.feasible, + "validation_weighted_coverage": choice.coverage, + "validation_weighted_risk": choice.risk, + "test": kept_rates(test, signal, choice.tau), + } + + +def transfer(val: Any, test: Any, signal: str, tau: float | None) -> dict[str, object]: + val_share = float(val.gold_oos.mean()) + v = kept_rates(val, signal, tau) + t = kept_rates(test, signal, tau) + rw = kept_rates(test, signal, tau, prior_weights(test.gold_oos, val_share)) + return { + "validation_oos_share": val_share, + "test_oos_share": float(test.gold_oos.mean()), + "validation": v, + "test": t, + "test_reweighted_to_validation_oos_share": rw, + "share_of_gap_explained_by_oos_share": gap_explained( + v["selective_risk"], t["selective_risk"], rw["selective_risk"] + ), + } + + +def threshold_diagnostics(val: Any, test: Any, block: dict[str, Any]) -> dict[str, object]: + """Per target risk, the final hybrid's threshold transfer and the (b) sensitivity.""" + out: dict[str, object] = {} + for target, entry in block["fallback"].items(): + signal = entry["selected_signal"] + if signal is None: + out[target] = None + continue + tau = entry["hybrid"]["tau"] + out[target] = { + "signal": signal, + "tau": tau, + **transfer(val, test, signal, tau), + "sensitivity_reweighted_validation": sensitivity(val, test, signal, float(target)), + } + return out diff --git a/src/tinyrouter/selective.py b/src/tinyrouter/selective.py index 5ee9a9d..3b3fe11 100644 --- a/src/tinyrouter/selective.py +++ b/src/tinyrouter/selective.py @@ -35,10 +35,13 @@ def require_validation(split: str, what: str) -> None: def wilson_upper(successes: np.ndarray, n: np.ndarray, z: float) -> np.ndarray: - """Upper end of the Wilson score interval, elementwise; equals ``metrics.wilson_interval``.""" + """Upper end of the Wilson score interval, elementwise; equals ``metrics.wilson_interval``. + + Counts may be fractional (weighted rows); they must satisfy 0 <= successes <= n, n > 0. + """ k, m = np.asarray(successes, dtype=np.float64), np.asarray(n, dtype=np.float64) - if np.any(m < 1) or np.any(k < 0) or np.any(k > m): - raise ValueError("need 0 <= successes <= n and n >= 1") + if np.any(m <= 0) or np.any(k < 0) or np.any(k > m): + raise ValueError("need 0 <= successes <= n and n > 0") p, z2 = k / m, z * z centre = (p + z2 / (2 * m)) / (1 + z2 / m) half = z * np.sqrt(p * (1 - p) / m + z2 / (4 * m * m)) / (1 + z2 / m) @@ -144,6 +147,30 @@ def high_confidence_oos_misroute( return float(np.sum(gold_oos & ~pred_oos & accepted) / gold_oos.sum()) +@dataclass(frozen=True) +class Scored: + """One split's confidence scores and 8-way errors, taken together from one routed split. + + The threshold choosers take this object instead of a split name and + loose arrays, so the name they check and the numbers they use cannot + come from different splits. Build it with ``analysis.Routed.scored``. + ``weight`` (optional, one per row) reweights rows, e.g. to a declared + OOS share in the prior-shift diagnostic. + """ + + split: str + confidence: np.ndarray + error: np.ndarray + weight: np.ndarray | None = None + + def __post_init__(self) -> None: + shapes = {np.shape(self.confidence), np.shape(self.error)} + if self.weight is not None: + shapes.add(np.shape(self.weight)) + if len(shapes) != 1 or np.ndim(self.confidence) != 1 or np.size(self.confidence) == 0: + raise ValueError(f"confidence, error and weight must be equal 1-d arrays, got {shapes}") + + @dataclass(frozen=True) class ThresholdChoice: """The chosen confidence threshold and what it did on validation. @@ -165,24 +192,35 @@ def feasible(self) -> bool: return self.tau is not None +def deferred_below(confidence: np.ndarray, tau: float | None) -> np.ndarray: + """Rows sent to the LLM: confidence strictly below ``tau`` (a row equal to tau is kept). + + ``tau`` None (no feasible threshold) defers every row. + """ + confidence = np.asarray(confidence, dtype=np.float64) + if tau is None: + return np.ones(confidence.size, dtype=bool) + return confidence < tau + + def select_threshold( - split: str, - confidence: np.ndarray, - error: np.ndarray, - target_risk: float, - z: float = ONE_SIDED_95_Z, + scored: Scored, target_risk: float, z: float = ONE_SIDED_95_Z ) -> ThresholdChoice: """Largest-coverage threshold whose selective-risk Wilson upper bound is <= ``target_risk``. Candidates are the distinct validation scores; threshold t accepts - every row with confidence >= t. + every row with confidence >= t. With weights, the kept count and the + errors are weighted sums and coverage is the kept share of the total + weight. """ - require_validation(split, "a deferral threshold") - confidence, error = np.asarray(confidence, dtype=np.float64), np.asarray(error, dtype=bool) + require_validation(scored.split, "a deferral threshold") + confidence = np.asarray(scored.confidence, dtype=np.float64) + error = np.asarray(scored.error, dtype=bool) + weight = np.ones(confidence.size) if scored.weight is None else np.asarray(scored.weight, float) order, starts, sizes = _tie_groups(confidence) - accepted = np.cumsum(sizes) - errors = np.cumsum(np.add.reduceat(error[order].astype(np.float64), starts)) - upper = wilson_upper(errors, accepted, z) + accepted = np.cumsum(np.add.reduceat(weight[order], starts)) + errors = np.cumsum(np.add.reduceat((weight * error)[order], starts)) + upper = wilson_upper(np.minimum(errors, accepted), accepted, z) ok = np.flatnonzero(upper <= target_risk) if ok.size == 0: return ThresholdChoice(None, target_risk, 0.0, None, None, int(starts.size)) @@ -190,18 +228,16 @@ def select_threshold( return ThresholdChoice( tau=float(confidence[order][starts[best]]), target_risk=target_risk, - coverage=float(accepted[best] / confidence.size), + coverage=float(accepted[best] / weight.sum()), risk=float(errors[best] / accepted[best]), risk_upper=float(upper[best]), candidates=int(starts.size), ) -def coverage_thresholds( - split: str, confidence: np.ndarray, grid: tuple[float, ...] = COVERAGE_GRID -) -> list[float]: +def coverage_thresholds(scored: Scored, grid: tuple[float, ...] = COVERAGE_GRID) -> list[float]: """For each c in ``grid``, the score of the ceil(c * n)-th most confident validation row.""" - require_validation(split, "operating-curve thresholds") - ranked = np.sort(np.asarray(confidence, dtype=np.float64))[::-1] + require_validation(scored.split, "operating-curve thresholds") + ranked = np.sort(np.asarray(scored.confidence, dtype=np.float64))[::-1] n = ranked.size return [float(ranked[max(1, int(np.ceil(c * n - 1e-9))) - 1]) for c in grid] diff --git a/tests/test_analysis.py b/tests/test_analysis.py index c947188..fb8bfd8 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -1,19 +1,23 @@ import numpy as np import pytest +from sklearn.metrics import average_precision_score, roc_auc_score from tinyrouter.analysis import ( AGGREGATIONS, Routed, analyze_run, + fallback, fit_run_temperature, + oos_detection, route, router_metrics, select_aggregation, select_signal, + signal_quality, ) from tinyrouter.calibrate import LeakageError, SplitLogits from tinyrouter.labels import AGENTS, load_label_space -from tinyrouter.selective import select_threshold +from tinyrouter.selective import ThresholdChoice, select_threshold F, T, OOS = AGENTS.index("finance_agent"), AGENTS.index("travel_agent"), AGENTS.index("oos") SPACE = load_label_space() @@ -123,9 +127,7 @@ def test_signal_choice_prefers_coverage_then_aurc(): [F, F, F, F], {"msp": np.array([4.0, 3, 2, 1]), "entropy": np.array([1.0, 2, 3, 4])}, ) - choices = { - s: select_threshold("validation", val.signals[s], val.error, 0.8) for s in val.signals - } + choices = {s: select_threshold(val.scored(s), 0.8) for s in val.signals} # Both keep every row at this target; msp ranks the error last, so its AURC is lower. assert select_signal(val, choices) == "msp" @@ -207,23 +209,140 @@ def test_the_temperature_is_fitted_on_validation_only(kind): fit_run_temperature(synthetic_splits()["test"], kind) -def choices_of(scalars): +def choices_of(scalars, curves): + """Everything chosen on validation: T, aggregation, every tau, curve taus, (b) taus.""" fallback = { a: { - t: {s: v["tau"] for s, v in f["by_signal"].items()} + t: ({s: v["tau"] for s, v in f["by_signal"].items()}, f["selected_signal"]) for t, f in scalars[a]["fallback"].items() } for a in AGGREGATIONS } - return scalars["calibration"]["temperature"], scalars["selected_aggregation"], fallback + operating = {a: {s: c["operating"]["tau"] for s, c in curves[a].items()} for a in AGGREGATIONS} + sensitivity = { + t: (d["sensitivity_reweighted_validation"]["tau"], d["signal"], d["tau"]) + for t, d in scalars["diagnostics"].items() + } + return ( + scalars["calibration"]["temperature"], + scalars["selected_aggregation"], + fallback, + operating, + sensitivity, + ) def test_test_labels_do_not_move_any_choice(): splits = synthetic_splits() pred, cost = fake_llm(500) - before, _ = analyze_run(splits["validation"], splits["test"], "encoder", SPACE, pred, cost) + before = analyze_run(splits["validation"], splits["test"], "encoder", SPACE, pred, cost) shuffled = np.random.default_rng(9).permutation(splits["test"].labels) test = SplitLogits("test", splits["test"].logits, shuffled) - after, _ = analyze_run(splits["validation"], test, "encoder", SPACE, pred, cost) - assert choices_of(before) == choices_of(after) - assert before["final"]["small_only"] != after["final"]["small_only"] + after = analyze_run(splits["validation"], test, "encoder", SPACE, pred, cost) + assert choices_of(*before) == choices_of(*after) + assert before[0]["final"]["small_only"] != after[0]["final"]["small_only"] + + +def test_test_scores_do_not_move_any_choice(): + """Replacing the test logits with noise must leave every validation choice alone.""" + splits = synthetic_splits() + pred, cost = fake_llm(500) + before = analyze_run(splits["validation"], splits["test"], "encoder", SPACE, pred, cost) + noise = np.random.default_rng(11).normal(size=splits["test"].logits.shape).astype(np.float32) + test = SplitLogits("test", noise, splits["test"].labels) + after = analyze_run(splits["validation"], test, "encoder", SPACE, pred, cost) + assert choices_of(*before) == choices_of(*after) + assert before[1]["argmax"]["msp"]["risk_coverage"] != after[1]["argmax"]["msp"]["risk_coverage"] + + +def oos_rows_unsure(n=200, seed=3): + """A test split where every gold-oos row gets low confidence and in-scope rows high.""" + rng = np.random.default_rng(seed) + gold = np.where(rng.random(n) < 0.25, OOS, F) + confidence = np.where(gold == OOS, rng.uniform(0.1, 0.6, n), rng.uniform(0.4, 1.0, n)) + oos_score = np.where(gold == OOS, rng.uniform(0.3, 1.0, n), rng.uniform(0.0, 0.5, n)) + return routed("test", np.full(n, F), gold, {"msp": confidence}), confidence, oos_score + + +def test_low_confidence_on_oos_rows_gives_high_oos_auroc_matching_sklearn(): + r, confidence, _ = oos_rows_unsure() + quality = signal_quality(r, "msp") + gold_oos = r.gold == OOS + assert quality["oos_auroc"] > 0.5 + assert quality["oos_auroc"] == pytest.approx(roc_auc_score(gold_oos, -confidence)) + assert quality["oos_auprc"] == pytest.approx(average_precision_score(gold_oos, -confidence)) + + +def test_oos_detection_ranks_by_the_oos_score_itself(): + base, _, oos_score = oos_rows_unsure() + r = Routed(base.split, base.pred, base.gold, base.signals, oos_score) + got = oos_detection(r) + gold_oos = r.gold == OOS + assert got["auroc"] > 0.5 + assert got["auroc"] == pytest.approx(roc_auc_score(gold_oos, oos_score)) + assert got["auprc"] == pytest.approx(average_precision_score(gold_oos, oos_score)) + + +def test_oos_score_is_one_minus_the_largest_in_scope_probability_at_t(): + split = one_row_logits() + t = 2.0 + p = np.exp(split.logits[0] / t) / np.exp(split.logits[0] / t).sum() + in_scope = np.delete(p, SPACE.oos_intent_id).max() + got = route(split, t, "argmax", SPACE, "encoder").oos_score + assert got == pytest.approx([1 - in_scope], rel=1e-5) + assert route(split, t, "argmax", SPACE, "majority").oos_score is None + + +def test_signal_choice_by_hand_coverage_first_then_aurc(): + val = routed( + "validation", + [F, F, F, T], + [F, F, F, F], + {"msp": np.array([4.0, 3, 2, 1]), "entropy": np.array([1.0, 2, 3, 4])}, + ) + + def choice(coverage): + return ThresholdChoice(0.0, 0.05, coverage, 0.0, 0.0, 4) + + # entropy has the worse AURC (its error ranks first) but more coverage: it wins. + assert select_signal(val, {"msp": choice(0.5), "entropy": choice(0.75)}) == "entropy" + # Equal coverage: the lower validation AURC (msp) wins, whatever the dict order. + assert select_signal(val, {"entropy": choice(0.5), "msp": choice(0.5)}) == "msp" + + +def test_entropy_signal_is_lower_for_a_uniform_row_than_a_one_hot_row(): + z = np.full((2, SPACE.num_intents), 0.0) + z[1, 0] = 50.0 + split = SplitLogits("validation", z.astype(np.float32), np.array([0, 0])) + entropy = route(split, 1.0, "argmax", SPACE, "encoder").signals["entropy"] + assert entropy[0] < entropy[1] + assert entropy[0] == pytest.approx(-np.log(SPACE.num_intents)) + + +def test_summed_aggregation_uses_the_temperature_scaled_probabilities(): + """At T = 1 oos outweighs two finance intents; at T = 10 the pair wins.""" + names = SPACE.intent_names + finance = [i for i, n in enumerate(names) if SPACE.intent_to_agent[n] == "finance_agent"] + z = np.full((1, SPACE.num_intents), -1000.0) + z[0, SPACE.oos_intent_id] = 3.0 + z[0, finance[:2]] = 2.0 + split = SplitLogits("validation", z.astype(np.float32), np.array([SPACE.oos_intent_id])) + assert route(split, 1.0, "summed", SPACE, "encoder").pred.tolist() == [OOS] + assert route(split, 10.0, "summed", SPACE, "encoder").pred.tolist() == [F] + + +def test_a_test_row_exactly_at_tau_is_kept_by_the_small_model(monkeypatch): + import tinyrouter.analysis as analysis + + monkeypatch.setattr(analysis, "TARGET_RISKS", (0.6,)) + # Validation: two error-free rows (upper bound 0.575) then an error (3 rows: 0.746), + # so at target 0.6 tau is the second score, 0.8. + val = routed("validation", [F, F, T], [F, F, F], {"msp": np.array([0.9, 0.8, 0.1])}) + # Test row 0 sits exactly on tau and must be kept; row 1 is just below it. + test = routed("test", [F, T, OOS], [F, F, OOS], {"msp": np.array([0.8, 0.79, 0.95])}) + llm = np.array([F, F, OOS]) + got = fallback(val, test, llm, np.zeros(3))["0.60"]["by_signal"]["msp"] + assert got["tau"] == 0.8 + assert got["test"]["coverage"] == pytest.approx(2 / 3) + assert got["test"]["llm_call_rate"] == pytest.approx(1 / 3) + assert got["test"]["accuracy_8"] == 1.0 diff --git a/tests/test_analysis_run.py b/tests/test_analysis_run.py index 2ed4830..e19e511 100644 --- a/tests/test_analysis_run.py +++ b/tests/test_analysis_run.py @@ -246,3 +246,59 @@ def test_outputs_refuse_nan(tmp_path): with pytest.raises(ValueError): write_checked(out, {"x": float("nan")}, lambda _b: None) assert not out.exists() + + +def test_combine_reports_no_std_when_fewer_than_two_seeds_have_a_value(): + single = combine([None, 0.023, None]) + assert single["mean"] == pytest.approx(0.023) + assert single["std"] is None and single["n"] == 1 + assert single["values"] == [None, 0.023, None] + + +def test_two_indexes_pointing_at_one_archive_stop_the_analysis(tmp_path): + root = fake_index_root(tmp_path) + path = root / "curves" / "oos-ablation.json" + body = json.loads(path.read_text(encoding="utf-8")) + shared = json.loads((root / "curves" / "modernbert.json").read_text(encoding="utf-8")) + donor = shared["points"][0] + for field in ("run_name", "logits_file", "logits_sha256"): + body["points"][0][field] = donor[field] + path.write_text(json.dumps(body), encoding="utf-8") + with pytest.raises(IncompleteError, match="75 points on 74 archives"): + collect_points(root) + + +def stat(mean): + return {"mean": mean, "std": 0.0} + + +def fake_group(argmax_auroc, summed_auroc): + def block(auroc): + return {"oos_detection": {"test": {"auroc": stat(auroc), "auprc": stat(auroc / 2)}}} + + hybrid = { + k: stat(0.1) + for k in ("oos_recall", "high_conf_oos_misroute_rate", "llm_call_rate", "accuracy_8") + } + return { + "result": { + "argmax": block(argmax_auroc), + "summed": block(summed_auroc), + "final": { + "small_only": {"oos_recall": stat(0.6)}, + "fallback": {"0.02": {"hybrid": {"test": hybrid}}}, + }, + } + } + + +def test_ablation_detection_uses_one_fixed_aggregation_for_both_models(): + groups = { + "modernbert/k100": fake_group(0.98, 0.5), + "modernbert-oos0/k100": fake_group(0.97, 0.9), + } + out = analysis_run.ablation_comparison(groups) + assert out["detection_aggregation"] == "argmax" + assert out["oos_250"]["oos_detection_test"]["auroc"]["mean"] == 0.98 + assert out["oos_0"]["oos_detection_test"]["auroc"]["mean"] == 0.97 + assert out["oos_0"]["hybrid_test"]["0.02"]["llm_call_rate"]["mean"] == 0.1 diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py new file mode 100644 index 0000000..5a60b54 --- /dev/null +++ b/tests/test_diagnostics.py @@ -0,0 +1,96 @@ +import numpy as np +import pytest + +from tinyrouter.analysis import Routed +from tinyrouter.diagnostics import ( + DECLARED_OOS_SHARE, + gap_explained, + kept_rates, + prior_weights, + sensitivity, + threshold_diagnostics, + transfer, +) +from tinyrouter.labels import AGENTS + +F, T, OOS = AGENTS.index("finance_agent"), AGENTS.index("travel_agent"), AGENTS.index("oos") + + +def routed(split, pred, gold, msp): + return Routed(split, np.array(pred), np.array(gold), {"msp": np.array(msp, dtype=float)}) + + +def test_prior_weights_hit_the_share_and_average_one(): + gold = np.array([True, False, False, False]) + w = prior_weights(gold, 0.5) + assert w.tolist() == pytest.approx([2.0, 2 / 3, 2 / 3, 2 / 3]) + assert w.mean() == pytest.approx(1.0) + assert w[gold].sum() / w.sum() == pytest.approx(0.5) + + +def test_kept_rates_by_hand(): + # Kept at tau 0.5: rows 0-2. Row 1 in-scope error, row 2 kept OOS routed to finance. + r = routed("test", [F, T, F, OOS], [F, F, OOS, OOS], [0.9, 0.8, 0.7, 0.2]) + got = kept_rates(r, "msp", 0.5) + assert got["coverage"] == pytest.approx(3 / 4) + assert got["selective_risk"] == pytest.approx(2 / 3) + assert got["in_scope_risk"] == pytest.approx(1 / 2) + assert got["kept_oos_error_rate"] == 1.0 + assert got["kept_oos_share"] == pytest.approx(1 / 3) + assert got["high_conf_oos_misroute_rate"] == pytest.approx(1 / 2) + + +def test_reweighting_moves_risk_by_the_oos_share_by_hand(): + r = routed("test", [F, T, F, OOS], [F, F, OOS, OOS], [0.9, 0.8, 0.7, 0.2]) + # Weights for a 25% OOS share: oos rows 0.5, in-scope rows 1.5. Kept weight 3.5, + # errors 1.5 (row 1) + 0.5 (row 2). + got = kept_rates(r, "msp", 0.5, prior_weights(r.gold == OOS, 0.25)) + assert got["selective_risk"] == pytest.approx(2.0 / 3.5) + + +def test_gap_explained_by_hand_and_undefined_cases(): + assert gap_explained(0.015, 0.0734, 0.0229) == pytest.approx((0.0734 - 0.0229) / 0.0584) + assert gap_explained(0.05, 0.04, 0.03) is None + assert gap_explained(None, 0.04, 0.03) is None + + +def small_pair(): + rng = np.random.default_rng(0) + n_val, n_test = 400, 400 + val_gold = np.where(rng.random(n_val) < 0.05, OOS, F) + test_gold = np.where(rng.random(n_test) < 0.3, OOS, F) + val = routed("validation", np.full(n_val, F), val_gold, np.where(val_gold == OOS, 0.6, 0.9)) + test = routed("test", np.full(n_test, F), test_gold, np.where(test_gold == OOS, 0.6, 0.9)) + return val, test + + +def test_transfer_explains_a_pure_prior_shift_completely(): + val, test = small_pair() + got = transfer(val, test, "msp", 0.5) + assert got["validation_oos_share"] < got["test_oos_share"] + rw = got["test_reweighted_to_validation_oos_share"]["selective_risk"] + assert rw == pytest.approx(got["validation"]["selective_risk"]) + assert got["share_of_gap_explained_by_oos_share"] == pytest.approx(1.0) + + +def test_sensitivity_chooses_tau_on_reweighted_validation_only(): + val, test = small_pair() + got = sensitivity(val, test, "msp", 0.05) + # With OOS reweighted to 18.2%, keeping the OOS rows breaks 5%, so only 0.9 is kept. + assert got["tau"] == 0.9 + assert got["validation_weighted_coverage"] == pytest.approx(1 - DECLARED_OOS_SHARE) + assert got["test"]["kept_oos_share"] == 0.0 + + +def test_diagnostics_follow_the_final_hybrid_and_skip_a_run_without_signals(): + val, test = small_pair() + block = { + "fallback": { + "0.05": {"selected_signal": "msp", "hybrid": {"tau": 0.5}}, + "0.02": {"selected_signal": None, "hybrid": None}, + } + } + got = threshold_diagnostics(val, test, block) + assert got["0.02"] is None + assert got["0.05"]["tau"] == 0.5 and got["0.05"]["signal"] == "msp" + assert got["0.05"]["validation"]["coverage"] == 1.0 diff --git a/tests/test_selective.py b/tests/test_selective.py index ea62b87..610356c 100644 --- a/tests/test_selective.py +++ b/tests/test_selective.py @@ -6,10 +6,12 @@ from tinyrouter.metrics import wilson_interval from tinyrouter.selective import ( ONE_SIDED_95_Z, + Scored, aurc, auroc, average_precision, coverage_thresholds, + deferred_below, detection_counts, high_confidence_oos_misroute, risk_at_coverages, @@ -95,7 +97,7 @@ def test_select_threshold_matches_a_brute_force_search(target): rng = np.random.default_rng(7) confidence = np.round(rng.random(400), 2) error = rng.random(400) < 0.3 * (1 - confidence) - choice = select_threshold("validation", confidence, error, target) + choice = select_threshold(Scored("validation", confidence, error), target) expected = brute_force_threshold(confidence, error, target) if expected is None: assert not choice.feasible and choice.coverage == 0.0 @@ -105,33 +107,39 @@ def test_select_threshold_matches_a_brute_force_search(target): def test_select_threshold_accepts_everything_when_the_target_is_loose(): - choice = select_threshold("validation", np.array([3.0, 2.0, 1.0]), np.array([0, 1, 1]), 1.0) + choice = select_threshold( + Scored("validation", np.array([3.0, 2.0, 1.0]), np.array([0, 1, 1])), 1.0 + ) assert choice.tau == 1.0 and choice.coverage == 1.0 and choice.risk == pytest.approx(2 / 3) def test_select_threshold_reports_an_unreachable_target(): - choice = select_threshold("validation", np.array([3.0, 2.0]), np.array([0, 0]), 1e-4) + choice = select_threshold(Scored("validation", np.array([3.0, 2.0]), np.array([0, 0])), 1e-4) assert choice.tau is None and not choice.feasible and choice.coverage == 0.0 def test_select_threshold_never_splits_tied_scores(): # Upper bounds: 1 row, 0 errors 0.73; 2 rows, 1 error 0.88; 3 rows, 1 error 0.746. # Splitting the tie would keep the error-free first row alone at target 0.74. - choice = select_threshold("validation", np.array([1.0, 1.0, 0.0]), np.array([0, 1, 0]), 0.74) + choice = select_threshold( + Scored("validation", np.array([1.0, 1.0, 0.0]), np.array([0, 1, 0])), 0.74 + ) assert choice.tau is None @pytest.mark.parametrize("split", ["test", "train"]) def test_threshold_and_curve_choices_refuse_any_split_but_validation(split): with pytest.raises(LeakageError, match=split): - select_threshold(split, np.array([1.0, 0.0]), np.array([0, 1]), 0.05) + select_threshold(Scored(split, np.array([1.0, 0.0]), np.array([0, 1])), 0.05) with pytest.raises(LeakageError, match=split): - coverage_thresholds(split, np.array([1.0, 0.0])) + coverage_thresholds(Scored(split, np.array([1.0, 0.0]), np.array([0, 1]))) def test_coverage_thresholds_are_the_validation_scores_at_each_coverage(): scores = np.array([0.1, 0.9, 0.5, 0.7]) - assert coverage_thresholds("validation", scores, (0.25, 0.5, 1.0)) == [0.9, 0.7, 0.1] + assert coverage_thresholds( + Scored("validation", scores, np.zeros(4, bool)), (0.25, 0.5, 1.0) + ) == [0.9, 0.7, 0.1] def test_detection_counts_by_hand_and_zero_precision_when_nothing_is_predicted(): @@ -154,5 +162,36 @@ def test_the_bound_is_one_sided_95_percent(): assert ONE_SIDED_95_Z == pytest.approx(norm.ppf(0.95)) # 60 rows, no error: upper 0.043 one-sided, 0.060 two-sided; the target sits between. - choice = select_threshold("validation", np.arange(60.0), np.zeros(60, bool), 0.05) + choice = select_threshold(Scored("validation", np.arange(60.0), np.zeros(60, bool)), 0.05) assert choice.coverage == 1.0 + + +def test_a_row_exactly_at_tau_is_kept_and_no_tau_defers_everything(): + confidence = np.array([0.9, 0.5, 0.4999]) + assert deferred_below(confidence, 0.5).tolist() == [False, False, True] + assert deferred_below(confidence, None).tolist() == [True, True, True] + + +def test_scored_refuses_arrays_that_do_not_line_up(): + with pytest.raises(ValueError, match="equal"): + Scored("validation", np.zeros(3), np.zeros(2, bool)) + with pytest.raises(ValueError, match="equal"): + Scored("validation", np.zeros(3), np.zeros(3, bool), np.ones(4)) + + +def test_weighted_threshold_counts_weighted_rows_by_hand(): + # Rows by confidence: ok, ok, error. Error weight 3, others 0.5 each (total 4). + scored = Scored( + "validation", + np.array([3.0, 2.0, 1.0]), + np.array([0, 0, 1], bool), + np.array([0.5, 0.5, 3.0]), + ) + loose = select_threshold(scored, 1.0) + assert loose.coverage == 1.0 and loose.risk == pytest.approx(3 / 4) + # Kept weight 1 with no error: upper bound z^2 / (1 + z^2) = 0.73. + upper = ONE_SIDED_95_Z**2 / (1 + ONE_SIDED_95_Z**2) + mid = select_threshold(scored, upper + 1e-9) + assert mid.tau == 2.0 and mid.coverage == pytest.approx(1 / 4) + unweighted = select_threshold(Scored("validation", scored.confidence, scored.error), 0.75) + assert unweighted.coverage == 1.0 From 1109acb867c5ed706371c367a46ea643eaca0c0f Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:55:00 +0800 Subject: [PATCH 4/5] Test that a deferred OOS row is not a high-confidence misroute The diagnostics hand case had no OOS row routed to an agent and deferred, so dropping the kept condition went unnoticed. --- tests/test_diagnostics.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 5a60b54..24bb4ae 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -30,14 +30,15 @@ def test_prior_weights_hit_the_share_and_average_one(): def test_kept_rates_by_hand(): # Kept at tau 0.5: rows 0-2. Row 1 in-scope error, row 2 kept OOS routed to finance. - r = routed("test", [F, T, F, OOS], [F, F, OOS, OOS], [0.9, 0.8, 0.7, 0.2]) + # Row 4 is OOS routed to finance too, but deferred, so it is not a high-confidence misroute. + r = routed("test", [F, T, F, OOS, F], [F, F, OOS, OOS, OOS], [0.9, 0.8, 0.7, 0.2, 0.1]) got = kept_rates(r, "msp", 0.5) - assert got["coverage"] == pytest.approx(3 / 4) + assert got["coverage"] == pytest.approx(3 / 5) assert got["selective_risk"] == pytest.approx(2 / 3) assert got["in_scope_risk"] == pytest.approx(1 / 2) assert got["kept_oos_error_rate"] == 1.0 assert got["kept_oos_share"] == pytest.approx(1 / 3) - assert got["high_conf_oos_misroute_rate"] == pytest.approx(1 / 2) + assert got["high_conf_oos_misroute_rate"] == pytest.approx(1 / 3) def test_reweighting_moves_risk_by_the_oos_share_by_hand(): From 09baa9f7c6f58101faf859f9537857527bc3626d Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:16:41 +0800 Subject: [PATCH 5/5] Use Kish effective sample size for weighted thresholds; pin tie-break and summed OOS score The weighted threshold (used only by the prior-shift sensitivity) now bounds the weighted error rate with n_eff = (sum w)^2 / sum w^2 of the kept rows instead of the nominal weighted count. Taus of seeds that chose different signals keep per-seed values and signal names instead of a mean. New tests: equal coverage goes to the lower AURC even when that signal is listed later, and the summed OOS score is high when the oos agent carries the mass. --- DEVLOG.md | 29 +- results/analysis/summary.json | 2215 ++++++++++++++++++++------------ src/tinyrouter/analysis_run.py | 34 +- src/tinyrouter/selective.py | 15 +- tests/test_analysis.py | 24 + tests/test_analysis_run.py | 26 + tests/test_selective.py | 33 +- 7 files changed, 1543 insertions(+), 833 deletions(-) diff --git a/DEVLOG.md b/DEVLOG.md index 6a77ce4..a9ff26b 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -4,13 +4,38 @@ --- +## 2026-09-29（深夜，三）：PR #17 複查小修（R1 到 R4） + +### 本次工作 / 執行摘要 +- 複查確認主結果（(a)、消融、歸因）與洩漏實驗無誤，獨立重算到第 6 位一致；剩下三個 MEDIUM 與一個 LOW。 +- **R1**：新增測試，coverage 平手時 AURC 較好的訊號（`msp_t`）在 `SIGNALS` 裡排在較差的（`msp`）後面，仍要選到它。原本的測試裡較好的剛好排第一，拿掉 AURC 判定也會過。 +- **R2**：新增 summed 聚合的 `oos_score` 測試：oos agent 拿走大部分機率時，分數要接近 1，且等於 1 減最大 in-scope agent 機率。 +- **R3**：`select_threshold` 的加權分支（只有 (b) 用到）改用 Kish 有效樣本數：每個切點 p = 加權誤判率、n_eff = (Σw)²/Σw²（只算被收下的列）、k = p·n_eff 代入 Wilson。原本用名目加權計數，100 筆 OOS 權重約 5.64 時偏樂觀。權重全為 1 時與未加權結果相同（有測試）；權重不均時上界比名目計數寬（有測試）。 +- **R4**：三個 seed 選到的訊號不同時，τ 不再算 mean 與 std，只保留逐 seed 的值與訊號名（final hybrid、各聚合的 hybrid、diagnostics 與 (b) 的 τ）。`by_signal` 底下同一個訊號的 τ 照常平均。 + +### 核心發現 / 數據 +- (b) 敏感度（ModernBERT k=100，目標 2%）：test risk 2.60 ± 0.48（2.17、3.12、2.50），coverage 87.33 ± 1.85；原本 2.93 ± 0.63、88.55 ± 1.76。結論不變：依部署 OOS 比例加權後仍略高於 2%。 +- `summary.json` 逐欄比對：變動只在 (b) 的欄位（`diagnostics.*.sensitivity_reweighted_validation`）與 R4 的 τ 表示方式；其他數字全部不變，`curves.json` 逐位元組相同。 + +### Blockers / 遇到的問題 +- (無) + +### Next +- [ ] 同上一則 + +### Files / Budget +- `src/tinyrouter/selective.py`、`analysis_run.py`；`tests/test_selective.py`、`test_analysis.py`、`test_analysis_run.py`；`results/analysis/summary.json`；`DEVLOG.md` +- API 花費：US$0 + +--- + ## 2026-09-29（深夜，二）：PR #17 審查修正（守門缺口與兩個會誤導的結論） ### 本次工作 / 執行摘要 - 審查結論：計算正確（無洩漏、可逐位元重現、獨立重算吻合），退回原因是 9 條突變存活與兩個報告結論。這次一次修完 F1 到 F11。 - **守門不再只看 split 名稱（F2）**：`select_threshold`、`coverage_thresholds` 改成只收 `Scored`，由 `Routed.scored(signal)` 從同一個 routed split 取出名稱、分數與錯誤，不能再「傳 validation 名稱配 test 陣列」。打亂 labels 的測試之外，新增「test logits 換成亂數」的測試，並把操作曲線的 τ、hybrid 的訊號、(b) 敏感度的 τ 都納入比對。 - **RQ2 OOS 偵測改用專用分數（F3）**：`1 − max in-scope 機率`（擬合的 T 下；argmax 聚合用 151 類、summed 用 8 類），寫在每個聚合的 `oos_detection`。四個信心訊號只留給 RQ3。`summary.json` 新增 `ablation_comparison`：偵測固定用 argmax 聚合，router 行為用 final router，全部由程式產生。 -- **門檻失效的診斷（F4）**：`diagnostics.py`，對每個 encoder 點的 final hybrid 報 val 與 test 的 selective risk、只看 in-scope 的 risk、被收下 OOS 的誤判率、test 依 validation OOS 比例重新加權後的 risk、比例解釋的缺口比例，另報選項 (b) 的敏感度（validation 依 18.2% OOS 加權後選 τ 再套 test，註明這個比例是看過 test 才知道的，只作診斷）。加權的 Wilson 上界用加權後的計數。 +- **門檻失效的診斷（F4）**：`diagnostics.py`，對每個 encoder 點的 final hybrid 報 val 與 test 的 selective risk、只看 in-scope 的 risk、被收下 OOS 的誤判率、test 依 validation OOS 比例重新加權後的 risk、比例解釋的缺口比例，另報選項 (b) 的敏感度（validation 依 18.2% OOS 加權後選 τ 再套 test，註明這個比例是看過 test 才知道的，只作診斷）。加權的 Wilson 上界在複查時改用 Kish 有效樣本數（見「深夜，三」那一則）。 - **決定（協調者依 Drew 的專案原則拍板）**：主結果採 (a) 照實報，validation-only 協定不變；(b) 只當診斷。 - 其他：F1 OOS AUROC／AUPRC 方向的呼叫端測試（對 sklearn）；F5 `select_signal` 用手算 choices 測；F6 entropy 方向；F7 summed 用溫度校準機率（T 會改變 argmax 的案例）；F8 分數恰好等於 τ 的列由小模型保留（`deferred_below`，有測試）；F9 可行 seed 少於 2 個時 std 記 null 並保留 n（例：bert/k1 目標 2% 只有 1 個 seed 可行）；F10 兩個 index 指向同一個 archive 會紅；F11 正式程式碼的 `assert isinstance` 改成明確 raise。 @@ -19,7 +44,7 @@ - **消融改正後**（偵測固定 argmax 聚合、分數 `1 − max in-scope p`）：OOS 250 筆 AUROC 98.33 ± 0.07、AUPRC 94.34 ± 0.25；OOS 0 筆 97.80 ± 0.04、91.52 ± 0.26。有 OOS 訓練資料的模型較好，與前一則的方向相反。 - **消融的主比較是 router 行為**（目標 2%，OOS 250 對 0）：hybrid OOS recall 62.43 ± 1.33 對 40.60 ± 1.04；高信心 OOS 誤派 34.53 ± 5.53 對 37.67 ± 2.90；LLM 呼叫率 1.27 ± 1.59 對 12.07 ± 0.73。沒有 OOS 訓練資料時，要多交 LLM 近 10 倍的查詢，OOS 仍少攔約 22pp。 - **門檻失效的歸因**（ModernBERT k=100，目標 2%）：val 1.50 ± 0.13，test 7.34 ± 0.86，test 依 validation OOS 比例加權後 2.29 ± 0.11，比例解釋缺口的 86.66 ± 1.39%。其餘來自 test 的 OOS 較難：被收下 OOS 的誤判率 val 19.91 ± 2.93 對 test 36.48 ± 3.04；in-scope risk 0.90 ± 0.23 對 1.20 ± 0.07。目標 2% 下其他 encoder 點比例解釋約 78% 到 97%（BERT k=5、k=10 超過 100%，加權後反而低於 val）；目標 5% 的 ModernBERT k=1 只有 27%，那一點 coverage 很低、數字不穩。 -- **(b) 敏感度**（同點）：test risk 2.93 ± 0.63（三個 seed 2.58、3.65、2.55），coverage 88.55 ± 1.76。依部署 OOS 比例加權後仍略高於 2%。 +- **(b) 敏感度**（同點，已依複查改用 Kish 有效樣本數重算）：test risk 2.60 ± 0.48（三個 seed 2.17、3.12、2.50），coverage 87.33 ± 1.85（85.45、89.15、87.38）。依部署 OOS 比例加權後仍略高於 2%。 - **結論**：validation 選的門檻在 test 上守不住目標 risk，主要來自 OOS 比例差異（約 86%），其餘是 test OOS 較難；依部署 OOS 比例加權仍不足，部署時應在接近真實流量的資料上重新校準門檻。 - 重構後 `curves.json` 與改前逐位元組相同（所有 τ 與曲線不變），`summary.json` 除新增欄位外只有 48 處 std 由 0 改成 null（F9）；`make analysis` 連跑兩次輸出相同。 diff --git a/results/analysis/summary.json b/results/analysis/summary.json index 84cc4c0..2a85a34 100644 --- a/results/analysis/summary.json +++ b/results/analysis/summary.json @@ -503,7 +503,19 @@ "msp_t" ], "hybrid":{ - "tau":null, + "tau":{ + "values":[ + null, + null, + null + ], + "signals":[ + "margin", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" + }, "feasible":false, "validation_coverage":{ "mean":0.0, @@ -908,9 +920,17 @@ ], "hybrid":{ "tau":{ - "mean":1.954367, - "std":2.481873, - "n":2 + "values":[ + null, + 0.199418, + 3.709316 + ], + "signals":[ + "margin", + "msp", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":[ false, @@ -1708,9 +1728,17 @@ ], "hybrid":{ "tau":{ - "mean":0.401006, - "std":null, - "n":1 + "values":[ + null, + 0.401006, + null + ], + "signals":[ + "msp_t", + "msp", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":[ false, @@ -2172,8 +2200,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.340128, - "std":1.222357 + "values":[ + -1.751573, + 0.370436, + 0.360752 + ], + "signals":[ + "entropy", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -3262,14 +3299,17 @@ ], "hybrid":{ "tau":{ - "mean":0.401006, - "std":null, - "n":1, "values":[ null, 0.401006, null - ] + ], + "signals":[ + "msp_t", + "msp", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":[ false, @@ -3473,13 +3513,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.340128, - "std":1.222357, "values":[ -1.751573, 0.370436, 0.360752 - ] + ], + "signals":[ + "entropy", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -3678,14 +3722,17 @@ "margin" ], "tau":{ - "mean":0.401006, - "std":null, - "n":1, "values":[ null, 0.401006, null - ] + ], + "signals":[ + "msp_t", + "msp", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -3889,14 +3936,17 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.401006, - "std":null, - "n":1, "values":[ null, 0.401006, null - ] + ], + "signals":[ + "msp_t", + "msp", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":[ false, @@ -3991,13 +4041,17 @@ "msp" ], "tau":{ - "mean":-0.340128, - "std":1.222357, "values":[ -1.751573, 0.370436, 0.360752 - ] + ], + "signals":[ + "entropy", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -4197,14 +4251,17 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.399272, - "std":0.001517, - "n":2, "values":[ null, 0.3982, - 0.400345 - ] + 0.398901 + ], + "signals":[ + "entropy", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":[ false, @@ -4212,52 +4269,52 @@ true ], "validation_weighted_coverage":{ - "mean":0.029721, - "std":0.028937, + "mean":0.030175, + "std":0.028987, "values":[ 0.0, 0.057805, - 0.031357 + 0.03272 ] }, "validation_weighted_risk":{ - "mean":0.006706, - "std":0.002813, + "mean":0.010692, + "std":0.00845, "n":2, "values":[ null, 0.004717, - 0.008696 + 0.016667 ] }, "test":{ "coverage":{ - "mean":0.028121, - "std":0.025809, + "mean":0.028667, + "std":0.026001, "values":[ 0.0, 0.050727, - 0.033636 + 0.035273 ] }, "selective_risk":{ - "mean":0.042275, - "std":0.024303, + "mean":0.043473, + "std":0.025998, "n":2, "values":[ null, 0.02509, - 0.059459 + 0.061856 ] }, "in_scope_risk":{ - "mean":0.018525, - "std":0.020942, + "mean":0.017816, + "std":0.019939, "n":2, "values":[ null, 0.003717, - 0.033333 + 0.031915 ] }, "kept_oos_error_rate":{ @@ -4271,22 +4328,22 @@ ] }, "kept_oos_share":{ - "mean":0.031435, - "std":0.006233, + "mean":0.033385, + "std":0.003475, "n":2, "values":[ null, 0.035842, - 0.027027 + 0.030928 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.003667, - "std":0.003215, + "mean":0.004, + "std":0.003464, "values":[ 0.0, 0.006, - 0.005 + 0.006 ] } } @@ -4703,8 +4760,17 @@ ], "hybrid":{ "tau":{ - "mean":0.539077, - "std":0.411834 + "values":[ + 0.065162, + 0.741964, + 0.810105 + ], + "signals":[ + "msp", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -5923,8 +5989,17 @@ ], "hybrid":{ "tau":{ - "mean":1.621489, - "std":1.281221 + "values":[ + 0.893141, + 0.870469, + 3.100858 + ], + "signals":[ + "msp_t", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -7664,13 +7739,17 @@ ], "hybrid":{ "tau":{ - "mean":0.20164, - "std":0.286486, "values":[ 0.532444, 0.03688, 0.035594 - ] + ], + "signals":[ + "msp_t", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -8071,86 +8150,86 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.952936, - "std":0.038084, + "mean":0.968823, + "std":0.025388, "values":[ 0.994706, 0.943961, - 0.920141 + 0.967803 ] }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.307911, - "std":0.084763, + "mean":0.270035, + "std":0.053582, "values":[ 0.216409, 0.323573, - 0.383751 + 0.270124 ] }, "validation_weighted_risk":{ - "mean":0.011627, - "std":0.001428, + "mean":0.009895, + "std":0.001581, "values":[ 0.01093, 0.010681, - 0.013269 + 0.008075 ] }, "test":{ "coverage":{ - "mean":0.299758, - "std":0.08497, + "mean":0.265818, + "std":0.056484, "values":[ 0.206727, 0.319273, - 0.373273 + 0.271455 ] }, "selective_risk":{ - "mean":0.017365, - "std":0.006698, + "mean":0.017203, + "std":0.006536, "values":[ 0.009675, 0.020501, - 0.021919 + 0.021433 ] }, "in_scope_risk":{ - "mean":0.002425, - "std":0.002809, + "mean":0.002636, + "std":0.003157, "values":[ 0.0, 0.001773, - 0.005503 + 0.006135 ] }, "kept_oos_error_rate":{ - "mean":0.715085, - "std":0.253242, + "mean":0.80008, + "std":0.253011, "values":[ 1.0, 0.515625, - 0.62963 + 0.884615 ] }, "kept_oos_share":{ - "mean":0.024141, - "std":0.013516, + "mean":0.021179, + "std":0.013777, "values":[ 0.009675, 0.036446, - 0.026303 + 0.017415 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.026, - "std":0.013, + "mean":0.022333, + "std":0.011015, "values":[ 0.011, 0.033, - 0.034 + 0.023 ] } } @@ -8163,13 +8242,17 @@ "msp" ], "tau":{ - "mean":0.20164, - "std":0.286486, "values":[ 0.532444, 0.03688, 0.035594 - ] + ], + "signals":[ + "msp_t", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -8369,13 +8452,17 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.3558, - "std":0.51156, "values":[ 0.946481, 0.064359, 0.05656 - ] + ], + "signals":[ + "msp_t", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ @@ -9325,8 +9412,17 @@ ], "hybrid":{ "tau":{ - "mean":-2.778161, - "std":2.424787 + "values":[ + -4.129125, + 0.021177, + -4.226536 + ], + "signals":[ + "entropy", + "margin", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -10085,8 +10181,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.502294, - "std":1.03685 + "values":[ + -1.076362, + 0.694625, + -1.125146 + ], + "signals":[ + "entropy", + "msp", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -11611,13 +11716,17 @@ ], "hybrid":{ "tau":{ - "mean":-1.339347, - "std":1.764824, "values":[ -2.465052, 0.694625, -2.247615 - ] + ], + "signals":[ + "entropy", + "msp", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -12015,13 +12124,17 @@ "entropy" ], "tau":{ - "mean":-1.339347, - "std":1.764824, "values":[ -2.465052, 0.694625, -2.247615 - ] + ], + "signals":[ + "entropy", + "msp", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -12221,13 +12334,17 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":-0.097541, - "std":0.939643, "values":[ -0.56666, 0.984293, -0.710255 - ] + ], + "signals":[ + "entropy", + "msp", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ @@ -12515,86 +12632,86 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":-1.602045, - "std":0.64528, + "mean":-1.450988, + "std":0.561202, "values":[ - -1.857494, - -0.868148, - -2.080493 + -1.649582, + -0.817493, + -1.88589 ] }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.962093, - "std":0.008978, + "mean":0.955938, + "std":0.00944, "values":[ - 0.953784, - 0.96088, - 0.971616 + 0.946143, + 0.956695, + 0.964977 ] }, "validation_weighted_risk":{ - "mean":0.043121, - "std":0.000325, + "mean":0.041017, + "std":0.000598, "values":[ - 0.043117, - 0.042798, - 0.043448 + 0.040389, + 0.041083, + 0.041578 ] }, "test":{ "coverage":{ - "mean":0.937818, - "std":0.008092, + "mean":0.931152, + "std":0.006172, "values":[ - 0.938, - 0.929636, - 0.945818 + 0.93, + 0.925636, + 0.937818 ] }, "selective_risk":{ - "mean":0.051499, - "std":0.003701, + "mean":0.048031, + "std":0.00401, "values":[ - 0.04749, - 0.05222, - 0.054787 + 0.043402, + 0.050285, + 0.050407 ] }, "in_scope_risk":{ - "mean":0.00809, - "std":0.001247, + "mean":0.007732, + "std":0.00123, "values":[ - 0.006746, - 0.008315, - 0.009209 + 0.00654, + 0.007658, + 0.008997 ] }, "kept_oos_error_rate":{ - "mean":0.324736, - "std":0.022477, + "mean":0.311136, + "std":0.028864, "values":[ - 0.301966, - 0.346908, - 0.325333 + 0.283407, + 0.341014, + 0.308989 ] }, "kept_oos_share":{ - "mean":0.137285, - "std":0.00728, + "mean":0.133016, + "std":0.005084, "values":[ - 0.138011, - 0.129669, - 0.144175 + 0.133138, + 0.127873, + 0.138038 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.229667, - "std":0.014503, + "mean":0.211667, + "std":0.016197, "values":[ - 0.215, - 0.23, - 0.244 + 0.193, + 0.222, + 0.22 ] } } @@ -16641,86 +16758,86 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.10549, - "std":0.016969, + "mean":0.106163, + "std":0.016518, "values":[ - 0.108044, - 0.121037, - 0.08739 + 0.1083, + 0.121508, + 0.088679 ] }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.866041, - "std":0.022301, + "mean":0.863676, + "std":0.022526, "values":[ - 0.877207, - 0.840363, - 0.880554 + 0.875114, + 0.837725, + 0.878189 ] }, "validation_weighted_risk":{ - "mean":0.042651, - "std":0.000563, + "mean":0.040552, + "std":0.000503, "values":[ - 0.042001, - 0.042974, - 0.042979 + 0.040021, + 0.040612, + 0.041022 ] }, "test":{ "coverage":{ - "mean":0.848242, - "std":0.032727, + "mean":0.846667, + "std":0.031476, "values":[ - 0.848364, - 0.815455, - 0.880909 + 0.848, + 0.814545, + 0.877455 ] }, "selective_risk":{ - "mean":0.052173, - "std":0.00864, + "mean":0.051723, + "std":0.007978, "values":[ - 0.047793, - 0.0466, - 0.062126 + 0.047599, + 0.046652, + 0.06092 ] }, "in_scope_risk":{ - "mean":0.013357, - "std":0.002508, + "mean":0.013293, + "std":0.002633, "values":[ 0.014028, - 0.015463, - 0.010582 + 0.015481, + 0.010371 ] }, "kept_oos_error_rate":{ - "mean":0.429215, - "std":0.078257, + "mean":0.428755, + "std":0.07815, "values":[ - 0.356522, + 0.355895, 0.419075, - 0.512048 + 0.511294 ] }, "kept_oos_share":{ - "mean":0.092839, - "std":0.013752, + "mean":0.092114, + "std":0.012959, "values":[ - 0.098586, - 0.077146, - 0.102786 + 0.098199, + 0.077232, + 0.100912 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.188, - "std":0.058796, + "mean":0.185667, + "std":0.055582, "values":[ - 0.164, + 0.163, 0.145, - 0.255 + 0.249 ] } } @@ -17133,8 +17250,17 @@ ], "hybrid":{ "tau":{ - "mean":0.340009, - "std":0.451489 + "values":[ + 0.077821, + 0.080865, + 0.86134 + ], + "signals":[ + "msp", + "msp", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -18353,8 +18479,17 @@ ], "hybrid":{ "tau":{ - "mean":1.460889, - "std":0.979203 + "values":[ + 0.879145, + 0.912108, + 2.591415 + ], + "signals":[ + "msp_t", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -19895,13 +20030,17 @@ ], "hybrid":{ "tau":{ - "mean":1.460889, - "std":0.979203, "values":[ 0.879145, 0.912108, 2.591415 - ] + ], + "signals":[ + "msp_t", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -20299,13 +20438,17 @@ "margin" ], "tau":{ - "mean":1.460889, - "std":0.979203, "values":[ 0.879145, 0.912108, 2.591415 - ] + ], + "signals":[ + "msp_t", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -20505,59 +20648,63 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":2.216768, - "std":2.140198, "values":[ 0.965718, 0.996595, - 4.687991 - ] + 4.667069 + ], + "signals":[ + "msp_t", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.2352, - "std":0.119814, + "mean":0.236472, + "std":0.119865, "values":[ 0.354037, 0.114431, - 0.237131 + 0.240949 ] }, "validation_weighted_risk":{ - "mean":0.007906, - "std":0.006886, + "mean":0.008224, + "std":0.007127, "values":[ 0.012592, 0.0, - 0.011125 + 0.01208 ] }, "test":{ "coverage":{ - "mean":0.233515, - "std":0.114225, + "mean":0.234788, + "std":0.114452, "values":[ 0.341091, 0.113636, - 0.245818 + 0.249636 ] }, "selective_risk":{ - "mean":0.01204, - "std":0.002611, + "mean":0.012207, + "std":0.002878, "values":[ 0.011727, 0.0096, - 0.014793 + 0.015295 ] }, "in_scope_risk":{ - "mean":0.003034, - "std":0.001426, + "mean":0.003019, + "std":0.001427, "values":[ 0.004479, 0.001629, - 0.002994 + 0.00295 ] }, "kept_oos_error_rate":{ @@ -20570,21 +20717,21 @@ ] }, "kept_oos_share":{ - "mean":0.025803, - "std":0.019416, + "mean":0.025985, + "std":0.019221, "values":[ 0.047974, 0.0176, - 0.011834 + 0.012382 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.011667, - "std":0.005859, + "mean":0.012, + "std":0.006245, "values":[ 0.014, 0.005, - 0.016 + 0.017 ] } } @@ -21755,8 +21902,17 @@ ], "hybrid":{ "tau":{ - "mean":-1.420356, - "std":2.635913 + "values":[ + -4.464043, + 0.097783, + 0.105192 + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -22515,8 +22671,17 @@ ], "hybrid":{ "tau":{ - "mean":0.042789, - "std":1.179061 + "values":[ + -1.318183, + 0.691634, + 0.754916 + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -22979,8 +23144,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.478771, - "std":1.269717 + "values":[ + -1.944545, + 0.225611, + 0.282621 + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -24045,13 +24219,17 @@ ], "hybrid":{ "tau":{ - "mean":-1.18306, - "std":1.810917, "values":[ -1.318183, 0.691634, -2.922631 - ] + ], + "signals":[ + "entropy", + "msp_t", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -24248,13 +24426,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.537914, - "std":1.219665, "values":[ -1.944545, 0.225611, 0.105192 - ] + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -24453,13 +24635,17 @@ "entropy" ], "tau":{ - "mean":-1.18306, - "std":1.810917, "values":[ -1.318183, 0.691634, -2.922631 - ] + ], + "signals":[ + "entropy", + "msp_t", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -24659,84 +24845,88 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":-0.444952, - "std":1.42457, "values":[ - -0.469773, + -0.466672, 0.991866, -1.856949 - ] + ], + "signals":[ + "entropy", + "msp_t", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.712088, - "std":0.147527, + "mean":0.710511, + "std":0.147208, "values":[ - 0.733533, + 0.728803, 0.555012, 0.847719 ] }, "validation_weighted_risk":{ - "mean":0.014132, - "std":0.000953, + "mean":0.013333, + "std":0.000853, "values":[ - 0.015008, + 0.012608, 0.013117, 0.014273 ] }, "test":{ "coverage":{ - "mean":0.690242, - "std":0.153996, + "mean":0.689212, + "std":0.153769, "values":[ - 0.713818, + 0.710727, 0.525818, 0.831091 ] }, "selective_risk":{ - "mean":0.015332, - "std":0.010066, + "mean":0.015177, + "std":0.010182, "values":[ - 0.010443, + 0.009977, 0.008645, 0.026909 ] }, "in_scope_risk":{ - "mean":0.001993, - "std":0.00136, + "mean":0.001995, + "std":0.001359, "values":[ - 0.001343, + 0.001348, 0.001081, 0.003556 ] }, "kept_oos_error_rate":{ - "mean":0.224607, - "std":0.070676, + "mean":0.221868, + "std":0.073476, "values":[ - 0.178218, + 0.17, 0.189655, 0.305949 ] }, "kept_oos_share":{ - "mean":0.056263, - "std":0.01902, + "mean":0.056167, + "std":0.019057, "values":[ - 0.051452, + 0.051164, 0.040111, 0.077226 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.055333, - "std":0.046145, + "mean":0.054667, + "std":0.046576, "values":[ - 0.036, + 0.034, 0.022, 0.108 ] @@ -24751,13 +24941,17 @@ "msp_t" ], "tau":{ - "mean":-0.537914, - "std":1.219665, "values":[ -1.944545, 0.225611, 0.105192 - ] + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -24957,86 +25151,90 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.100221, - "std":1.05829, "values":[ - -1.108145, - 0.862094, - 0.546715 - ] + -1.053598, + 0.876201, + 0.590232 + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.911481, - "std":0.009812, + "mean":0.907934, + "std":0.010828, "values":[ - 0.903655, - 0.908297, - 0.922489 + 0.900199, + 0.903294, + 0.920308 ] }, "validation_weighted_risk":{ - "mean":0.042493, - "std":0.000527, + "mean":0.040814, + "std":0.000702, "values":[ - 0.043095, - 0.042274, - 0.042111 + 0.04033, + 0.040493, + 0.041619 ] }, "test":{ "coverage":{ - "mean":0.897273, - "std":0.016461, + "mean":0.889212, + "std":0.013895, "values":[ - 0.885091, - 0.890727, - 0.916 + 0.877455, + 0.885636, + 0.904545 ] }, "selective_risk":{ - "mean":0.050902, - "std":0.00775, + "mean":0.047431, + "std":0.007183, "values":[ - 0.044577, - 0.048581, - 0.059547 + 0.040821, + 0.046397, + 0.055075 ] }, "in_scope_risk":{ - "mean":0.006758, - "std":0.001448, + "mean":0.006406, + "std":0.001406, "values":[ - 0.005912, - 0.005933, - 0.00843 + 0.005705, + 0.005488, + 0.008024 ] }, "kept_oos_error_rate":{ - "mean":0.407227, - "std":0.002518, + "mean":0.394299, + "std":0.009886, "values":[ - 0.406383, - 0.410058, - 0.405239 + 0.387387, + 0.405622, + 0.389886 ] }, "kept_oos_share":{ - "mean":0.110301, - "std":0.016656, + "mean":0.105818, + "std":0.015912, "values":[ - 0.096549, - 0.105532, - 0.128821 + 0.092002, + 0.102238, + 0.123216 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.222, - "std":0.037027, + "mean":0.204333, + "std":0.033561, "values":[ - 0.191, - 0.212, - 0.263 + 0.172, + 0.202, + 0.239 ] } } @@ -29743,8 +29941,17 @@ ], "hybrid":{ "tau":{ - "mean":-1.067557, - "std":1.191662 + "values":[ + -1.807531, + 0.307113, + -1.702253 + ], + "signals":[ + "entropy", + "msp_t", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -30207,8 +30414,17 @@ ], "hybrid":{ "tau":{ - "mean":-1.288443, - "std":2.321544 + "values":[ + -3.969124, + 0.046444, + 0.05735 + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -31427,8 +31643,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.498523, - "std":1.180254 + "values":[ + -1.861188, + 0.201693, + 0.163928 + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -32692,13 +32917,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.534049, - "std":1.151601, "values":[ -1.861188, 0.201693, 0.05735 - ] + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -33099,59 +33328,59 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":-0.136809, - "std":0.011912, + "mean":-0.138166, + "std":0.010366, "values":[ -0.150062, -0.13337, - -0.126994 + -0.131066 ] }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.785596, - "std":0.020965, + "mean":0.786475, + "std":0.019442, "values":[ 0.797836, 0.797563, - 0.761387 + 0.764025 ] }, "validation_weighted_risk":{ - "mean":0.014038, - "std":0.000892, + "mean":0.014817, + "std":0.000502, "values":[ 0.014712, 0.014375, - 0.013026 + 0.015363 ] }, "test":{ "coverage":{ - "mean":0.789455, - "std":0.024618, + "mean":0.789758, + "std":0.024094, "values":[ 0.805273, 0.802, - 0.761091 + 0.762 ] }, "selective_risk":{ - "mean":0.021834, - "std":0.005878, + "mean":0.021908, + "std":0.00575, "values":[ 0.025062, 0.025391, - 0.01505 + 0.015271 ] }, "in_scope_risk":{ "mean":0.001793, - "std":0.000552, + "std":0.000553, "values":[ 0.002311, 0.001857, - 0.001211 + 0.00121 ] }, "kept_oos_error_rate":{ @@ -33164,21 +33393,21 @@ ] }, "kept_oos_share":{ - "mean":0.020079, - "std":0.005403, + "mean":0.020153, + "std":0.005276, "values":[ 0.022804, 0.023577, - 0.013856 + 0.014078 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.087667, - "std":0.025736, + "mean":0.088, + "std":0.025159, "values":[ 0.101, 0.104, - 0.058 + 0.059 ] } } @@ -33191,13 +33420,17 @@ "msp_t" ], "tau":{ - "mean":-0.534049, - "std":1.151601, "values":[ -1.861188, 0.201693, 0.05735 - ] + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -33397,13 +33630,17 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.16363, - "std":0.630569, "values":[ -0.557901, 0.609019, 0.439773 - ] + ], + "signals":[ + "entropy", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ @@ -35469,8 +35706,17 @@ ], "hybrid":{ "tau":{ - "mean":1.080174, - "std":0.555253 + "values":[ + 0.962084, + 0.593466, + 1.684973 + ], + "signals":[ + "msp", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -36726,13 +36972,17 @@ ], "hybrid":{ "tau":{ - "mean":1.080174, - "std":0.555253, "values":[ 0.962084, 0.593466, 1.684973 - ] + ], + "signals":[ + "msp", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -37065,13 +37315,17 @@ "margin" ], "tau":{ - "mean":1.080174, - "std":0.555253, "values":[ 0.962084, 0.593466, 1.684973 - ] + ], + "signals":[ + "msp", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -37265,67 +37519,63 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":1.162692, - "std":0.805007, - "n":2, "values":[ - null, + 0.962084, 0.593466, - 1.731918 - ] + 1.684973 + ], + "signals":[ + "msp", + "msp_t", + "margin" + ], + "note":"signals differ across seeds; no mean" }, - "feasible":[ - false, - true, - true - ], + "feasible":true, "validation_weighted_coverage":{ - "mean":0.024358, - "std":0.022254, + "mean":0.030448, + "std":0.014122, "values":[ - 0.0, + 0.015542, 0.043627, - 0.029448 + 0.032175 ] }, "validation_weighted_risk":{ - "mean":0.014005, - "std":0.006711, - "n":2, + "mean":0.0119, + "std":0.010345, "values":[ - null, + 0.0, 0.01875, - 0.009259 + 0.016949 ] }, "test":{ "coverage":{ - "mean":0.023818, - "std":0.022578, + "mean":0.028485, + "std":0.016547, "values":[ - 0.0, + 0.011818, 0.044909, - 0.026545 + 0.028727 ] }, "selective_risk":{ - "mean":0.094504, - "std":0.001961, - "n":2, + "mean":0.077161, + "std":0.040823, "values":[ - null, + 0.030769, 0.093117, - 0.09589 + 0.107595 ] }, "in_scope_risk":{ - "mean":0.052606, - "std":0.003177, - "n":2, + "mean":0.050616, + "std":0.018104, "values":[ - null, + 0.030769, 0.054852, - 0.05036 + 0.066225 ] }, "kept_oos_error_rate":{ @@ -37339,13 +37589,12 @@ ] }, "kept_oos_share":{ - "mean":0.044216, - "std":0.005275, - "n":2, + "mean":0.028263, + "std":0.024551, "values":[ - null, + 0.0, 0.040486, - 0.047945 + 0.044304 ] }, "high_conf_oos_misroute_rate":{ @@ -38227,8 +38476,17 @@ ], "hybrid":{ "tau":{ - "mean":-1.903141, - "std":1.891129 + "values":[ + 0.275794, + -3.117299, + -2.867919 + ], + "signals":[ + "msp", + "entropy", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -38987,8 +39245,17 @@ ], "hybrid":{ "tau":{ - "mean":0.961059, - "std":0.490732 + "values":[ + 0.680271, + 1.5277, + 0.675207 + ], + "signals":[ + "msp_t", + "margin", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -39451,8 +39718,17 @@ ], "hybrid":{ "tau":{ - "mean":0.461076, - "std":0.067367 + "values":[ + 0.530907, + 0.396479, + 0.455842 + ], + "signals":[ + "margin", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -40517,13 +40793,17 @@ ], "hybrid":{ "tau":{ - "mean":0.961059, - "std":0.490732, "values":[ 0.680271, 1.5277, 0.675207 - ] + ], + "signals":[ + "msp_t", + "margin", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -40720,13 +41000,17 @@ ], "hybrid":{ "tau":{ - "mean":0.461076, - "std":0.067367, "values":[ 0.530907, 0.396479, 0.455842 - ] + ], + "signals":[ + "margin", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -40925,13 +41209,17 @@ "msp" ], "tau":{ - "mean":0.961059, - "std":0.490732, "values":[ 0.680271, 1.5277, 0.675207 - ] + ], + "signals":[ + "msp_t", + "margin", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -41131,13 +41419,17 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":1.836352, - "std":1.65303, "values":[ 0.932785, 3.74422, 0.83205 - ] + ], + "signals":[ + "msp_t", + "margin", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ @@ -41223,13 +41515,17 @@ "msp" ], "tau":{ - "mean":0.461076, - "std":0.067367, "values":[ 0.530907, 0.396479, 0.455842 - ] + ], + "signals":[ + "margin", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -41429,85 +41725,89 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.948894, - "std":0.561235, "values":[ - 1.594872, - 0.580977, + 1.623891, + 0.584037, 0.670832 - ] + ], + "signals":[ + "margin", + "msp", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.771798, - "std":0.029052, + "mean":0.770555, + "std":0.027958, "values":[ - 0.785442, - 0.791516, + 0.783806, + 0.789423, 0.738435 ] }, "validation_weighted_risk":{ - "mean":0.041769, - "std":0.001308, + "mean":0.040952, + "std":0.00093, "values":[ - 0.042273, - 0.04275, + 0.042014, + 0.040558, 0.040284 ] }, "test":{ "coverage":{ - "mean":0.764848, - "std":0.039766, + "mean":0.763515, + "std":0.039061, "values":[ - 0.775273, - 0.798364, + 0.772, + 0.797636, 0.720909 ] }, "selective_risk":{ - "mean":0.052865, - "std":0.011082, + "mean":0.052182, + "std":0.010723, "values":[ - 0.055816, - 0.062173, + 0.054169, + 0.061773, 0.040605 ] }, "in_scope_risk":{ - "mean":0.016668, - "std":0.004022, + "mean":0.016355, + "std":0.003833, "values":[ - 0.017884, - 0.019942, + 0.017172, + 0.019713, 0.012179 ] }, "kept_oos_error_rate":{ - "mean":0.621439, - "std":0.058887, + "mean":0.62053, + "std":0.059007, "values":[ - 0.568027, - 0.684588, + 0.566434, + 0.683453, 0.611702 ] }, "kept_oos_share":{ - "mean":0.059968, - "std":0.011203, + "mean":0.05938, + "std":0.010553, "values":[ - 0.068949, - 0.063539, + 0.067358, + 0.063369, 0.047415 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.157667, - "std":0.03885, + "mean":0.155667, + "std":0.037899, "values":[ - 0.167, - 0.191, + 0.162, + 0.19, 0.115 ] } @@ -41921,8 +42221,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.969815, - "std":2.014441 + "values":[ + 0.296954, + -3.292709, + 0.08631 + ], + "signals":[ + "msp_t", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -42385,8 +42694,17 @@ ], "hybrid":{ "tau":{ - "mean":-2.199503, - "std":1.914925 + "values":[ + -3.317417, + -3.292709, + 0.011616 + ], + "signals":[ + "entropy", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -43145,8 +43463,17 @@ ], "hybrid":{ "tau":{ - "mean":0.313659, - "std":0.257516 + "values":[ + 0.537617, + 0.032278, + 0.37108 + ], + "signals":[ + "margin", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -43609,8 +43936,17 @@ ], "hybrid":{ "tau":{ - "mean":0.137719, - "std":0.195342 + "values":[ + 0.017753, + 0.032278, + 0.363124 + ], + "signals":[ + "margin", + "margin", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -44675,13 +45011,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.969815, - "std":2.014441, "values":[ 0.296954, -3.292709, 0.08631 - ] + ], + "signals":[ + "msp_t", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -44878,13 +45218,17 @@ ], "hybrid":{ "tau":{ - "mean":-2.199503, - "std":1.914925, "values":[ -3.317417, -3.292709, 0.011616 - ] + ], + "signals":[ + "entropy", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -45083,13 +45427,17 @@ "margin" ], "tau":{ - "mean":-0.969815, - "std":2.014441, "values":[ 0.296954, -3.292709, 0.08631 - ] + ], + "signals":[ + "msp_t", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -45289,86 +45637,90 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.634354, - "std":1.549989, "values":[ - 0.742111, - -0.966702, - 2.127651 - ] + 0.810608, + -0.766965, + 2.206076 + ], + "signals":[ + "msp_t", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.912401, - "std":0.022449, + "mean":0.903428, + "std":0.021754, "values":[ - 0.892749, - 0.936865, - 0.907588 + 0.881201, + 0.924677, + 0.904405 ] }, "validation_weighted_risk":{ - "mean":0.015589, - "std":4.8e-05, + "mean":0.013906, + "std":0.000565, "values":[ - 0.015592, - 0.015636, - 0.015539 + 0.014558, + 0.013579, + 0.013582 ] }, "test":{ "coverage":{ - "mean":0.885455, - "std":0.017603, + "mean":0.873273, + "std":0.018461, "values":[ - 0.873273, - 0.905636, - 0.877455 + 0.854545, + 0.891455, + 0.873818 ] }, "selective_risk":{ - "mean":0.029281, - "std":0.006288, + "mean":0.025959, + "std":0.004828, "values":[ - 0.025817, - 0.036539, - 0.025487 + 0.021702, + 0.031205, + 0.024969 ] }, "in_scope_risk":{ - "mean":0.00583, - "std":0.001422, + "mean":0.005331, + "std":0.000999, "values":[ - 0.005776, - 0.007278, - 0.004436 + 0.005137, + 0.006413, + 0.004444 ] }, "kept_oos_error_rate":{ - "mean":0.218933, - "std":0.033905, + "mean":0.204943, + "std":0.024117, "values":[ - 0.208421, - 0.256849, - 0.191529 + 0.191847, + 0.232775, + 0.190207 ] }, "kept_oos_share":{ - "mean":0.109553, - "std":0.009527, + "mean":0.102912, + "std":0.012297, "values":[ - 0.098897, - 0.117246, - 0.112516 + 0.088723, + 0.109525, + 0.110487 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.117667, - "std":0.028113, + "mean":0.102, + "std":0.022517, "values":[ - 0.099, - 0.15, - 0.104 + 0.08, + 0.125, + 0.101 ] } } @@ -45381,13 +45733,17 @@ "margin" ], "tau":{ - "mean":-2.199503, - "std":1.914925, "values":[ -3.317417, -3.292709, 0.011616 - ] + ], + "signals":[ + "entropy", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -45587,86 +45943,90 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":-1.643991, - "std":1.495057, "values":[ - -2.048214, - -2.895375, - 0.011616 - ] + -2.030432, + -2.89495, + 0.030141 + ], + "signals":[ + "entropy", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.991175, - "std":0.012057, + "mean":0.989871, + "std":0.011176, "values":[ - 0.977437, - 0.996087, - 1.0 + 0.977165, + 0.994267, + 0.99818 ] }, "validation_weighted_risk":{ - "mean":0.042495, - "std":0.000725, + "mean":0.04124, + "std":0.000593, "values":[ - 0.041885, - 0.043297, - 0.042303 + 0.041618, + 0.041546, + 0.040557 ] }, "test":{ "coverage":{ - "mean":0.985091, - "std":0.018589, + "mean":0.984485, + "std":0.01843, "values":[ - 0.964, + 0.963455, 0.992182, - 0.999091 + 0.997818 ] }, "selective_risk":{ - "mean":0.072545, - "std":0.009102, + "mean":0.072222, + "std":0.009038, "values":[ - 0.062429, + 0.062087, 0.075133, - 0.080073 + 0.079446 ] }, "in_scope_risk":{ - "mean":0.012249, - "std":0.000827, + "mean":0.012251, + "std":0.000828, "values":[ 0.012955, 0.011338, - 0.012453 + 0.012458 ] }, "kept_oos_error_rate":{ - "mean":0.363342, - "std":0.028567, + "mean":0.362237, + "std":0.028498, "values":[ - 0.330909, + 0.329684, 0.374348, - 0.38477 + 0.382679 ] }, "kept_oos_share":{ - "mean":0.170986, - "std":0.013644, + "mean":0.1706, + "std":0.013653, "values":[ - 0.155602, + 0.155124, 0.175738, - 0.18162 + 0.18094 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.338667, - "std":0.058227, + "mean":0.336667, + "std":0.05783, "values":[ - 0.273, + 0.271, 0.359, - 0.384 + 0.38 ] } } @@ -46083,8 +46443,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.305304, - "std":1.466578 + "values":[ + 0.570888, + -1.998417, + 0.511616 + ], + "signals":[ + "msp", + "entropy", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -46547,8 +46916,17 @@ ], "hybrid":{ "tau":{ - "mean":-1.223336, - "std":2.323956 + "values":[ + 0.118481, + 0.11832, + -3.906809 + ], + "signals":[ + "msp", + "msp", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -47767,8 +48145,17 @@ ], "hybrid":{ "tau":{ - "mean":0.084543, - "std":0.113018 + "values":[ + 0.213919, + 0.034672, + 0.005039 + ], + "signals":[ + "msp_t", + "margin", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -48833,13 +49220,17 @@ ], "hybrid":{ "tau":{ - "mean":-0.205695, - "std":1.552599, "values":[ 0.703851, -1.998417, 0.677481 - ] + ], + "signals":[ + "msp_t", + "entropy", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -49036,13 +49427,17 @@ ], "hybrid":{ "tau":{ - "mean":0.112426, - "std":0.104565, "values":[ 0.213919, 0.11832, 0.005039 - ] + ], + "signals":[ + "msp_t", + "msp", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -49241,13 +49636,17 @@ "msp_t" ], "tau":{ - "mean":-0.205695, - "std":1.552599, "values":[ 0.703851, -1.998417, 0.677481 - ] + ], + "signals":[ + "msp_t", + "entropy", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -49447,86 +49846,90 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.291703, - "std":1.114023, "values":[ - 0.93586, - -0.994659, - 0.933909 - ] + 0.938235, + -0.981731, + 0.939351 + ], + "signals":[ + "msp_t", + "entropy", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.754238, - "std":0.013729, + "mean":0.750571, + "std":0.012643, "values":[ - 0.761141, - 0.738428, - 0.763145 + 0.757685, + 0.735974, + 0.758053 ] }, "validation_weighted_risk":{ - "mean":0.015116, - "std":7.9e-05, + "mean":0.014146, + "std":0.001128, "values":[ - 0.01518, - 0.015028, - 0.01514 + 0.012847, + 0.014708, + 0.014882 ] }, "test":{ "coverage":{ - "mean":0.744606, - "std":0.013828, + "mean":0.741212, + "std":0.012552, "values":[ - 0.751091, - 0.728727, - 0.754 + 0.748909, + 0.726727, + 0.748 ] }, "selective_risk":{ - "mean":0.019235, - "std":0.002146, + "mean":0.018911, + "std":0.002094, "values":[ - 0.018155, - 0.021707, - 0.017844 + 0.018208, + 0.021266, + 0.017258 ] }, "in_scope_risk":{ - "mean":0.003913, - "std":0.002015, + "mean":0.00384, + "std":0.002076, "values":[ - 0.002513, - 0.006222, - 0.003002 + 0.00252, + 0.006234, + 0.002768 ] }, "kept_oos_error_rate":{ - "mean":0.419394, - "std":0.007393, + "mean":0.425624, + "std":0.009532, "values":[ - 0.427632, - 0.417219, - 0.413333 + 0.433333, + 0.414966, + 0.428571 ] }, "kept_oos_share":{ - "mean":0.03688, - "std":0.000756, + "mean":0.035741, + "std":0.001493, "values":[ - 0.036795, - 0.037675, - 0.036171 + 0.036417, + 0.036778, + 0.03403 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.063333, - "std":0.001528, + "mean":0.062, + "std":0.002646, "values":[ 0.065, - 0.063, - 0.062 + 0.061, + 0.06 ] } } @@ -49539,13 +49942,17 @@ "margin" ], "tau":{ - "mean":0.112426, - "std":0.104565, "values":[ 0.213919, 0.11832, 0.005039 - ] + ], + "signals":[ + "msp_t", + "msp", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -49745,84 +50152,88 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":1.001867, - "std":0.745569, "values":[ - 0.682157, - 0.469472, - 1.853974 - ] + 0.686319, + 0.4701, + 1.873395 + ], + "signals":[ + "msp_t", + "msp", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.861729, - "std":0.014265, + "mean":0.859454, + "std":0.014212, "values":[ - 0.878182, - 0.854187, - 0.852817 + 0.875817, + 0.852367, + 0.850179 ] }, "validation_weighted_risk":{ - "mean":0.042969, - "std":7.5e-05, + "mean":0.040861, + "std":0.000216, "values":[ - 0.042886, - 0.043029, - 0.042994 + 0.040612, + 0.040985, + 0.040987 ] }, "test":{ "coverage":{ - "mean":0.854848, - "std":0.009999, + "mean":0.854061, + "std":0.009886, "values":[ - 0.866182, + 0.865091, 0.851091, - 0.847273 + 0.846 ] }, "selective_risk":{ - "mean":0.05233, - "std":0.005423, + "mean":0.051954, + "std":0.005708, "values":[ - 0.049958, + 0.04897, 0.058535, - 0.048498 + 0.048356 ] }, "in_scope_risk":{ - "mean":0.010271, - "std":0.004741, + "mean":0.010044, + "std":0.0049, "values":[ - 0.008984, + 0.008529, 0.015522, - 0.006305 + 0.00608 ] }, "kept_oos_error_rate":{ - "mean":0.493918, - "std":0.029084, + "mean":0.493122, + "std":0.03142, "values":[ - 0.470449, + 0.466667, 0.484848, - 0.526455 + 0.527851 ] }, "kept_oos_share":{ - "mean":0.087185, - "std":0.005446, + "mean":0.086981, + "std":0.005429, "values":[ - 0.088791, + 0.088272, 0.091647, - 0.081116 + 0.081023 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.202, - "std":0.005196, + "mean":0.201, + "std":0.006245, "values":[ - 0.199, + 0.196, 0.208, 0.199 ] @@ -50237,8 +50648,17 @@ ], "hybrid":{ "tau":{ - "mean":0.256583, - "std":0.91838 + "values":[ + 0.824672, + -0.802947, + 0.748024 + ], + "signals":[ + "msp_t", + "entropy", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -50701,8 +51121,17 @@ ], "hybrid":{ "tau":{ - "mean":-1.153793, - "std":1.377153 + "values":[ + 0.435952, + -1.915747, + -1.981585 + ], + "signals":[ + "msp", + "entropy", + "entropy" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -51461,8 +51890,17 @@ ], "hybrid":{ "tau":{ - "mean":1.99402, - "std":0.962075 + "values":[ + 2.537916, + 2.560953, + 0.88319 + ], + "signals":[ + "margin", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -51925,8 +52363,17 @@ ], "hybrid":{ "tau":{ - "mean":1.234831, - "std":0.600639 + "values":[ + 0.54394, + 1.6329, + 1.527653 + ], + "signals":[ + "msp", + "margin", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -52991,13 +53438,17 @@ ], "hybrid":{ "tau":{ - "mean":1.99402, - "std":0.962075, "values":[ 2.537916, 2.560953, 0.88319 - ] + ], + "signals":[ + "margin", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -53194,13 +53645,17 @@ ], "hybrid":{ "tau":{ - "mean":1.234831, - "std":0.600639, "values":[ 0.54394, 1.6329, 1.527653 - ] + ], + "signals":[ + "msp", + "margin", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -53399,13 +53854,17 @@ "msp_t" ], "tau":{ - "mean":1.99402, - "std":0.962075, "values":[ 2.537916, 2.560953, 0.88319 - ] + ], + "signals":[ + "margin", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -53605,13 +54064,17 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":2.099428, - "std":1.044613, "values":[ 2.833788, 2.560953, 0.903542 - ] + ], + "signals":[ + "margin", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ @@ -53697,13 +54160,17 @@ "margin" ], "tau":{ - "mean":1.234831, - "std":0.600639, "values":[ 0.54394, 1.6329, 1.527653 - ] + ], + "signals":[ + "msp", + "margin", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -53903,29 +54370,33 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":1.567414, - "std":0.814044, "values":[ - 0.634666, + 0.634982, 1.933021, 2.134554 - ] + ], + "signals":[ + "msp", + "margin", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.519945, - "std":0.086301, + "mean":0.519854, + "std":0.086159, "values":[ - 0.6099, + 0.609627, 0.437834, 0.512101 ] }, "validation_weighted_risk":{ - "mean":0.040406, - "std":0.001167, + "mean":0.040263, + "std":0.000922, "values":[ - 0.041745, + 0.041316, 0.039872, 0.0396 ] @@ -54855,8 +55326,17 @@ ], "hybrid":{ "tau":{ - "mean":-2.411483, - "std":2.10235 + "values":[ + -3.457853, + -3.785324, + 0.008728 + ], + "signals":[ + "entropy", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -56075,8 +56555,17 @@ ], "hybrid":{ "tau":{ - "mean":0.242781, - "std":0.034221 + "values":[ + 0.28005, + 0.235519, + 0.212773 + ], + "signals":[ + "msp", + "msp_t", + "msp" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -57340,13 +57829,17 @@ ], "hybrid":{ "tau":{ - "mean":-2.411483, - "std":2.10235, "values":[ -3.457853, -3.785324, 0.008728 - ] + ], + "signals":[ + "entropy", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -57747,86 +58240,86 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":0.783936, - "std":0.105887, + "mean":0.793789, + "std":0.110399, "values":[ - 0.856062, + 0.878939, 0.833374, - 0.662372 + 0.669055 ] }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.87678, - "std":0.029431, + "mean":0.871624, + "std":0.033376, "values":[ - 0.856832, + 0.843458, 0.862926, - 0.910581 + 0.908488 ] }, "validation_weighted_risk":{ - "mean":0.01486, - "std":0.001011, + "mean":0.013466, + "std":0.000209, "values":[ - 0.01529, + 0.013375, 0.013705, - 0.015586 + 0.013318 ] }, "test":{ "coverage":{ - "mean":0.851758, - "std":0.030085, + "mean":0.847515, + "std":0.033774, "values":[ - 0.831818, + 0.820182, 0.837091, - 0.886364 + 0.885273 ] }, "selective_risk":{ - "mean":0.031132, - "std":0.010948, + "mean":0.03059, + "std":0.011547, "values":[ - 0.021202, + 0.01973, 0.029322, - 0.042872 + 0.042719 ] }, "in_scope_risk":{ - "mean":0.005557, - "std":0.001588, + "mean":0.005493, + "std":0.001642, "values":[ - 0.004763, + 0.004566, 0.004523, - 0.007385 + 0.007389 ] }, "kept_oos_error_rate":{ - "mean":0.273066, - "std":0.062221, + "mean":0.27166, + "std":0.065095, "values":[ - 0.204787, + 0.2, 0.287841, - 0.326568 + 0.327138 ] }, "kept_oos_share":{ - "mean":0.093633, - "std":0.015429, + "mean":0.091872, + "std":0.016877, "values":[ - 0.082186, + 0.077588, 0.087533, - 0.111179 + 0.110495 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.123333, - "std":0.050402, + "mean":0.120667, + "std":0.053154, "values":[ - 0.077, + 0.07, 0.116, - 0.177 + 0.176 ] } } @@ -57839,13 +58332,17 @@ "margin" ], "tau":{ - "mean":-2.411483, - "std":2.10235, "values":[ -3.457853, -3.785324, 0.008728 - ] + ], + "signals":[ + "entropy", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "validation_oos_share":{ "mean":0.032258, @@ -58045,86 +58542,90 @@ "sensitivity_reweighted_validation":{ "note":"validation rows reweighted to a 18.2% OOS share (the test split's, known only after looking at test) before choosing tau; diagnostic only, not the reported method", "tau":{ - "mean":-1.382348, - "std":1.398516, "values":[ - -2.053999, - -2.318337, - 0.225291 - ] + -2.006056, + -2.306693, + 0.275901 + ], + "signals":[ + "entropy", + "entropy", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_weighted_coverage":{ - "mean":0.961548, - "std":0.018973, + "mean":0.959274, + "std":0.018699, "values":[ - 0.942775, - 0.961153, - 0.980716 + 0.940683, + 0.95906, + 0.978078 ] }, "validation_weighted_risk":{ - "mean":0.043017, - "std":0.000467, + "mean":0.041128, + "std":0.000376, "values":[ - 0.043136, - 0.042502, - 0.043413 + 0.041297, + 0.040697, + 0.041391 ] }, "test":{ "coverage":{ - "mean":0.95503, - "std":0.021507, + "mean":0.952424, + "std":0.020759, "values":[ - 0.936, - 0.950727, - 0.978364 + 0.933091, + 0.949818, + 0.974364 ] }, "selective_risk":{ - "mean":0.069762, - "std":0.015615, + "mean":0.068434, + "std":0.015597, "values":[ - 0.055167, - 0.067891, - 0.086229 + 0.05339, + 0.067381, + 0.084531 ] }, "in_scope_risk":{ - "mean":0.011735, - "std":0.001123, + "mean":0.011669, + "std":0.00101, "values":[ - 0.010801, + 0.010806, 0.011422, - 0.012981 + 0.01278 ] }, "kept_oos_error_rate":{ - "mean":0.392607, - "std":0.054922, + "mean":0.3882, + "std":0.05694, "values":[ - 0.335227, - 0.397906, - 0.444688 + 0.327536, + 0.396574, + 0.440489 ] }, "kept_oos_share":{ - "mean":0.150844, - "std":0.016963, + "mean":0.149166, + "std":0.016987, "values":[ - 0.136752, - 0.146108, - 0.169671 + 0.134451, + 0.145291, + 0.167755 ] }, "high_conf_oos_misroute_rate":{ - "mean":0.315333, - "std":0.085565, + "mean":0.307667, + "std":0.085196, "values":[ - 0.236, - 0.304, - 0.406 + 0.226, + 0.301, + 0.396 ] } } @@ -58455,7 +58956,19 @@ "msp_t" ], "hybrid":{ - "tau":null, + "tau":{ + "values":[ + null, + null, + null + ], + "signals":[ + "msp_t", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" + }, "feasible":false, "validation_coverage":{ "mean":0.0, @@ -58702,8 +59215,17 @@ ], "hybrid":{ "tau":{ - "mean":2.189096, - "std":2.13581 + "values":[ + 0.933808, + 4.655183, + 0.978296 + ], + "signals":[ + "msp_t", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -59462,8 +59984,17 @@ ], "hybrid":{ "tau":{ - "mean":2.004105, - "std":1.803545 + "values":[ + 0.945491, + 4.086562, + 0.980262 + ], + "signals":[ + "msp_t", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -60393,13 +60924,17 @@ ], "hybrid":{ "tau":{ - "mean":2.004105, - "std":1.803545, "values":[ 0.945491, 4.086562, 0.980262 - ] + ], + "signals":[ + "msp_t", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -62948,13 +63483,17 @@ ], "hybrid":{ "tau":{ - "mean":1.08125, - "std":0.695219, "values":[ 1.882091, 0.632676, 0.728981 - ] + ], + "signals":[ + "margin", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -66021,8 +66560,17 @@ ], "hybrid":{ "tau":{ - "mean":1.887236, - "std":1.619518 + "values":[ + 0.948701, + 3.75729, + 0.955717 + ], + "signals":[ + "msp_t", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -67855,13 +68403,17 @@ ], "hybrid":{ "tau":{ - "mean":2.820058, - "std":1.614574, "values":[ 3.747166, 3.75729, 0.955717 - ] + ], + "signals":[ + "margin", + "margin", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -68058,13 +68610,17 @@ ], "hybrid":{ "tau":{ - "mean":0.962006, - "std":0.603048, "values":[ 1.655836, 0.563929, 0.666255 - ] + ], + "signals":[ + "margin", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -70613,13 +71169,17 @@ ], "hybrid":{ "tau":{ - "mean":1.83595, - "std":0.849718, "values":[ 0.857446, 2.387806, 2.262597 - ] + ], + "signals":[ + "msp_t", + "margin", + "margin" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -71135,8 +71695,17 @@ ], "hybrid":{ "tau":{ - "mean":1.916379, - "std":1.633921 + "values":[ + 3.803064, + 0.96944, + 0.976632 + ], + "signals":[ + "margin", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ @@ -72969,13 +73538,17 @@ ], "hybrid":{ "tau":{ - "mean":1.916379, - "std":1.633921, "values":[ 3.803064, 0.96944, 0.976632 - ] + ], + "signals":[ + "margin", + "msp_t", + "msp_t" + ], + "note":"signals differ across seeds; no mean" }, "feasible":true, "validation_coverage":{ diff --git a/src/tinyrouter/analysis_run.py b/src/tinyrouter/analysis_run.py index 81dd308..89ee7b1 100644 --- a/src/tinyrouter/analysis_run.py +++ b/src/tinyrouter/analysis_run.py @@ -144,12 +144,22 @@ def load_haiku(results_root: Path) -> dict[str, HaikuSplit]: return {s: split_arrays(records, s, n) for s, n in EXPECTED_ROWS.items()} -def combine(values: list, keep_values: bool = True) -> object: +SIGNAL_KEYS = ("selected_signal", "signal") +TAU_KEY = "tau" +BY_SIGNAL_KEY = "by_signal" + + +def combine(values: list, keep_values: bool = True, signals: list | None = None) -> object: """Seeds merged leaf by leaf: numbers -> mean, sample std (ddof=1) and the values. Seeds whose value is None (e.g. no feasible threshold) are left out of the mean and counted in ``n``; with fewer than two numbers the std is None, not 0. + + A ``tau`` is a score on the scale of its signal. Below a dict that + names the signal (``selected_signal`` or ``signal``), ``signals`` holds + each seed's signal; when they differ, a mean of the taus means + nothing, so ``tau`` keeps only the per-seed values and signal names. """ if all(v is None for v in values): return None @@ -157,7 +167,10 @@ def combine(values: list, keep_values: bool = True) -> object: keys = set(values[0]) if any(set(v) != keys for v in values): raise ValueError("seed results have different keys") - return {k: combine([v[k] for v in values], keep_values) for k in values[0]} + for key in SIGNAL_KEYS: + if key in keys: + signals = [v[key] for v in values] + return {k: combine_key(k, [v[k] for v in values], keep_values, signals) for k in values[0]} numbers = [v for v in values if isinstance(v, int | float) and not isinstance(v, bool)] if numbers and len(numbers) + values.count(None) == len(values): arr = np.array(numbers, dtype=np.float64) @@ -175,6 +188,23 @@ def combine(values: list, keep_values: bool = True) -> object: return values[0] if all(v == values[0] for v in values) else values +def combine_key(key: str, values: list, keep_values: bool, signals: list | None) -> object: + if key == BY_SIGNAL_KEY and all(isinstance(v, dict) for v in values): + # Each entry of by_signal is one signal for every seed; its tau averages fine. + n = len(values) + return { + name: combine([v[name] for v in values], keep_values, [name] * n) for name in values[0] + } + mixed = signals is not None and len(set(signals)) > 1 + if key == TAU_KEY and mixed: + return { + "values": values, + "signals": signals, + "note": "signals differ across seeds; no mean", + } + return combine(values, keep_values, signals) + + def rounded(value: object) -> object: if isinstance(value, float): return round(value, 6) diff --git a/src/tinyrouter/selective.py b/src/tinyrouter/selective.py index 3b3fe11..f58e9d9 100644 --- a/src/tinyrouter/selective.py +++ b/src/tinyrouter/selective.py @@ -209,9 +209,13 @@ def select_threshold( """Largest-coverage threshold whose selective-risk Wilson upper bound is <= ``target_risk``. Candidates are the distinct validation scores; threshold t accepts - every row with confidence >= t. With weights, the kept count and the - errors are weighted sums and coverage is the kept share of the total - weight. + every row with confidence >= t. With weights, the risk is the weighted + error rate p of the kept rows, coverage is their share of the total + weight, and the bound uses Kish's effective sample size of the kept + rows, n_eff = (sum w)^2 / sum w^2, with p * n_eff errors. Unequal + weights carry less information than as many equal ones, so the bound + is wider than with the nominal weighted count. Unit weights give + n_eff = n and the unweighted result. """ require_validation(scored.split, "a deferral threshold") confidence = np.asarray(scored.confidence, dtype=np.float64) @@ -219,8 +223,11 @@ def select_threshold( weight = np.ones(confidence.size) if scored.weight is None else np.asarray(scored.weight, float) order, starts, sizes = _tie_groups(confidence) accepted = np.cumsum(np.add.reduceat(weight[order], starts)) + squares = np.cumsum(np.add.reduceat((weight**2)[order], starts)) errors = np.cumsum(np.add.reduceat((weight * error)[order], starts)) - upper = wilson_upper(np.minimum(errors, accepted), accepted, z) + rate = np.minimum(errors / accepted, 1.0) + n_eff = accepted**2 / squares + upper = wilson_upper(rate * n_eff, n_eff, z) ok = np.flatnonzero(upper <= target_risk) if ok.size == 0: return ThresholdChoice(None, target_risk, 0.0, None, None, int(starts.size)) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index fb8bfd8..6a6fdbc 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -293,6 +293,18 @@ def test_oos_score_is_one_minus_the_largest_in_scope_probability_at_t(): assert route(split, t, "argmax", SPACE, "majority").oos_score is None +def test_summed_oos_score_is_high_when_the_oos_agent_carries_the_mass(): + z = np.full((1, SPACE.num_intents), 0.0) + z[0, SPACE.oos_intent_id] = 12.0 + split = SplitLogits("validation", z.astype(np.float32), np.array([SPACE.oos_intent_id])) + got = route(split, 1.0, "summed", SPACE, "encoder").oos_score + p = np.exp(z[0]) / np.exp(z[0]).sum() + agents = SPACE.aggregate_probs(p[None, :])[0] + assert agents.argmax() == OOS + assert got == pytest.approx([1 - np.delete(agents, OOS).max()], rel=1e-6) + assert got[0] > 0.9 + + def test_signal_choice_by_hand_coverage_first_then_aurc(): val = routed( "validation", @@ -310,6 +322,18 @@ def choice(coverage): assert select_signal(val, {"entropy": choice(0.5), "msp": choice(0.5)}) == "msp" +def test_equal_coverage_goes_to_the_lower_aurc_even_when_it_is_listed_later(): + # msp is first in SIGNALS but ranks the error first; msp_t ranks it last. + val = routed( + "validation", + [F, F, F, T], + [F, F, F, F], + {"msp": np.array([1.0, 2, 3, 4]), "msp_t": np.array([4.0, 3, 2, 1])}, + ) + same = ThresholdChoice(0.0, 0.05, 0.5, 0.0, 0.0, 4) + assert select_signal(val, {"msp": same, "msp_t": same}) == "msp_t" + + def test_entropy_signal_is_lower_for_a_uniform_row_than_a_one_hot_row(): z = np.full((2, SPACE.num_intents), 0.0) z[1, 0] = 50.0 diff --git a/tests/test_analysis_run.py b/tests/test_analysis_run.py index e19e511..646a4ea 100644 --- a/tests/test_analysis_run.py +++ b/tests/test_analysis_run.py @@ -302,3 +302,29 @@ def test_ablation_detection_uses_one_fixed_aggregation_for_both_models(): assert out["oos_250"]["oos_detection_test"]["auroc"]["mean"] == 0.98 assert out["oos_0"]["oos_detection_test"]["auroc"]["mean"] == 0.97 assert out["oos_0"]["hybrid_test"]["0.02"]["llm_call_rate"]["mean"] == 0.1 + + +def test_taus_of_different_signals_are_not_averaged(): + seeds = [ + {"selected_signal": s, "hybrid": {"tau": t, "coverage": c}} + for s, t, c in (("msp_t", 0.3, 0.9), ("entropy", -3.3, 0.8), ("margin", 0.1, 0.7)) + ] + out = combine(seeds) + assert out["hybrid"]["tau"] == { + "values": [0.3, -3.3, 0.1], + "signals": ["msp_t", "entropy", "margin"], + "note": "signals differ across seeds; no mean", + } + assert out["hybrid"]["coverage"]["mean"] == pytest.approx(0.8) + same = combine([{"signal": "msp", "tau": t} for t in (0.2, 0.4, 0.6)]) + assert same["tau"]["mean"] == pytest.approx(0.4) + + +def test_by_signal_taus_are_averaged_per_signal_even_when_the_hybrid_signals_differ(): + seeds = [ + {"selected_signal": s, "hybrid": {"tau": t}, "by_signal": {"msp": {"tau": t}}} + for s, t in (("msp", 0.2), ("entropy", 0.4), ("msp", 0.6)) + ] + out = combine(seeds) + assert out["by_signal"]["msp"]["tau"]["mean"] == pytest.approx(0.4) + assert "mean" not in out["hybrid"]["tau"] diff --git a/tests/test_selective.py b/tests/test_selective.py index 610356c..ab25110 100644 --- a/tests/test_selective.py +++ b/tests/test_selective.py @@ -189,9 +189,34 @@ def test_weighted_threshold_counts_weighted_rows_by_hand(): ) loose = select_threshold(scored, 1.0) assert loose.coverage == 1.0 and loose.risk == pytest.approx(3 / 4) - # Kept weight 1 with no error: upper bound z^2 / (1 + z^2) = 0.73. - upper = ONE_SIDED_95_Z**2 / (1 + ONE_SIDED_95_Z**2) + # Kish: n_eff = 4^2 / (0.25 + 0.25 + 9) = 16 / 9.5, with 0.75 * n_eff errors. + n_eff = 16 / 9.5 + expected = wilson_upper(np.array([0.75 * n_eff]), np.array([n_eff]), ONE_SIDED_95_Z)[0] + assert loose.risk_upper == pytest.approx(expected) + # Two kept rows of weight 0.5, no error: n_eff = 2, bound z^2 / (2 + z^2) = 0.575. + upper = ONE_SIDED_95_Z**2 / (2 + ONE_SIDED_95_Z**2) mid = select_threshold(scored, upper + 1e-9) assert mid.tau == 2.0 and mid.coverage == pytest.approx(1 / 4) - unweighted = select_threshold(Scored("validation", scored.confidence, scored.error), 0.75) - assert unweighted.coverage == 1.0 + + +@pytest.mark.parametrize("target", [0.05, 0.1, 0.3]) +def test_unit_weights_give_the_unweighted_choice(target): + rng = np.random.default_rng(5) + confidence = np.round(rng.random(300), 2) + error = rng.random(300) < 0.3 * (1 - confidence) + plain = select_threshold(Scored("validation", confidence, error), target) + unit = select_threshold(Scored("validation", confidence, error, np.ones(300)), target) + assert unit.tau == plain.tau and unit.coverage == pytest.approx(plain.coverage) + assert unit.risk_upper == pytest.approx(plain.risk_upper) + + +def test_unequal_weights_widen_the_bound_beyond_the_nominal_count(): + rng = np.random.default_rng(6) + error = rng.random(200) < 0.1 + weight = np.where(rng.random(200) < 0.1, 5.0, 0.5) + weight = weight / weight.mean() + got = select_threshold(Scored("validation", np.ones(200), error, weight), 1.0) + nominal = wilson_upper( + np.array([(weight * error).sum()]), np.array([weight.sum()]), ONE_SIDED_95_Z + ) + assert got.risk_upper > nominal[0]