diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml index dff08ffea..f9b98d09f 100644 --- a/.github/.licenserc.yaml +++ b/.github/.licenserc.yaml @@ -61,6 +61,9 @@ header: # written verbatim as the agent's answer (.agent-eval/run.py). - '.agent-eval/tasks/**/*.patch' - '.agent-eval/tasks/**/oracle_answer.md' + # Generated study tables and figures (the judge-panel study's CSV and SVG, written by + # research/judge-panel): a header would break the CSV readers and is lost on regeneration. + - 'dokimi/docs/assets/judge-panel/**' # Changelog fragments (__.md) and the template # `scriv create` copies into each: `scriv collect` moves their text into # CHANGELOG.md, which carries the header itself. diff --git a/dokimi/changelog.d/20261108_120000_napetrov_judge-panel-cli.md b/dokimi/changelog.d/20261108_120000_napetrov_judge-panel-cli.md new file mode 100644 index 000000000..6b736fb8a --- /dev/null +++ b/dokimi/changelog.d/20261108_120000_napetrov_judge-panel-cli.md @@ -0,0 +1,3 @@ +### Added + +- **Judge panel for skill novelty**: `skill novelty analyze` and `probe` now take `--judge PROVIDER/MODEL`. Repeat the flag to form a panel. `--judge-aggregate {median,majority,unanimous}` (default `median`) turns the panel's verdicts into one per answer. The record gets a `judge_panel` block: each judge's own paired 2×2, the judge range of the added share, and a `judge_sensitive` flag set when that range exceeds 0.10. Each run keeps every judge's verdict (`analyze` writes them to `.judge_verdicts.json`), and the report names the panel. A warning is printed when two judges share a vendor family. The `--estimate-only` dollar figure and call counts include the extra judges. A model given twice, under any spelling, is refused. Background: `docs/judge-panel-study.md`. diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index f078ad73d..401cef3bc 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -356,6 +356,9 @@ code. the novelty numbers come from: method, K0 per model, the 2×2, need-graph statistics, content categories, shortening with measured behaviour, and the caveats each result carries. +- [judge-panel-study.md](judge-panel-study.md) — how much the judge moves + the results: per-judge numbers, agreement and self-preference, and the + judge panels Dokimi has today against the proposal. - [grading-tiers.md](grading-tiers.md) — design note (partly implemented): grading probes with verifiable and grounded checks before the LLM judge, keeping the judge's verdict beside them so its accuracy against ground truth can be measured, with diff --git a/dokimi/docs/assets/judge-panel/aggregation_rules.csv b/dokimi/docs/assets/judge-panel/aggregation_rules.csv new file mode 100644 index 000000000..e43eabc0e --- /dev/null +++ b/dokimi/docs/assets/judge-panel/aggregation_rules.csv @@ -0,0 +1,36 @@ +rule,answer_model,n,added,gated,known,degraded,noise_added,noise_degraded +majority,claude-opus-5,62,0.21,0.161,0.694,0.032,, +majority,claude-haiku-4-5,60,0.317,0.317,0.483,0.033,0.033,0.017 +majority,gpt-5.6-luna,62,0.242,0.21,0.661,0.048,, +majority,grok-4.6,62,0.339,0.339,0.468,0.016,0.032,0.016 +majority,nemotron-nano-12b,60,0.483,0.433,0.267,0.0,0.05,0.083 +mean_score>=0.4,claude-opus-5,62,0.29,0.258,0.565,0.016,, +mean_score>=0.4,claude-haiku-4-5,60,0.433,0.417,0.317,0.017,0.017,0.017 +mean_score>=0.4,gpt-5.6-luna,62,0.339,0.323,0.516,0.048,, +mean_score>=0.4,grok-4.6,62,0.387,0.371,0.339,0.0,0.081,0.048 +mean_score>=0.4,nemotron-nano-12b,60,0.517,0.517,0.117,0.0,0.0,0.033 +median_score>=0.4,claude-opus-5,62,0.21,0.161,0.694,0.032,, +median_score>=0.4,claude-haiku-4-5,60,0.317,0.317,0.483,0.033,0.033,0.017 +median_score>=0.4,gpt-5.6-luna,62,0.242,0.21,0.661,0.048,, +median_score>=0.4,grok-4.6,62,0.339,0.339,0.468,0.016,0.032,0.016 +median_score>=0.4,nemotron-nano-12b,60,0.483,0.433,0.267,0.0,0.05,0.083 +unanimous,claude-opus-5,62,0.242,0.21,0.565,0.032,, +unanimous,claude-haiku-4-5,60,0.333,0.317,0.383,0.033,0.033,0.033 +unanimous,gpt-5.6-luna,62,0.339,0.258,0.516,0.032,, +unanimous,grok-4.6,62,0.323,0.323,0.403,0.016,0.048,0.048 +unanimous,nemotron-nano-12b,60,0.55,0.517,0.117,0.0,0.033,0.033 +any_correct,claude-opus-5,62,0.081,0.081,0.855,0.032,, +any_correct,claude-haiku-4-5,60,0.15,0.117,0.683,0.033,0.05,0.067 +any_correct,gpt-5.6-luna,62,0.161,0.145,0.806,0.0,, +any_correct,grok-4.6,62,0.371,0.371,0.516,0.0,0.065,0.016 +any_correct,nemotron-nano-12b,60,0.383,0.333,0.517,0.017,0.1,0.067 +dawid_skene,claude-opus-5,62,0.226,0.161,0.661,0.048,, +dawid_skene,claude-haiku-4-5,60,0.333,0.333,0.45,0.033,0.033,0.017 +dawid_skene,gpt-5.6-luna,62,0.242,0.21,0.661,0.048,, +dawid_skene,grok-4.6,62,0.339,0.339,0.468,0.016,0.032,0.032 +dawid_skene,nemotron-nano-12b,60,0.483,0.433,0.25,0.0,0.033,0.083 +majority_strict,claude-opus-5,62,0.306,0.29,0.452,0.016,, +majority_strict,claude-haiku-4-5,60,0.433,0.4,0.15,0.017,0.033,0.0 +majority_strict,gpt-5.6-luna,62,0.339,0.274,0.371,0.016,, +majority_strict,grok-4.6,62,0.484,0.484,0.177,0.0,0.016,0.048 +majority_strict,nemotron-nano-12b,60,0.5,0.5,0.05,0.017,0.0,0.017 diff --git a/dokimi/docs/assets/judge-panel/agreement_with_ds.csv b/dokimi/docs/assets/judge-panel/agreement_with_ds.csv new file mode 100644 index 000000000..bfbcbbe66 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/agreement_with_ds.csv @@ -0,0 +1,16 @@ +rater,n,agree_with_ds_consensus +judge:gpt-5.6-luna,2664,0.912 +judge:claude-sonnet-5,2932,0.959 +judge:deepseek-v3.2,2932,0.926 +judge:glm-5,2932,0.949 +judge:gpt-6-luna,2932,0.917 +judge:grok-4.6,2931,0.965 +judge:kimi-k3,2932,0.938 +judge:qwen3-vl-235b,2932,0.935 +rule:majority,2932,0.993 +rule:mean_score>=0.4,2932,0.907 +rule:median_score>=0.4,2932,0.993 +rule:unanimous,2932,0.894 +rule:any_correct,2932,0.889 +rule:dawid_skene,2932,1.0 +rule:majority_strict,2932,0.788 diff --git a/dokimi/docs/assets/judge-panel/disagreement_by_group.csv b/dokimi/docs/assets/judge-panel/disagreement_by_group.csv new file mode 100644 index 000000000..fd6ad2246 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/disagreement_by_group.csv @@ -0,0 +1,19 @@ +by,group,n,share_items_not_unanimous +arm,full,444,0.173 +arm,no_skill,444,0.297 +arm,no_skill_repeat,268,0.31 +arm,placebo,444,0.212 +arm,root,444,0.191 +arm,routed,444,0.185 +arm,shuffled,444,0.187 +style,adversarial,914,0.27 +style,causal,964,0.195 +style,decision,1008,0.191 +style,task,46,0.174 +adversarial,False,2018,0.193 +adversarial,True,914,0.27 +model,claude-haiku-4-5,630,0.206 +model,claude-opus-5,528,0.203 +model,gpt-5.6-luna,528,0.201 +model,grok-4.6,616,0.114 +model,nemotron-nano-12b,630,0.354 diff --git a/dokimi/docs/assets/judge-panel/gated_added_by_judge.svg b/dokimi/docs/assets/judge-panel/gated_added_by_judge.svg new file mode 100644 index 000000000..35ce1d19a --- /dev/null +++ b/dokimi/docs/assets/judge-panel/gated_added_by_judge.svg @@ -0,0 +1,82 @@ + + +Placebo-gated added share (lenient) by judge + +0.0 + +0.1 + +0.2 + +0.3 + +0.4 + +0.5 + +0.6 +claude-opus-5 + + + + + + + + + + +gpt-5.6-luna + + + + + + + + + + +claude-haiku-4-5 + + + + + + + + + + +grok-4.6 + + + + + + + + + + +nemotron-nano-12b + + + + + + + + + + +gpt-5.6-luna +claude-sonnet-5 +gpt-6-luna +grok-4.6 +kimi-k3 +deepseek-v3.2 +glm-5 +qwen3-vl-235b +square: original judge (gpt-5.6-luna); triangle: 7-judge majority; grey bar: judge range. n = 60-62 paired non-reversal probes per model. + \ No newline at end of file diff --git a/dokimi/docs/assets/judge-panel/gated_added_by_judge_strict.svg b/dokimi/docs/assets/judge-panel/gated_added_by_judge_strict.svg new file mode 100644 index 000000000..71fa77c97 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/gated_added_by_judge_strict.svg @@ -0,0 +1,76 @@ + + +Placebo-gated added share (strict) by judge + +0.0 + +0.1 + +0.2 + +0.3 + +0.4 + +0.5 + +0.6 +claude-opus-5 + + + + + + + + + +gpt-5.6-luna + + + + + + + + + +claude-haiku-4-5 + + + + + + + + + +grok-4.6 + + + + + + + + + +nemotron-nano-12b + + + + + + + + + +gpt-5.6-luna +claude-sonnet-5 +gpt-6-luna +grok-4.6 +kimi-k3 +deepseek-v3.2 +glm-5 +qwen3-vl-235b + \ No newline at end of file diff --git a/dokimi/docs/assets/judge-panel/generations/generation_added_share.csv b/dokimi/docs/assets/judge-panel/generations/generation_added_share.csv new file mode 100644 index 000000000..d45cc144f --- /dev/null +++ b/dokimi/docs/assets/judge-panel/generations/generation_added_share.csv @@ -0,0 +1,71 @@ +family,old_judge,new_judge,mode,answer_model,added_old,added_new,delta +openai,gpt-5.6-luna,gpt-6-luna,lenient,claude-opus-5,0.129,0.113,-0.016 +openai,gpt-5.6-luna,gpt-6-luna,lenient,claude-haiku-4-5,0.117,0.15,0.033 +openai,gpt-5.6-luna,gpt-6-luna,lenient,gpt-5.6-luna,0.145,0.226,0.081 +openai,gpt-5.6-luna,gpt-6-luna,lenient,grok-4.6,0.339,0.339,0.0 +openai,gpt-5.6-luna,gpt-6-luna,lenient,nemotron-nano-12b,0.433,0.417,-0.017 +openai,gpt-5.6-luna,gpt-6-luna,strict,claude-opus-5,0.242,0.274,0.032 +openai,gpt-5.6-luna,gpt-6-luna,strict,claude-haiku-4-5,0.333,0.25,-0.083 +openai,gpt-5.6-luna,gpt-6-luna,strict,gpt-5.6-luna,0.323,0.21,-0.113 +openai,gpt-5.6-luna,gpt-6-luna,strict,grok-4.6,0.339,0.29,-0.048 +openai,gpt-5.6-luna,gpt-6-luna,strict,nemotron-nano-12b,0.417,0.3,-0.117 +openai,gpt-5.6-luna,gpt-6-sol,lenient,claude-opus-5,0.129,0.161,0.032 +openai,gpt-5.6-luna,gpt-6-sol,lenient,claude-haiku-4-5,0.117,0.183,0.067 +openai,gpt-5.6-luna,gpt-6-sol,lenient,gpt-5.6-luna,0.145,0.161,0.016 +openai,gpt-5.6-luna,gpt-6-sol,lenient,grok-4.6,0.339,0.29,-0.048 +openai,gpt-5.6-luna,gpt-6-sol,lenient,nemotron-nano-12b,0.433,0.5,0.067 +openai,gpt-5.6-luna,gpt-6-sol,strict,claude-opus-5,0.242,0.194,-0.048 +openai,gpt-5.6-luna,gpt-6-sol,strict,claude-haiku-4-5,0.333,0.267,-0.067 +openai,gpt-5.6-luna,gpt-6-sol,strict,gpt-5.6-luna,0.323,0.258,-0.065 +openai,gpt-5.6-luna,gpt-6-sol,strict,grok-4.6,0.339,0.387,0.048 +openai,gpt-5.6-luna,gpt-6-sol,strict,nemotron-nano-12b,0.417,0.35,-0.067 +moonshot,kimi-k2.5,kimi-k3,lenient,claude-opus-5,0.258,0.226,-0.032 +moonshot,kimi-k2.5,kimi-k3,lenient,claude-haiku-4-5,0.317,0.233,-0.083 +moonshot,kimi-k2.5,kimi-k3,lenient,gpt-5.6-luna,0.306,0.226,-0.081 +moonshot,kimi-k2.5,kimi-k3,lenient,grok-4.6,0.371,0.355,-0.016 +moonshot,kimi-k2.5,kimi-k3,lenient,nemotron-nano-12b,0.517,0.45,-0.067 +moonshot,kimi-k2.5,kimi-k3,strict,claude-opus-5,0.274,0.29,0.016 +moonshot,kimi-k2.5,kimi-k3,strict,claude-haiku-4-5,0.367,0.4,0.033 +moonshot,kimi-k2.5,kimi-k3,strict,gpt-5.6-luna,0.355,0.355,0.0 +moonshot,kimi-k2.5,kimi-k3,strict,grok-4.6,0.452,0.419,-0.032 +moonshot,kimi-k2.5,kimi-k3,strict,nemotron-nano-12b,0.517,0.5,-0.017 +anthropic,claude-haiku-4-5,claude-sonnet-5,lenient,claude-opus-5,0.145,0.21,0.065 +anthropic,claude-haiku-4-5,claude-sonnet-5,lenient,claude-haiku-4-5,0.2,0.3,0.1 +anthropic,claude-haiku-4-5,claude-sonnet-5,lenient,gpt-5.6-luna,0.21,0.226,0.016 +anthropic,claude-haiku-4-5,claude-sonnet-5,lenient,grok-4.6,0.339,0.323,-0.016 +anthropic,claude-haiku-4-5,claude-sonnet-5,lenient,nemotron-nano-12b,0.533,0.55,0.017 +anthropic,claude-haiku-4-5,claude-sonnet-5,strict,claude-opus-5,0.355,0.274,-0.081 +anthropic,claude-haiku-4-5,claude-sonnet-5,strict,claude-haiku-4-5,0.317,0.367,0.05 +anthropic,claude-haiku-4-5,claude-sonnet-5,strict,gpt-5.6-luna,0.371,0.306,-0.065 +anthropic,claude-haiku-4-5,claude-sonnet-5,strict,grok-4.6,0.5,0.452,-0.048 +anthropic,claude-haiku-4-5,claude-sonnet-5,strict,nemotron-nano-12b,0.483,0.483,0.0 +anthropic,claude-sonnet-5,claude-opus-5-5,lenient,claude-opus-5,0.21,0.177,-0.032 +anthropic,claude-sonnet-5,claude-opus-5-5,lenient,claude-haiku-4-5,0.3,0.2,-0.1 +anthropic,claude-sonnet-5,claude-opus-5-5,lenient,gpt-5.6-luna,0.226,0.226,0.0 +anthropic,claude-sonnet-5,claude-opus-5-5,lenient,grok-4.6,0.323,0.355,0.032 +anthropic,claude-sonnet-5,claude-opus-5-5,lenient,nemotron-nano-12b,0.55,0.583,0.033 +anthropic,claude-sonnet-5,claude-opus-5-5,strict,claude-opus-5,0.274,0.274,0.0 +anthropic,claude-sonnet-5,claude-opus-5-5,strict,claude-haiku-4-5,0.367,0.383,0.017 +anthropic,claude-sonnet-5,claude-opus-5-5,strict,gpt-5.6-luna,0.306,0.306,0.0 +anthropic,claude-sonnet-5,claude-opus-5-5,strict,grok-4.6,0.452,0.5,0.048 +anthropic,claude-sonnet-5,claude-opus-5-5,strict,nemotron-nano-12b,0.483,0.383,-0.1 +anthropic,claude-haiku-4-5,claude-opus-5-5,lenient,claude-opus-5,0.145,0.177,0.032 +anthropic,claude-haiku-4-5,claude-opus-5-5,lenient,claude-haiku-4-5,0.2,0.2,0.0 +anthropic,claude-haiku-4-5,claude-opus-5-5,lenient,gpt-5.6-luna,0.21,0.226,0.016 +anthropic,claude-haiku-4-5,claude-opus-5-5,lenient,grok-4.6,0.339,0.355,0.016 +anthropic,claude-haiku-4-5,claude-opus-5-5,lenient,nemotron-nano-12b,0.533,0.583,0.05 +anthropic,claude-haiku-4-5,claude-opus-5-5,strict,claude-opus-5,0.355,0.274,-0.081 +anthropic,claude-haiku-4-5,claude-opus-5-5,strict,claude-haiku-4-5,0.317,0.383,0.067 +anthropic,claude-haiku-4-5,claude-opus-5-5,strict,gpt-5.6-luna,0.371,0.306,-0.065 +anthropic,claude-haiku-4-5,claude-opus-5-5,strict,grok-4.6,0.5,0.5,0.0 +anthropic,claude-haiku-4-5,claude-opus-5-5,strict,nemotron-nano-12b,0.483,0.383,-0.1 +deepseek,deepseek-r1,deepseek-v3.2,lenient,claude-opus-5,0.129,0.177,0.048 +deepseek,deepseek-r1,deepseek-v3.2,lenient,claude-haiku-4-5,0.15,0.3,0.15 +deepseek,deepseek-r1,deepseek-v3.2,lenient,gpt-5.6-luna,0.145,0.21,0.065 +deepseek,deepseek-r1,deepseek-v3.2,lenient,grok-4.6,0.371,0.355,-0.016 +deepseek,deepseek-r1,deepseek-v3.2,lenient,nemotron-nano-12b,0.467,0.567,0.1 +deepseek,deepseek-r1,deepseek-v3.2,strict,claude-opus-5,0.242,0.242,0.0 +deepseek,deepseek-r1,deepseek-v3.2,strict,claude-haiku-4-5,0.45,0.383,-0.067 +deepseek,deepseek-r1,deepseek-v3.2,strict,gpt-5.6-luna,0.387,0.306,-0.081 +deepseek,deepseek-r1,deepseek-v3.2,strict,grok-4.6,0.468,0.435,-0.032 +deepseek,deepseek-r1,deepseek-v3.2,strict,nemotron-nano-12b,0.533,0.467,-0.067 diff --git a/dokimi/docs/assets/judge-panel/generations/generation_kappa.csv b/dokimi/docs/assets/judge-panel/generations/generation_kappa.csv new file mode 100644 index 000000000..23b744c20 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/generations/generation_kappa.csv @@ -0,0 +1,8 @@ +family,old_judge,new_judge,n,kappa,lenient_rate_delta +openai,gpt-5.6-luna,gpt-6-luna,2664,0.715,-0.0422 +openai,gpt-5.6-luna,gpt-6-sol,2664,0.696,-0.0428 +moonshot,kimi-k2.5,kimi-k3,2932,0.65,0.1176 +anthropic,claude-haiku-4-5,claude-sonnet-5,2932,0.656,0.0011 +anthropic,claude-sonnet-5,claude-opus-5-5,2932,0.705,0.0654 +anthropic,claude-haiku-4-5,claude-opus-5-5,2932,0.625,0.0665 +deepseek,deepseek-r1,deepseek-v3.2,2932,0.609,-0.0962 diff --git a/dokimi/docs/assets/judge-panel/generations/generation_panels.csv b/dokimi/docs/assets/judge-panel/generations/generation_panels.csv new file mode 100644 index 000000000..0788a7d84 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/generations/generation_panels.csv @@ -0,0 +1,21 @@ +panel,judges,mode,answer_model,n,added +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,lenient,claude-opus-5,62,0.177 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,lenient,claude-haiku-4-5,60,0.267 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,lenient,gpt-5.6-luna,62,0.226 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,lenient,grok-4.6,62,0.355 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,lenient,nemotron-nano-12b,60,0.533 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,strict,claude-opus-5,62,0.323 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,strict,claude-haiku-4-5,60,0.35 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,strict,gpt-5.6-luna,62,0.403 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,strict,grok-4.6,62,0.484 +old,gpt-5.6-luna+kimi-k2.5+claude-haiku-4-5,strict,nemotron-nano-12b,60,0.533 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,lenient,claude-opus-5,62,0.177 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,lenient,claude-haiku-4-5,60,0.2 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,lenient,gpt-5.6-luna,62,0.242 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,lenient,grok-4.6,62,0.355 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,lenient,nemotron-nano-12b,60,0.5 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,strict,claude-opus-5,62,0.274 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,strict,claude-haiku-4-5,60,0.35 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,strict,gpt-5.6-luna,62,0.29 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,strict,grok-4.6,62,0.452 +new,gpt-6-luna+kimi-k3+claude-opus-5-5,strict,nemotron-nano-12b,60,0.433 diff --git a/dokimi/docs/assets/judge-panel/generations/judge_leniency.csv b/dokimi/docs/assets/judge-panel/generations/judge_leniency.csv new file mode 100644 index 000000000..40b8c3078 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/generations/judge_leniency.csv @@ -0,0 +1,16 @@ +judge,n,lenient_rate,strict_rate +claude-haiku-4-5,2932,0.6514,0.4635 +claude-opus-5-5,2932,0.7179,0.3888 +claude-sonnet-5,2932,0.6525,0.4202 +deepseek-r1,2932,0.735,0.5184 +deepseek-v3.2,2932,0.6388,0.4802 +glm-5,2932,0.6194,0.5123 +gpt-5.6-luna,2664,0.756,0.3622 +gpt-6-luna,2932,0.7138,0.2684 +gpt-6-sol,2932,0.7132,0.2889 +grok-4.6,2931,0.6448,0.3449 +kimi-k2.5,2932,0.5986,0.5003 +kimi-k3,2932,0.7162,0.4601 +minimax-m2.5,2931,0.6752,0.5404 +nemotron-super-3-120b,2932,0.7248,0.3765 +qwen3-vl-235b,2932,0.6518,0.528 diff --git a/dokimi/docs/assets/judge-panel/generations/judge_mean_kappa.csv b/dokimi/docs/assets/judge-panel/generations/judge_mean_kappa.csv new file mode 100644 index 000000000..964125c89 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/generations/judge_mean_kappa.csv @@ -0,0 +1,16 @@ +judge,mean_kappa_vs_others +claude-haiku-4-5,0.614 +claude-opus-5-5,0.627 +claude-sonnet-5,0.622 +deepseek-r1,0.561 +deepseek-v3.2,0.605 +glm-5,0.621 +gpt-5.6-luna,0.584 +gpt-6-luna,0.541 +gpt-6-sol,0.562 +grok-4.6,0.606 +kimi-k2.5,0.622 +kimi-k3,0.643 +minimax-m2.5,0.576 +nemotron-super-3-120b,0.507 +qwen3-vl-235b,0.605 diff --git a/dokimi/docs/assets/judge-panel/headline_range.csv b/dokimi/docs/assets/judge-panel/headline_range.csv new file mode 100644 index 000000000..7d3e37885 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/headline_range.csv @@ -0,0 +1,41 @@ +mode,answer_model,metric,min,max,median,orig +lenient,claude-opus-5,added,0.113,0.242,0.202,0.129 +lenient,claude-opus-5,gated,0.081,0.21,0.153,0.097 +lenient,claude-opus-5,known,0.645,0.823,0.685,0.823 +lenient,claude-opus-5,degraded,0.016,0.048,0.032,0.032 +lenient,claude-haiku-4-5,added,0.117,0.333,0.283,0.117 +lenient,claude-haiku-4-5,gated,0.1,0.333,0.275,0.1 +lenient,claude-haiku-4-5,known,0.417,0.733,0.508,0.733 +lenient,claude-haiku-4-5,degraded,0.017,0.033,0.033,0.017 +lenient,gpt-5.6-luna,added,0.145,0.323,0.226,0.145 +lenient,gpt-5.6-luna,gated,0.113,0.242,0.202,0.113 +lenient,gpt-5.6-luna,known,0.565,0.79,0.677,0.79 +lenient,gpt-5.6-luna,degraded,0.0,0.065,0.04,0.032 +lenient,grok-4.6,added,0.323,0.371,0.339,0.339 +lenient,grok-4.6,gated,0.323,0.371,0.339,0.339 +lenient,grok-4.6,known,0.435,0.516,0.468,0.516 +lenient,grok-4.6,degraded,0.0,0.032,0.016,0.0 +lenient,nemotron-nano-12b,added,0.417,0.567,0.508,0.433 +lenient,nemotron-nano-12b,gated,0.367,0.517,0.458,0.367 +lenient,nemotron-nano-12b,known,0.15,0.45,0.275,0.433 +lenient,nemotron-nano-12b,degraded,0.0,0.033,0.0,0.033 +strict,claude-opus-5,added,0.242,0.323,0.282,0.242 +strict,claude-opus-5,gated,0.226,0.29,0.266,0.226 +strict,claude-opus-5,known,0.242,0.548,0.435,0.355 +strict,claude-opus-5,degraded,0.0,0.032,0.016,0.016 +strict,claude-haiku-4-5,added,0.25,0.433,0.375,0.333 +strict,claude-haiku-4-5,gated,0.233,0.417,0.35,0.317 +strict,claude-haiku-4-5,known,0.083,0.267,0.167,0.117 +strict,claude-haiku-4-5,degraded,0.017,0.033,0.017,0.033 +strict,gpt-5.6-luna,added,0.21,0.403,0.315,0.323 +strict,gpt-5.6-luna,gated,0.194,0.371,0.282,0.274 +strict,gpt-5.6-luna,known,0.177,0.484,0.363,0.226 +strict,gpt-5.6-luna,degraded,0.032,0.065,0.032,0.065 +strict,grok-4.6,added,0.29,0.468,0.435,0.339 +strict,grok-4.6,gated,0.29,0.452,0.419,0.339 +strict,grok-4.6,known,0.113,0.274,0.185,0.113 +strict,grok-4.6,degraded,0.0,0.016,0.0,0.016 +strict,nemotron-nano-12b,added,0.3,0.533,0.475,0.417 +strict,nemotron-nano-12b,gated,0.283,0.517,0.475,0.4 +strict,nemotron-nano-12b,known,0.017,0.1,0.042,0.017 +strict,nemotron-nano-12b,degraded,0.0,0.017,0.0,0.017 diff --git a/dokimi/docs/assets/judge-panel/pairwise_kappa.csv b/dokimi/docs/assets/judge-panel/pairwise_kappa.csv new file mode 100644 index 000000000..f35be510b --- /dev/null +++ b/dokimi/docs/assets/judge-panel/pairwise_kappa.csv @@ -0,0 +1,29 @@ +judge_a,judge_b,n,kappa_6way,kappa_lenient,kappa_strict,agree_lenient +gpt-5.6-luna,claude-sonnet-5,2664,0.595,0.748,0.728,0.897 +gpt-5.6-luna,deepseek-v3.2,2664,0.516,0.703,0.639,0.876 +gpt-5.6-luna,glm-5,2664,0.529,0.696,0.615,0.871 +gpt-5.6-luna,gpt-6-luna,2664,0.715,0.878,0.75,0.953 +gpt-5.6-luna,grok-4.6,2663,0.654,0.753,0.767,0.898 +gpt-5.6-luna,kimi-k3,2664,0.661,0.848,0.693,0.942 +gpt-5.6-luna,qwen3-vl-235b,2664,0.52,0.743,0.587,0.894 +claude-sonnet-5,deepseek-v3.2,2932,0.639,0.775,0.768,0.897 +claude-sonnet-5,glm-5,2932,0.668,0.836,0.747,0.924 +claude-sonnet-5,gpt-6-luna,2932,0.549,0.773,0.621,0.901 +claude-sonnet-5,grok-4.6,2931,0.681,0.862,0.774,0.937 +claude-sonnet-5,kimi-k3,2932,0.708,0.823,0.835,0.923 +claude-sonnet-5,qwen3-vl-235b,2932,0.608,0.79,0.739,0.905 +deepseek-v3.2,glm-5,2932,0.716,0.822,0.826,0.917 +deepseek-v3.2,gpt-6-luna,2932,0.456,0.723,0.516,0.878 +deepseek-v3.2,grok-4.6,2931,0.59,0.808,0.684,0.912 +deepseek-v3.2,kimi-k3,2932,0.619,0.729,0.78,0.881 +deepseek-v3.2,qwen3-vl-235b,2932,0.678,0.835,0.854,0.924 +glm-5,gpt-6-luna,2932,0.472,0.715,0.495,0.872 +glm-5,grok-4.6,2931,0.612,0.849,0.651,0.93 +glm-5,kimi-k3,2932,0.656,0.757,0.792,0.891 +glm-5,qwen3-vl-235b,2932,0.711,0.827,0.857,0.92 +gpt-6-luna,grok-4.6,2931,0.64,0.773,0.684,0.9 +gpt-6-luna,kimi-k3,2932,0.594,0.84,0.581,0.935 +gpt-6-luna,qwen3-vl-235b,2932,0.456,0.744,0.462,0.889 +grok-4.6,kimi-k3,2931,0.656,0.814,0.729,0.918 +grok-4.6,qwen3-vl-235b,2931,0.563,0.812,0.621,0.914 +kimi-k3,qwen3-vl-235b,2932,0.63,0.772,0.763,0.901 diff --git a/dokimi/docs/assets/judge-panel/panel_size.csv b/dokimi/docs/assets/judge-panel/panel_size.csv new file mode 100644 index 000000000..a4d844e5e --- /dev/null +++ b/dokimi/docs/assets/judge-panel/panel_size.csv @@ -0,0 +1,8 @@ +panel_size,subsets,mean_agree_with_full_ds,worst_agree,sd_gated_added_across_subsets,range_gated_added_across_subsets +1,7,0.941,0.917,0.0388,0.115 +2,21,0.943,0.924,0.0259,0.105 +3,35,0.966,0.947,0.0251,0.095 +4,35,0.969,0.949,0.0206,0.079 +5,21,0.98,0.97,0.0206,0.069 +6,7,0.982,0.973,0.011,0.029 +7,1,0.993,0.993,0.0,0.0 diff --git a/dokimi/docs/assets/judge-panel/per_judge_headline.csv b/dokimi/docs/assets/judge-panel/per_judge_headline.csv new file mode 100644 index 000000000..f986758a5 --- /dev/null +++ b/dokimi/docs/assets/judge-panel/per_judge_headline.csv @@ -0,0 +1,81 @@ +mode,answer_model,judge,n,added,gated,known,degraded,noise_added,noise_degraded +lenient,claude-opus-5,gpt-5.6-luna,62,0.129,0.097,0.823,0.032,, +lenient,claude-opus-5,claude-sonnet-5,62,0.21,0.161,0.645,0.032,, +lenient,claude-opus-5,deepseek-v3.2,62,0.177,0.129,0.742,0.032,, +lenient,claude-opus-5,glm-5,62,0.194,0.177,0.661,0.048,, +lenient,claude-opus-5,gpt-6-luna,62,0.113,0.081,0.79,0.032,, +lenient,claude-opus-5,grok-4.6,62,0.242,0.21,0.661,0.016,, +lenient,claude-opus-5,kimi-k3,62,0.226,0.145,0.677,0.048,, +lenient,claude-opus-5,qwen3-vl-235b,62,0.21,0.177,0.694,0.032,, +lenient,claude-haiku-4-5,gpt-5.6-luna,60,0.117,0.1,0.733,0.017,, +lenient,claude-haiku-4-5,claude-sonnet-5,60,0.3,0.3,0.467,0.033,0.017,0.017 +lenient,claude-haiku-4-5,deepseek-v3.2,60,0.3,0.3,0.483,0.033,0.033,0.033 +lenient,claude-haiku-4-5,glm-5,60,0.333,0.333,0.417,0.033,0.017,0.033 +lenient,claude-haiku-4-5,gpt-6-luna,60,0.15,0.15,0.667,0.033,0.033,0.083 +lenient,claude-haiku-4-5,grok-4.6,60,0.333,0.317,0.433,0.033,0.017,0.017 +lenient,claude-haiku-4-5,kimi-k3,60,0.233,0.183,0.6,0.017,0.033,0.033 +lenient,claude-haiku-4-5,qwen3-vl-235b,60,0.267,0.25,0.533,0.033,0.017,0.0 +lenient,gpt-5.6-luna,gpt-5.6-luna,62,0.145,0.113,0.79,0.032,, +lenient,gpt-5.6-luna,claude-sonnet-5,62,0.226,0.21,0.661,0.032,, +lenient,gpt-5.6-luna,deepseek-v3.2,62,0.21,0.161,0.694,0.032,, +lenient,gpt-5.6-luna,glm-5,62,0.323,0.242,0.565,0.048,, +lenient,gpt-5.6-luna,gpt-6-luna,62,0.226,0.194,0.694,0.048,, +lenient,gpt-5.6-luna,grok-4.6,62,0.29,0.242,0.597,0.065,, +lenient,gpt-5.6-luna,kimi-k3,62,0.226,0.194,0.742,0.0,, +lenient,gpt-5.6-luna,qwen3-vl-235b,62,0.29,0.226,0.613,0.048,, +lenient,grok-4.6,gpt-5.6-luna,62,0.339,0.339,0.516,0.0,, +lenient,grok-4.6,claude-sonnet-5,62,0.323,0.323,0.452,0.016,0.032,0.016 +lenient,grok-4.6,deepseek-v3.2,62,0.355,0.355,0.468,0.016,0.048,0.048 +lenient,grok-4.6,glm-5,62,0.323,0.323,0.435,0.032,0.048,0.065 +lenient,grok-4.6,gpt-6-luna,62,0.339,0.339,0.5,0.016,0.065,0.016 +lenient,grok-4.6,grok-4.6,62,0.371,0.371,0.435,0.016,0.048,0.048 +lenient,grok-4.6,kimi-k3,62,0.355,0.355,0.468,0.016,0.065,0.048 +lenient,grok-4.6,qwen3-vl-235b,62,0.339,0.339,0.484,0.0,0.032,0.016 +lenient,nemotron-nano-12b,gpt-5.6-luna,60,0.433,0.367,0.433,0.033,, +lenient,nemotron-nano-12b,claude-sonnet-5,60,0.55,0.483,0.15,0.0,0.05,0.05 +lenient,nemotron-nano-12b,deepseek-v3.2,60,0.567,0.5,0.2,0.0,0.1,0.05 +lenient,nemotron-nano-12b,glm-5,60,0.517,0.467,0.2,0.0,0.033,0.05 +lenient,nemotron-nano-12b,gpt-6-luna,60,0.417,0.4,0.45,0.033,0.05,0.1 +lenient,nemotron-nano-12b,grok-4.6,60,0.533,0.517,0.233,0.0,0.017,0.083 +lenient,nemotron-nano-12b,kimi-k3,60,0.45,0.383,0.35,0.0,0.1,0.067 +lenient,nemotron-nano-12b,qwen3-vl-235b,60,0.5,0.45,0.317,0.0,0.05,0.033 +strict,claude-opus-5,gpt-5.6-luna,62,0.242,0.226,0.355,0.016,, +strict,claude-opus-5,claude-sonnet-5,62,0.274,0.274,0.419,0.032,, +strict,claude-opus-5,deepseek-v3.2,62,0.242,0.226,0.548,0.016,, +strict,claude-opus-5,glm-5,62,0.323,0.29,0.5,0.0,, +strict,claude-opus-5,gpt-6-luna,62,0.274,0.258,0.242,0.016,, +strict,claude-opus-5,grok-4.6,62,0.29,0.258,0.371,0.0,, +strict,claude-opus-5,kimi-k3,62,0.29,0.274,0.452,0.016,, +strict,claude-opus-5,qwen3-vl-235b,62,0.306,0.29,0.5,0.0,, +strict,claude-haiku-4-5,gpt-5.6-luna,60,0.333,0.317,0.117,0.033,, +strict,claude-haiku-4-5,claude-sonnet-5,60,0.367,0.333,0.167,0.033,0.033,0.1 +strict,claude-haiku-4-5,deepseek-v3.2,60,0.383,0.35,0.2,0.017,0.033,0.0 +strict,claude-haiku-4-5,glm-5,60,0.367,0.35,0.25,0.017,0.0,0.033 +strict,claude-haiku-4-5,gpt-6-luna,60,0.25,0.233,0.117,0.017,0.017,0.033 +strict,claude-haiku-4-5,grok-4.6,60,0.417,0.4,0.083,0.017,0.05,0.0 +strict,claude-haiku-4-5,kimi-k3,60,0.4,0.35,0.167,0.017,0.067,0.017 +strict,claude-haiku-4-5,qwen3-vl-235b,60,0.433,0.417,0.267,0.017,0.0,0.017 +strict,gpt-5.6-luna,gpt-5.6-luna,62,0.323,0.274,0.226,0.065,, +strict,gpt-5.6-luna,claude-sonnet-5,62,0.306,0.258,0.339,0.048,, +strict,gpt-5.6-luna,deepseek-v3.2,62,0.306,0.274,0.435,0.032,, +strict,gpt-5.6-luna,glm-5,62,0.403,0.371,0.419,0.032,, +strict,gpt-5.6-luna,gpt-6-luna,62,0.21,0.194,0.177,0.065,, +strict,gpt-5.6-luna,grok-4.6,62,0.306,0.29,0.258,0.032,, +strict,gpt-5.6-luna,kimi-k3,62,0.355,0.306,0.387,0.032,, +strict,gpt-5.6-luna,qwen3-vl-235b,62,0.323,0.306,0.484,0.032,, +strict,grok-4.6,gpt-5.6-luna,62,0.339,0.339,0.113,0.016,, +strict,grok-4.6,claude-sonnet-5,62,0.452,0.452,0.145,0.016,0.032,0.032 +strict,grok-4.6,deepseek-v3.2,62,0.435,0.419,0.242,0.0,0.048,0.032 +strict,grok-4.6,glm-5,62,0.468,0.452,0.242,0.0,0.048,0.048 +strict,grok-4.6,gpt-6-luna,62,0.29,0.29,0.113,0.016,0.016,0.048 +strict,grok-4.6,grok-4.6,62,0.435,0.419,0.113,0.0,0.016,0.0 +strict,grok-4.6,kimi-k3,62,0.419,0.403,0.226,0.0,0.032,0.081 +strict,grok-4.6,qwen3-vl-235b,62,0.435,0.435,0.274,0.0,0.032,0.032 +strict,nemotron-nano-12b,gpt-5.6-luna,60,0.417,0.4,0.017,0.017,, +strict,nemotron-nano-12b,claude-sonnet-5,60,0.483,0.483,0.033,0.017,0.0,0.0 +strict,nemotron-nano-12b,deepseek-v3.2,60,0.467,0.467,0.05,0.017,0.0,0.017 +strict,nemotron-nano-12b,glm-5,60,0.533,0.517,0.083,0.0,0.0,0.017 +strict,nemotron-nano-12b,gpt-6-luna,60,0.3,0.283,0.017,0.0,0.017,0.0 +strict,nemotron-nano-12b,grok-4.6,60,0.383,0.383,0.017,0.0,0.017,0.0 +strict,nemotron-nano-12b,kimi-k3,60,0.5,0.483,0.05,0.0,0.017,0.017 +strict,nemotron-nano-12b,qwen3-vl-235b,60,0.533,0.517,0.1,0.0,0.0,0.017 diff --git a/dokimi/docs/assets/judge-panel/self_preference.csv b/dokimi/docs/assets/judge-panel/self_preference.csv new file mode 100644 index 000000000..d4c40efae --- /dev/null +++ b/dokimi/docs/assets/judge-panel/self_preference.csv @@ -0,0 +1,81 @@ +judge,answer_model,mode,relation,n,judge_rate,others_rate,delta +gpt-5.6-luna,claude-opus-5,lenient,other,528,0.847,0.784,0.063 +gpt-5.6-luna,claude-opus-5,strict,other,528,0.479,0.589,-0.109 +gpt-5.6-luna,claude-haiku-4-5,lenient,other,540,0.719,0.653,0.065 +gpt-5.6-luna,claude-haiku-4-5,strict,other,540,0.319,0.421,-0.102 +gpt-5.6-luna,gpt-5.6-luna,lenient,self,528,0.905,0.832,0.073 +gpt-5.6-luna,gpt-5.6-luna,strict,self,528,0.47,0.582,-0.112 +gpt-5.6-luna,grok-4.6,lenient,other,528,0.621,0.601,0.021 +gpt-5.6-luna,grok-4.6,strict,other,528,0.33,0.421,-0.091 +gpt-5.6-luna,nemotron-nano-12b,lenient,other,540,0.691,0.557,0.134 +gpt-5.6-luna,nemotron-nano-12b,strict,other,540,0.219,0.285,-0.067 +claude-sonnet-5,claude-opus-5,lenient,family,528,0.771,0.795,-0.024 +claude-sonnet-5,claude-opus-5,strict,family,528,0.581,0.574,0.008 +claude-sonnet-5,claude-haiku-4-5,lenient,family,630,0.617,0.645,-0.027 +claude-sonnet-5,claude-haiku-4-5,strict,family,630,0.375,0.377,-0.002 +claude-sonnet-5,gpt-5.6-luna,lenient,other,528,0.839,0.842,-0.003 +claude-sonnet-5,gpt-5.6-luna,strict,other,528,0.578,0.566,0.011 +claude-sonnet-5,grok-4.6,lenient,other,616,0.581,0.587,-0.006 +claude-sonnet-5,grok-4.6,strict,other,616,0.369,0.384,-0.015 +claude-sonnet-5,nemotron-nano-12b,lenient,other,630,0.502,0.543,-0.041 +claude-sonnet-5,nemotron-nano-12b,strict,other,630,0.249,0.245,0.004 +deepseek-v3.2,claude-opus-5,lenient,other,528,0.807,0.79,0.017 +deepseek-v3.2,claude-opus-5,strict,other,528,0.661,0.563,0.098 +deepseek-v3.2,claude-haiku-4-5,lenient,other,630,0.608,0.646,-0.038 +deepseek-v3.2,claude-haiku-4-5,strict,other,630,0.446,0.367,0.08 +deepseek-v3.2,gpt-5.6-luna,lenient,other,528,0.799,0.847,-0.048 +deepseek-v3.2,gpt-5.6-luna,strict,other,528,0.646,0.557,0.089 +deepseek-v3.2,grok-4.6,lenient,other,616,0.583,0.587,-0.004 +deepseek-v3.2,grok-4.6,strict,other,616,0.435,0.374,0.061 +deepseek-v3.2,nemotron-nano-12b,lenient,other,630,0.449,0.55,-0.101 +deepseek-v3.2,nemotron-nano-12b,strict,other,630,0.268,0.242,0.026 +glm-5,claude-opus-5,lenient,other,528,0.756,0.797,-0.041 +glm-5,claude-opus-5,strict,other,528,0.68,0.56,0.12 +glm-5,claude-haiku-4-5,lenient,other,630,0.571,0.652,-0.08 +glm-5,claude-haiku-4-5,strict,other,630,0.456,0.365,0.09 +glm-5,gpt-5.6-luna,lenient,other,528,0.794,0.848,-0.055 +glm-5,gpt-5.6-luna,strict,other,528,0.697,0.549,0.148 +glm-5,grok-4.6,lenient,other,616,0.563,0.59,-0.026 +glm-5,grok-4.6,strict,other,616,0.469,0.369,0.1 +glm-5,nemotron-nano-12b,lenient,other,630,0.462,0.549,-0.087 +glm-5,nemotron-nano-12b,strict,other,630,0.316,0.235,0.081 +gpt-6-luna,claude-opus-5,lenient,other,528,0.809,0.789,0.019 +gpt-6-luna,claude-opus-5,strict,other,528,0.381,0.603,-0.222 +gpt-6-luna,claude-haiku-4-5,lenient,other,630,0.697,0.633,0.064 +gpt-6-luna,claude-haiku-4-5,strict,other,630,0.232,0.398,-0.166 +gpt-6-luna,gpt-5.6-luna,lenient,family,528,0.877,0.836,0.041 +gpt-6-luna,gpt-5.6-luna,strict,family,528,0.35,0.599,-0.248 +gpt-6-luna,grok-4.6,lenient,other,616,0.61,0.583,0.028 +gpt-6-luna,grok-4.6,strict,other,616,0.265,0.399,-0.134 +gpt-6-luna,nemotron-nano-12b,lenient,other,630,0.616,0.526,0.09 +gpt-6-luna,nemotron-nano-12b,strict,other,630,0.146,0.26,-0.114 +grok-4.6,claude-opus-5,lenient,other,527,0.759,0.797,-0.038 +grok-4.6,claude-opus-5,strict,other,527,0.514,0.584,-0.07 +grok-4.6,claude-haiku-4-5,lenient,other,630,0.613,0.646,-0.033 +grok-4.6,claude-haiku-4-5,strict,other,630,0.297,0.388,-0.091 +grok-4.6,gpt-5.6-luna,lenient,other,528,0.816,0.845,-0.029 +grok-4.6,gpt-5.6-luna,strict,other,528,0.464,0.583,-0.119 +grok-4.6,grok-4.6,lenient,self,616,0.575,0.588,-0.013 +grok-4.6,grok-4.6,strict,self,616,0.331,0.389,-0.058 +grok-4.6,nemotron-nano-12b,lenient,other,630,0.506,0.542,-0.036 +grok-4.6,nemotron-nano-12b,strict,other,630,0.165,0.257,-0.092 +kimi-k3,claude-opus-5,lenient,other,528,0.814,0.789,0.026 +kimi-k3,claude-opus-5,strict,other,528,0.619,0.568,0.051 +kimi-k3,claude-haiku-4-5,lenient,other,630,0.697,0.633,0.064 +kimi-k3,claude-haiku-4-5,strict,other,630,0.408,0.372,0.036 +kimi-k3,gpt-5.6-luna,lenient,other,528,0.898,0.833,0.064 +kimi-k3,gpt-5.6-luna,strict,other,528,0.638,0.558,0.081 +kimi-k3,grok-4.6,lenient,other,616,0.602,0.584,0.018 +kimi-k3,grok-4.6,strict,other,616,0.412,0.378,0.035 +kimi-k3,nemotron-nano-12b,lenient,other,630,0.613,0.526,0.087 +kimi-k3,nemotron-nano-12b,strict,other,630,0.276,0.241,0.035 +qwen3-vl-235b,claude-opus-5,lenient,other,528,0.773,0.795,-0.022 +qwen3-vl-235b,claude-opus-5,strict,other,528,0.684,0.559,0.124 +qwen3-vl-235b,claude-haiku-4-5,lenient,other,630,0.637,0.642,-0.005 +qwen3-vl-235b,claude-haiku-4-5,strict,other,630,0.5,0.359,0.141 +qwen3-vl-235b,gpt-5.6-luna,lenient,other,528,0.803,0.847,-0.044 +qwen3-vl-235b,gpt-5.6-luna,strict,other,528,0.699,0.549,0.15 +qwen3-vl-235b,grok-4.6,lenient,other,616,0.575,0.588,-0.013 +qwen3-vl-235b,grok-4.6,strict,other,616,0.461,0.37,0.091 +qwen3-vl-235b,nemotron-nano-12b,lenient,other,630,0.514,0.541,-0.026 +qwen3-vl-235b,nemotron-nano-12b,strict,other,630,0.348,0.231,0.117 diff --git a/dokimi/docs/judge-panel-study.md b/dokimi/docs/judge-panel-study.md new file mode 100644 index 000000000..350e20cab --- /dev/null +++ b/dokimi/docs/judge-panel-study.md @@ -0,0 +1,473 @@ + + +# How much the judge moves skill-novelty results + +The skill-novelty study grades every probe answer with one LLM judge (gpt-5.6-luna) and +reports, per answer model, the share of claims a skill *added*, the share the model already +*knew*, the share the skill *degraded*, and the *placebo-gated* added share (added, and the +placebo arm is wrong too). This note re-grades a stratified sample of the same answers with +seven more judges from seven model families. It asks four questions: + +1. How far does each headline number move when only the judge changes? +2. Do judges agree with each other, and do they favour their own family? +3. When the judges are a panel, how much do the conclusions depend on the aggregation rule, + and how many judges are enough? +4. What can a Dokimi user configure today, and what is missing? + +Short answer: judge choice moves the lenient added share by up to 0.23 for one answer model. +The original judge is the most lenient of the eight, so the published lenient added shares sit +at the low end of the panel. The extremes of the answer-model ranking do not change: on the lenient placebo-gated added +share, nemotron-12b is first and opus-5 last under every judge. The models in between swap. Majority, median +and Dawid-Skene agree on 99.3% of rows. Three judges from three families get most +of the stability of seven. A newer generation of the same judge (section 6) is about as +different as another vendor: κ 0.61–0.72, and up to 0.15 on one added share. + +## Setup + +| | | +|---|---| +| Answers | 2,664 existing answers: 444 (probe, answer model) cells × 6 arms (`no_skill`, `root`, `routed`, `full`, `placebo`, `shuffled`). Plus 268 fresh repeats of `no_skill` for the noise floor. | +| Answer models | claude-opus-5, gpt-5.6-luna, grok-4.6 (88 cells each); claude-haiku-4-5, nemotron-nano-12b (90 each). | +| Sampling | Per model, cells drawn from those with all six arms, stratified by skill × reversal (adversarial) probe, proportional to stratum size. Seed 20261108. | +| Judges | gpt-5.6-luna (original grades, as recorded), claude-sonnet-5, gpt-6-luna, grok-4.6, kimi-k3, deepseek-v3.2, glm-5, qwen3-vl-235b. | +| Prompt | The study's grading prompt unchanged: 6 verdicts, answer truncated to 6,000 characters, max 1,200 output tokens, JSON repair by claude-haiku-4-5. | +| Temperature | 0, except gpt-6-luna and kimi-k3, which reject it and run at their default. | +| Coverage | 2,932 rows × 7 new judges, 1 unparseable verdict. The original judge has no grade on the repeats. | +| Noise floor | Repeat calls were possible only for haiku-4.5, grok-4.6 and nemotron-nano-12b. The other two answer models are no longer served. | +| Date | 2026-11-08 | + +The headline categories follow the paper. Reversal probes are excluded, so the shares are over +60–62 paired probes per answer model. With n = 60 a share near 0.3 has a sampling SE of about +0.06. The judge-to-judge differences below are paired: the same answers, graded by a +different judge. They are therefore much less noisy than two independent samples of that +size would be. The confidence intervals in the paper are the right yardstick for the +sampling error. + +Scripts: `research/judge-panel/judge_panel.py` (sample, repeat, grade), `analyze.py` (all +tables) and `fig.py` (figures). Tables: `docs/assets/judge-panel/*.csv`. + +## 1. Per-judge headline numbers + +![Placebo-gated added share per answer model, one dot per judge](assets/judge-panel/gated_added_by_judge.svg) + +**Placebo-gated added share, lenient** (correct = the first three verdicts): + +| Judge | opus-5 | gpt-5.6 | haiku-4.5 | grok-4.6 | nemotron-12b | +|---|---:|---:|---:|---:|---:| +| gpt-5.6-luna (original) | 0.10 | 0.11 | 0.10 | 0.34 | 0.37 | +| claude-sonnet-5 | 0.16 | 0.21 | 0.30 | 0.32 | 0.48 | +| gpt-6-luna | 0.08 | 0.19 | 0.15 | 0.34 | 0.40 | +| grok-4.6 | 0.21 | 0.24 | 0.32 | 0.37 | 0.52 | +| kimi-k3 | 0.14 | 0.19 | 0.18 | 0.35 | 0.38 | +| deepseek-v3.2 | 0.13 | 0.16 | 0.30 | 0.35 | 0.50 | +| glm-5 | 0.18 | 0.24 | 0.33 | 0.32 | 0.47 | +| qwen3-vl-235b | 0.18 | 0.23 | 0.25 | 0.34 | 0.45 | +| **range** | **0.13** | **0.13** | **0.23** | **0.05** | **0.15** | +| 7-judge majority | 0.16 | 0.21 | 0.32 | 0.34 | 0.43 | + +**Already known, lenient:** + +| Judge | opus-5 | gpt-5.6 | haiku-4.5 | grok-4.6 | nemotron-12b | +|---|---:|---:|---:|---:|---:| +| gpt-5.6-luna (original) | 0.82 | 0.79 | 0.73 | 0.52 | 0.43 | +| claude-sonnet-5 | 0.65 | 0.66 | 0.47 | 0.45 | 0.15 | +| gpt-6-luna | 0.79 | 0.69 | 0.67 | 0.50 | 0.45 | +| grok-4.6 | 0.66 | 0.60 | 0.43 | 0.43 | 0.23 | +| kimi-k3 | 0.68 | 0.74 | 0.60 | 0.47 | 0.35 | +| deepseek-v3.2 | 0.74 | 0.69 | 0.48 | 0.47 | 0.20 | +| glm-5 | 0.66 | 0.56 | 0.42 | 0.43 | 0.20 | +| qwen3-vl-235b | 0.69 | 0.61 | 0.53 | 0.48 | 0.32 | + +**Placebo-gated added share, strict** (correct = `correct_complete` only): + +| Judge | opus-5 | gpt-5.6 | haiku-4.5 | grok-4.6 | nemotron-12b | +|---|---:|---:|---:|---:|---:| +| gpt-5.6-luna (original) | 0.23 | 0.27 | 0.32 | 0.34 | 0.40 | +| claude-sonnet-5 | 0.27 | 0.26 | 0.33 | 0.45 | 0.48 | +| gpt-6-luna | 0.26 | 0.19 | 0.23 | 0.29 | 0.28 | +| grok-4.6 | 0.26 | 0.29 | 0.40 | 0.42 | 0.38 | +| kimi-k3 | 0.27 | 0.31 | 0.35 | 0.40 | 0.48 | +| deepseek-v3.2 | 0.23 | 0.27 | 0.35 | 0.42 | 0.47 | +| glm-5 | 0.29 | 0.37 | 0.35 | 0.45 | 0.52 | +| qwen3-vl-235b | 0.29 | 0.31 | 0.42 | 0.43 | 0.52 | + +Ungated added shares, degraded shares and the strict tables of every category are in +`per_judge_headline.csv`, and their min/median/max across judges in `headline_range.csv`. + +What changes and what does not: + +- **The original judge is the most lenient.** Its lenient rate over all graded rows is 0.76, + against 0.62–0.72 for the other judges. It accepts more no-skill answers, so "known" goes up + and "added" goes down. It sits at or next to the bottom of the panel's lenient added share for opus, + GPT-5.6 and Haiku. The paper's lenient added shares are therefore conservative. The extreme + case is Haiku: the original judge says 0.10, the 7-judge majority 0.32. That moves Haiku + from "the skill adds little" to the same level as Grok. + +- **The strict scale moves less in relative terms.** Strict gated added varies by 0.06–0.23 + per model. gpt-6-luna is the outlier: it gives the lowest strict rate of any judge (0.27, + against 0.35–0.53), because it marks dropped qualifiers more often. Its strict added share + is the lowest for four of the five models. + +- **Grok-4.6 answers are judge-proof.** Gated added is 0.32–0.37 under every judge. Its + answers are the most clear-cut: only 11% of the rows lack unanimity, against 20–35% for + the other answer models. + +- **The extremes of the ranking are stable; the middle is not.** Under every one of the eight + judges, lenient gated added puts nemotron-12b first and opus-5 last. The models in between + swap: haiku-4.5 is below grok-4.6 under the original judge (0.10 against 0.34) and above it + under glm-5 (0.33 against 0.32), and GPT-5.6 and haiku-4.5 also exchange order across judges. + On strict, six of eight put + nemotron-12b first and grok-4.6 second. gpt-6-luna swaps the two, and grok-4.6 ranks its + own answers first and haiku-4.5 second. All eight put opus-5 or GPT-5.6 last. A statement + about the extremes, such as "the skill adds most for nemotron-12b and least for opus-5", + survives any choice of judge; one about the order of the middle models does not. A single number such as "13% added for opus" does not. + +- **Degraded stays small under every judge**: 0.00–0.07 lenient and 0.00–0.07 strict, within + the noise floor. + +### Noise floor per judge + +This is the added and degraded share between `no_skill` and a fresh repeat of the same call +(lenient): + +| Judge | haiku-4.5 add / degr | grok-4.6 add / degr | nemotron-12b add / degr | +|---|---|---|---| +| claude-sonnet-5 | .02 / .02 | .03 / .02 | .05 / .05 | +| gpt-6-luna | .03 / .08 | .07 / .02 | .05 / .10 | +| grok-4.6 | .02 / .02 | .05 / .05 | .02 / .08 | +| kimi-k3 | .03 / .03 | .07 / .05 | .10 / .07 | +| deepseek-v3.2 | .03 / .03 | .05 / .05 | .10 / .05 | +| glm-5 | .02 / .03 | .05 / .07 | .03 / .05 | +| qwen3-vl-235b | .02 / .00 | .03 / .02 | .05 / .03 | +| 7-judge majority | .03 / .02 | .03 / .02 | .05 / .08 | + +The floor is 0.00–0.10 for every judge, in line with the paper's 0.05–0.10. The judges +running at default temperature (gpt-6-luna, kimi-k3) are at the high end. The floor was measured +for haiku-4.5, grok-4.6 and nemotron-12b only; opus-5 and GPT-5.6 have no repeat draws, so +their judge shift has no floor to be compared with. The shift from the original judge to the +7-judge majority is clearly above the floor for haiku-4.5 (gated 0.10 to 0.32, majority floor +0.03). For nemotron-12b it is not separated from the noise: the gated share moves by 0.07 +(0.37 to 0.43) against a majority floor of 0.08 and a per-judge floor of up to 0.10, and the +ungated share by 0.05. For grok-4.6 it is not above the floor. + +## 2. Agreement and self-preference + +Cohen's kappa over all 28 judge pairs (full table: `pairwise_kappa.csv`): + +| Scale | min | mean | max | +|---|---:|---:|---:| +| 6 verdicts | 0.46 | 0.61 | 0.72 | +| lenient (binary) | 0.70 | 0.79 | 0.88 | +| strict (binary) | 0.46 | 0.70 | 0.86 | + +- The lenient range 0.70–0.88 matches the paper's two-judge kappa of 0.76–0.88. The 6-way + scale is too fine to report as a single judge's call. + +- **The highest pair is within one family**: gpt-5.6-luna and gpt-6-luna, κ = 0.88. Next + come sonnet-5/grok-4.6 (0.86), glm-5/grok-4.6 (0.85) and gpt-5.6-luna/kimi-k3 (0.85), the + last a pair of lenient judges. Two OpenAI judges are + not two independent opinions. A panel needs different families, not just different + models. + +- Disagreement is not spread evenly across rows. A row lacks unanimity in 30% of `no_skill` + rows (31% for the repeats), 21% of `placebo` rows and 17–19% of the with-skill arms. It + also lacks unanimity in 27% of reversal-probe rows, and in 35% of nemotron's answers + against 11% of grok's. The no-skill and placebo arms are the boundary that decides + "known" against "added", so that is where the judge matters most. + +**Self-preference.** For each judge and answer model we take the judge's lenient (strict) +rate minus the mean rate of the other judges on the same rows. That is the judge's offset. +The self-preference is the offset on its own family minus its mean offset on other families +(`self_preference.csv`): + +| Judge | Graded own family | Lenient self-pref. | Strict self-pref. | +|---|---|---:|---:| +| gpt-5.6-luna | itself | +0.00 | −0.02 | +| gpt-6-luna | gpt-5.6-luna | −0.01 | −0.09 | +| claude-sonnet-5 | opus-5, haiku-4.5 | −0.01 | +0.00 | +| grok-4.6 | itself | +0.02 | +0.04 | + +No judge favours its own family by more than 0.04. The raw offset of gpt-5.6-luna on its own +answers (+0.07 lenient) looks like self-preference, but it is the same +0.07 it gives every +model: general leniency, not favouritism. gpt-6-luna is if anything harsher on GPT-5.6 +(−0.09 strict). Within this sample, family exclusion is a hygiene rule, not a correction for a +measured bias. The same-family kappa above is the stronger reason to mix families. + +## 3. The judges as a panel + +Each rule decides a row correct (lenient) from the seven new judges' verdicts. A row needs +verdicts from at least four of them. Placebo-gated added / known share: + +| Rule | opus-5 | gpt-5.6 | haiku-4.5 | grok-4.6 | nemotron-12b | +|---|---|---|---|---|---| +| majority | .16 / .69 | .21 / .66 | .32 / .48 | .34 / .47 | .43 / .27 | +| median score ≥ 0.4 | .16 / .69 | .21 / .66 | .32 / .48 | .34 / .47 | .43 / .27 | +| Dawid-Skene (binary) | .16 / .66 | .21 / .66 | .33 / .45 | .34 / .47 | .43 / .25 | +| mean score ≥ 0.4 | .26 / .56 | .32 / .52 | .42 / .32 | .37 / .34 | .52 / .12 | +| unanimous | .21 / .56 | .26 / .52 | .32 / .38 | .32 / .40 | .52 / .12 | +| any correct | .08 / .85 | .14 / .81 | .12 / .68 | .37 / .52 | .33 / .52 | +| majority, strict scale | .29 / .45 | .27 / .37 | .40 / .15 | .48 / .18 | .50 / .05 | + +Agreement of each rater with the 7-judge Dawid-Skene consensus, over all rows +(`agreement_with_ds.csv`): + +| Rater | agree | | Rater | agree | +|---|---:|---|---|---:| +| majority | 0.993 | | grok-4.6 | 0.965 | +| median | 0.993 | | claude-sonnet-5 | 0.959 | +| mean ≥ 0.4 | 0.907 | | glm-5 | 0.949 | +| unanimous | 0.894 | | kimi-k3 | 0.938 | +| any correct | 0.889 | | qwen3-vl-235b | 0.935 | +| | | | deepseek-v3.2 | 0.926 | +| | | | gpt-6-luna | 0.917 | +| | | | gpt-5.6-luna (original) | 0.912 | + +- **Majority, median and Dawid-Skene are the same rule in practice** here. They agree on + 99.3% of rows and give the same headline numbers to within 0.035. Dawid-Skene estimates + that gpt-6-luna (specificity 0.79) and kimi-k3 (0.81) say "correct" too easily, while + glm-5 (0.98) and grok-4.6 (0.96) rarely do. With seven judges that knowledge changes almost + nothing. With two or three judges, one of them lenient, it could matter. That was not + tested here. + +- **The rule is as large a knob as the judge.** Unanimous and any-correct are not + conservative choices. They push one arm and pull the other, because the same rule applies + to `no_skill` and to `full`. "Any correct" calls more no-skill answers known, so added + falls: Haiku 0.12. "Unanimous" and the mean score do the reverse: Nemotron 0.52. The spread + between rules for one model (Haiku 0.12–0.42) is larger than the spread between judges. + The rule must be fixed before a run and reported with the result. + +- The mean score with the study's partial credit (0.4 for `partially_correct`) is not the + same as a lenient majority. A row where three judges say partially correct and four say + wrong has a mean of 0.17 but would need a 0.4 average, so the mean rule behaves almost like + a strict vote on the lenient scale. + +**How many judges are enough.** Subsets of the seven new judges vote by majority, with ties +counted as wrong. For each size we measure agreement with the 7-judge consensus and the spread +of the lenient gated added share between subsets of that size (`panel_size.csv`): + +| Judges | subsets | mean agree | worst agree | SD of gated added | range of gated added | +|---:|---:|---:|---:|---:|---:| +| 1 | 7 | 0.941 | 0.917 | 0.039 | 0.115 | +| 2 | 21 | 0.943 | 0.924 | 0.026 | 0.105 | +| 3 | 35 | 0.966 | 0.947 | 0.025 | 0.095 | +| 4 | 35 | 0.969 | 0.949 | 0.021 | 0.079 | +| 5 | 21 | 0.980 | 0.970 | 0.021 | 0.069 | +| 6 | 7 | 0.982 | 0.973 | 0.011 | 0.029 | + +Two judges are barely better than one, because a tie counts as wrong, so two judges act as a +unanimity rule. Three judges are the first real gain: the SD of the headline number falls from +0.039 to 0.025, below the noise floor. Five judges add little more. Recommendation: **three +judges from three families with majority vote** for routine runs, and **five** for a published +number. Report the per-judge numbers alongside either. + +## 4. Judge panels in Dokimi today + +The study above used its own scripts. This section is about the product, as of `main` on +2026-11-08. + +What exists: + +- **One judge per run.** `arms run` / `arms matrix-run` take `--judge`, `--judge-model`, + `--judge-provider` and `--judge-temperature`. The default is `claude-sonnet-5-5` from + `dokimi/defaults.py`. + +- **Role panels.** `--judge-panel ROLES` (`eval/panel.py` `JudgePanel`) runs several role + prompts (technical expert, developer advocate, doc reviewer) on **the same model**. Votes + are combined by a role-weighted mean of the 0–100 dimension scores, with a quorum of a + majority of the panelists. Disagreement is reported per answer as `std_aggregate`, + `agreement_score = 1 − std/50` and `disagreement_flag` when std > `DISAGREEMENT_STD` (15, a + constant). `gate/trust_gate.py` warns when the mean agreement is low. + +- **Mixed models, one per role.** `eval score panel --judge-configs role:model:provider,...` + is the only place where one panel mixes models, and each model is tied to a role. + +- **Post-hoc judge robustness.** `eval judge agreement` (`commands/judge_reliability.py`, + `eval/cross_judge.py`) re-scores an answers file with `--judges provider/model,...`. It + computes each judge's per-arm delta as a separate measure with a sign test and Holm + correction, and labels the arm `CONSISTENT`, `LEANING`, `JUDGE-DEPENDENT`, `NO DETECTABLE + EFFECT` or `NOT MEASURED`. Judges that fail `--validity` do not vote. `--strict` makes + `JUDGE-DEPENDENT` an error. This is the right shape, per-judge numbers rather than an + average that hides the spread, and it already guards against "(one judge)" results. + +- **Self-judging** is a warning on an exact model match (`utils.warn_judge_independence`). + It never refuses, and a judge from the same family passes silently. + +What is missing (each item was needed to produce this note): + +1. **Naming N judge models for a run.** `arms run` cannot take a list of models. Panels are + roles × one model, and robustness comes only from a second, after-the-fact command. +2. **Choosing the aggregation rule.** The weighted mean and quorum are hard-coded, and the + disagreement threshold is a constant. Section 3 shows the rule moves headline numbers as + much as the judge does. +3. **Family awareness.** Nothing knows that gpt-5.6-luna and gpt-6-luna are one family. The + independence check is exact-match only and can't be made to refuse. +4. **Agreement statistics.** No kappa, Krippendorff's α or per-judge reliability is computed + between answer judges. Cohen's kappa exists only for human labels in `repo hits`. +5. **Per-judge headline numbers in the arms report.** Per-role votes are in the JSON, but the + report shows only the panel aggregate. +6. **Disagreement by arm.** The judges disagree most on the baseline and placebo arms, which + decide every delta. No report breaks disagreement down by arm. +7. **No cost control for a panel.** A full panel costs N× the grading. Nothing grades with one + judge and sends only the contested rows to more. + +## 5. Proposal + +All of the following reuse code that exists: the `--judges` spec parser of `eval judge +agreement`, `JudgePanel` / `_aggregate`, `cross_judge.label`, and `same_evaluator`. + +**CLI** (proposed; `arms run` has none of these flags yet) + +```text +arms run ... \ + --judges anthropic/claude-sonnet-5-5,openai/gpt-6-luna,bedrock/kimi-k3 \ + --aggregate majority # majority | median | mean | unanimous | any | dawid-skene + --judge-family-policy refuse # refuse | warn | allow (judge vs answer model, and judge vs judge) + --judge-escalate contested # grade with judges[0]; send rows near the threshold, and + # the baseline/placebo arms, to the rest of the panel +``` + +- `--judges` on `arms run` and `arms matrix-run` takes the same `provider/model` list as + `eval judge agreement`. `--judge-model` / `--judge-provider` stay as the one-judge form. + `--judge-panel ROLES` stays orthogonal: roles × judges, with roles averaged inside a judge + first. + +- `--aggregate` defaults to `majority` for verdict-style grading and to `median` for 0–100 + scores. The latter is the robust form of today's mean, and matches majority in section 3. + `dawid-skene` needs ≥ 3 judges and ≥ 50 rows and falls back to `majority` with a warning + below that. The rule, and the judges' estimated sensitivity and specificity, go into the + run's provenance digest as `_aggregation_policy` does today. + +- `--judge-family-policy`: one `family` field per model in `defaults.py`, next to + `JUDGE_MODEL_FOR_PROVIDER`, covering the same table the novelty study keeps. `refuse` is + the default for `--judges`. It refuses a judge from the answer model's family, or two judges + from one family, unless `allow` is given, and records which policy was in force. + +- `--judge-escalate contested` is the cheap panel. Its cost and accuracy are not measured + yet. The case for it is that about 80% of rows are unanimous, and the contested ones + concentrate in two arms. This study's data can simulate it before it is built. + +**Config** (the `judge:` block in a product config, or a run YAML), so that a product fixes its +panel once: + +```yaml +judge: + judges: + - {provider: anthropic, model: claude-sonnet-5-5} + - {provider: openai, model: gpt-6-luna, temperature: null} # null = model default + - {provider: bedrock, model: kimi-k3, temperature: null} + aggregate: majority + min_quorum: 2 # rows with fewer valid votes are insufficient_evidence + family_policy: refuse + disagreement_threshold: 15 # today's DISAGREEMENT_STD, now configurable + escalate: contested +``` + +`temperature: null` is needed because gpt-6-luna and kimi-k3 reject `temperature=0`. + +**Report** (arms report and `eval judge agreement`): + +- A per-judge table of every headline number (arm means, deltas, added/known/degraded where + applicable), with the panel row under it. That is section 1 of this note, generated. + +- Pairwise Cohen's kappa and Krippendorff's α for the binary and full scales, plus a + same-family flag on pairs. + +- The share of rows that lack unanimity, per arm. + +- A **judge-sensitive** flag on each headline delta, raised inline (not only post hoc) when + either the sign differs between judges or the range across judges exceeds the noise floor. + It reuses the `JUDGE-DEPENDENT` label, and `--strict` makes it an error as it does today. + +**Order of work:** (1) `--judges` + `--aggregate` on `arms run`, with per-judge numbers in +the report. (2) The family table and policy. (3) Kappa/α and disagreement by arm. (4) +Escalation. Items 1–3 change no default output for a one-judge run. + +## 6. Old and new judge generations + +Judges get replaced. Section 1 used one judge per family, the newest each gateway served +then, plus the original gpt-5.6-luna. This section re-grades the same 2,932 rows with seven +more judges and compares generations within a family: + +- **Older judges:** kimi-k2.5, claude-haiku-4-5, deepseek-r1. +- **Newer judges:** gpt-6-sol, claude-opus-5-5. +- **Two new families:** nemotron-super-3-120b, minimax-m2.5. + +The newest names asked for (gpt-6.1-luna, deepseek-4.2, qwen3.8) are not served by the +gateway used here, so gpt-6-luna/gpt-6-sol, deepseek-v3.2 and qwen3-vl-235b are the newest +of their families in this note. Coverage: 2,932 rows × 7 judges, with 1 unparseable verdict. +gpt-6-sol and claude-opus-5-5 reject temperature 0 and run at their default. +Script: `research/judge-panel/gen.py`. Tables: `docs/assets/judge-panel/generations/*.csv`. + +| Family: old → new | κ old vs new | Lenient rate, old → new | Lenient added share, old → new, by answer model (opus-5 / haiku-4.5 / gpt-5.6 / grok-4.6 / nemotron-nano) | +|---|---|---|---| +| openai: gpt-5.6-luna → gpt-6-luna | 0.72 | 0.756 → 0.714 | 0.13→0.11 / 0.12→0.15 / 0.15→0.23 / 0.34→0.34 / 0.43→0.42 | +| openai: gpt-5.6-luna → gpt-6-sol | 0.70 | 0.756 → 0.713 | 0.13→0.16 / 0.12→0.18 / 0.15→0.16 / 0.34→0.29 / 0.43→0.50 | +| moonshot: kimi-k2.5 → kimi-k3 | 0.65 | 0.599 → 0.716 | 0.26→0.23 / 0.32→0.23 / 0.31→0.23 / 0.37→0.35 / 0.52→0.45 | +| anthropic: haiku-4.5 → sonnet-5 | 0.66 | 0.651 → 0.652 | 0.15→0.21 / 0.20→0.30 / 0.21→0.23 / 0.34→0.32 / 0.53→0.55 | +| anthropic: sonnet-5 → opus-5.5 | 0.71 | 0.652 → 0.718 | 0.21→0.18 / 0.30→0.20 / 0.23→0.23 / 0.32→0.35 / 0.55→0.58 | +| deepseek: r1 → v3.2 | 0.61 | 0.735 → 0.639 | 0.13→0.18 / 0.15→0.30 / 0.15→0.21 / 0.37→0.35 / 0.47→0.57 | + +The lenient rate is the share of a judge's verdicts that count as correct under lenient +scoring, over all 2,932 rows (2,664 for gpt-5.6-luna, which has no grades on the repeats; its +κ pairs use those 2,664 rows). κ per pair: `generation_kappa.csv`. One probe moves an added share by about 0.017. + +What a generation update does to judgment: + +- **A new generation is about as different as another vendor.** κ between generations of one + family is 0.61–0.72. The mean κ of a judge against the other 14 is 0.51–0.64 + (`judge_mean_kappa.csv`). Re-grading with a newer judge of the same family is a judge change, + not a re-run. + +- **Leniency moves, with no common direction.** + + - Kimi and Anthropic got more lenient (+0.12, +0.07). + - OpenAI and DeepSeek got stricter (−0.04, −0.10). + - OpenAI's strict rate fell most: gpt-5.6-luna marks 36% of answers `correct_complete`, + gpt-6-luna 27%. Strict headlines graded by gpt-5.6-luna are therefore not comparable with + ones graded by gpt-6. + +- **One answer model's added share moves by up to 0.15 between generations.** Two cases: + + - haiku-4.5 answers: deepseek-r1 0.15 vs v3.2 0.30. + - The same answers: sonnet-5 0.30 vs opus-5.5 0.20. + + That is two-thirds of the cross-family range in section 1 (0.23), and above the 0.10 noise floor. + +- **Newer is not closer to the consensus.** gpt-6-luna has the lowest mean κ of the + OpenAI, Anthropic, Kimi, DeepSeek, xAI, GLM and Qwen judges (0.54, against 0.58 for gpt-5.6-luna). kimi-k3 has the highest (0.64). + Recency does not stand in for judge quality. + +- **A three-family panel absorbs most of the change.** The median panel + {gpt-5.6-luna, kimi-k2.5, haiku-4.5} vs {gpt-6-luna, kimi-k3, opus-5.5} (`generation_panels.csv`): + + - Lenient added share moves by at most 0.067 (haiku-4.5 answers, 0.27 → 0.20), within the + noise floor. On opus-5 and grok-4.6 answers it does not move. + - Strict moves more, by up to 0.11 (gpt-5.6 answers, 0.40 → 0.29), which follows the + OpenAI strict shift above. + - In both panels, opus-5 answers gain least and nemotron-nano answers gain most. + +For Dokimi: record the judge's exact model id with every grade (it does), and treat a +judge-generation upgrade like a judge change: re-grade the baseline with the new judge +before comparing to old numbers, never mix grades of two generations in one comparison, and +prefer a panel, which turns a 0.15 single-judge shift into ≤0.07 lenient. + +## Limits + +- 60–62 paired probes per answer model: enough to compare judges on the same rows, not to + re-estimate the paper's numbers. Use the paper's intervals for those. + +- The original judge and two answer models are no longer served, so the noise floor covers + three answer models, and the original judge has no grades on the repeats. + +- Two judges ran at default temperature. Their noise-floor rows include their own sampling + noise. + +- Dawid-Skene here is binary (lenient) and one-coin per class. A 6-class version would + separate "lenient on qualifiers" from "lenient on wrong answers". + +- No deterministic ground truth covers these skills. The grounded-grader pilot covers other + skills, so there is no "true" label here: agreement is with the panel consensus, not with + the truth. See `grading-tiers.md` when it lands. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a87a843cc..453a52bf5 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1612,6 +1612,8 @@ {"key": "--extract-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--include-resources", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--judge", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-aggregate", "action": "_StoreAction", "choices": ["'majority'", "'median'", "'unanimous'"], "const": null, "default": "median", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge2", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, @@ -1708,6 +1710,8 @@ {"key": "--extract-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--extract-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--judge", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--judge-aggregate", "action": "_StoreAction", "choices": ["'majority'", "'median'", "'unanimous'"], "const": null, "default": "median", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-model", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge-provider", "action": "_StoreAction", "choices": ["'amazon-bedrock'", "'anthropic'", "'claude-cli'", "'google-vertex'", "'openai'", "'openai-codex'", "'openrouter'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--judge2", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 6bfd6bd21..f46baf6e9 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -103,20 +103,24 @@ be right or wrong) and `harmful_conflicting` (below), which overrides the other `redundant` or `degraded` means known. An atom that was never probed counts as new, which is the direction that does not claim the model knew something nobody asked it. The model is never asked whether it knew something. + - **Needed vs inert is judged.** The extractor model sees the task and answers, per atom, *"can acting on this task go wrong, or be much slower, without knowing it?"* The task is built from the artifact's `name` and `description` plus the need graph's `core` and `rare_critical` nodes, never from the body, so an artifact cannot make itself relevant by containing a sentence. Eval files are not part of it either. + - **Reliability.** In the study both axes were judged, and an independent reviewer agreed with 71.6% of assignments (853 atoms), with useful-vs-inert at ±3 points and known-vs-novel at ±2; `scaffolding` was the most stable share ([§4](skill-novelty-evidence.md)). Here only the ±3 carries over. Known vs new is read off the paired probes instead, so its error is the probe grading's: two judges agreed on 85% of verdicts on the correct-ish binary collapse (above). + - **Inert is not "safe to delete".** The `drop-inert 0.60` candidate, which cuts inert text under a 0.60 token budget (the suffix is a budget, as for every §10.6 candidate, not "all inert atoms at once"), was non-inferior on pooled data (+0.005, CI [−0.045, +0.057]) and *worse* on `nemotron-nano-12b` for `dpnp-quickstart` ([§10.6](skill-novelty-evidence.md)). + - An atom the value judge did not answer for gets no category. The artifact says how many (`N of M atoms got no need/scaffolding verdict`, split into calls that failed, rows the reply skipped, malformed rows, and replies with no usable verdict row; an empty list for a batch is @@ -131,6 +135,7 @@ marked. Read it as a review queue: - **Internal only.** It says the two statements disagree, not which is wrong. That needs a primary source, which this layer does not consult. + - **Lower bound.** Only atoms in the same call are compared. `analyze` sends the whole probed atom set (at most `max_atoms`, 120 by default) in one call, so every pair is compared unless a call fails or its reply has to be split. The artifact records the share as @@ -146,6 +151,7 @@ marked. Read it as a review queue: `harmful_conflicting` row reads `not measured (0 found)`. An artifact written before `conflict_coverage` existed has no such field and keeps the old reading, a 0 that counts as cleared, unless its atom set was capped; re-run `analyze` to check it. + - **Unchecked atoms.** A call whose reply does not parse, or whose rows name no pair of the batch (an empty list is fine: it means no conflicts), is split and re-asked only while both halves keep two atoms, since one statement has no pair to look at; a chunk of two or three whose @@ -153,6 +159,7 @@ marked. Read it as a review queue: a chunk (the 41st, say), are counted in `N of M atoms were not checked for conflicts`, split by cause (`K in calls that failed (last error: …)`, `K whose reply named no usable pair`, `K alone in a batch, with no pair to check`). Their ids are not written to the artifact. + - **False positives (anecdotal, not a rate).** Spot-checking 7 of the 31 pairs one run reported ([mattpocock/skills](https://github.com/mattpocock/skills) @ `49dd158`) found 4 real, 1 ambiguous and 2 false positives: steps of one procedure read as alternatives ("watch it fail" @@ -205,6 +212,7 @@ Consequences for authoring: - A skill that looks bloated against a frontier model may be correctly sized for a small one. Decide which tier you are writing for, and record it. + - `degraded` rises as the model weakens (0.019 → 0.076). Weak models copy your errors; strong models overrule them. A stale line in a skill is a bigger risk to a small model — **except** for reversed or out-of-date claims, where the opposite holds: strong models resisted planted inversions *less* @@ -274,7 +282,7 @@ Anthropic's). model that wrote the saved answer key (the atoms file's `extractor`) against the judge. The same model, or two models of one vendor family (the vendor is read off the model id: `claude`, `sonnet`, `haiku`, `opus` and `anthropic` are one family; `gpt` and `openai` another; `gemini` and -`mistral` likewise), prints a `WARNING` on stderr. The vendor is read off the name, so a model +`mistral` likewise; `grok` and `xai` another), prints a `WARNING` on stderr. The vendor is read off the name, so a model served locally under a vendor's name (`gpt-oss-120b` is OpenAI's) counts as that vendor; ids that name no known vendor never match. `analyze` records the result in the artifact's `run_config.extractor_judge` (`extractor`, `judge`, `same_model`, `same_vendor`); `probe` @@ -298,6 +306,49 @@ family with one of them, and the warning does not check that; say so when you qu A third family keeps all three apart. Changing the extractor changes the atoms, so compare only cells that name the same extractor. +## A judge panel + +One judge is one opinion. The [judge-panel study](judge-panel-study.md) graded the same answers with +seven judges. The added share of one answer model moved by up to 0.23 with the judge. The +original single judge was the most lenient of all, so its added shares sat at or near the +bottom of the panel's range. `analyze` and `probe` take a panel: + +```bash +python cli.py skill novelty probe --artifact skill:path/to/skill --atoms atoms.json --model grok-4.6 --provider openrouter --judge openrouter/openai/gpt-6-luna --judge openrouter/anthropic/claude-sonnet-5 --judge openrouter/moonshotai/kimi-k3 --judge-aggregate median +``` + +- `--judge PROVIDER/MODEL`, repeated, is the panel. It replaces `--judge-model` and + `--judge-provider`, and the two forms cannot be mixed. The first judge is the default + extractor. Every judge is checked against the answerer like a single judge. + +- `--judge-aggregate` turns the verdicts into one per answer: + + - `median` (default): the median of the verdict scale, rounding toward wrong on an even + panel. + - `majority`: correct when more than half the judges say so. + - `unanimous`: correct only when all of them say so. + + In the study, median and majority matched Dawid-Skene on 99.3% of rows. Unanimous and + any-correct moved the headline by up to 0.20. A judge whose grade fails is left out of the + vote. + +- The record gets a `judge_panel` block with: + + - the judges and the rule; + - each judge's own paired 2×2 (`per_judge_paired`); + - the judge range of the added share (`added_share_range`); + - `judge_sensitive`, set when the lenient range exceeds 0.10 (the study's noise floor). + + Each run keeps every judge's verdict in `judge_verdicts`. `analyze` writes them beside + the record as `.judge_verdicts.json`; the report names the panel instead of one judge. + +- Two judges of one vendor family (the families named above) print a warning. Same-family judges agree with each other + more than with the rest, so the panel then has fewer independent votes than judges. Three + judges from three families cut the spread across panels from SD 0.039 to 0.025. + +Each extra judge grades every answer again, so it adds the judge's share of the cost. +`--estimate-only` counts it, and `--max-usd` caps the whole panel. + ## Using it as an authoring tool The commands in order, with the costs and the three free steps that come first, are in @@ -370,6 +421,7 @@ Two derived views an author acts on directly: - **Dead weight**: the dim spans, sorted by token cost. The largest single win in most artifacts. Unread spans are not listed. + - **Sole carriers**: spans that are the only source of some atom. These are what a careless edit silently deletes; the shortener pins them. @@ -385,9 +437,11 @@ the highlight is load-bearing. Three measured constraints ship with it: - **Do not attribute by lexical overlap** with the atom statement. It misses negation — negation words are function words — so its extra loss on `negative-constraint` atoms is +0.143 against an LLM attributor's +0.392. The cheap attributor is weakest on exactly the knowledge that matters most. + - **Carry the heading path** wherever a span is checked. Atom recovery from a bare span is 0.833 and rises to 0.907 with the heading path prepended, for about 16 tokens: a span reading "Use 4 for most workloads" is not a bad citation, it is a sentence whose subject lives in its heading. + - **Render an unlocated state.** 19% of atoms are not recoverable from their cited *file* at all — syntheses inferred while reading the whole document, which the faithfulness check calls supported. Their span pointer is a fiction, and colouring a span for them is worse than saying nothing. @@ -460,6 +514,7 @@ Two limits on that rule, both measured: - **Per artifact it is unproven.** With 7–8 tasks per (artifact, model) cell the 95% interval is ~0.32 task-score points wide, so 44 of 60 individual cells were inconclusive. Pooled evidence licenses a shortening *policy*; it does not license a specific cut without running that cut. + - **The verdict does not transfer across model tiers.** Of 20 (artifact, candidate) pairs measured on three models each, only 9 agreed; the smallest model was the worst-hit in 9 of 20, and the same candidate was non-inferior on a frontier model and *worse* on a small one. Small models need the @@ -474,6 +529,7 @@ Two content patterns were measured as actively harmful, independent of topic: entirely: it replied with a preamble promising to look things up and scored 0.047, against 0.537 with no skill at all. Skill bodies should be answerable standalone; progressive disclosure is a loading strategy, not a sentence to write. + - **Investment in prose order.** The study's shuffled arm scored *higher* than the intact artifact on four of five models, but it injected more files than the artifact did, so that result does not carry over to this measurement. Ordering, transitions and narrative framing carried no @@ -522,6 +578,7 @@ the artifact's contribution runs the other way. The weakest model knew *no* loca - **Conceptual prose is the first thing to cut.** It is the half the model already has. Explaining what a cache miss is spends tokens; stating this stack's exact flag, threshold or ordering does not. + - **Write the procedure down, precisely.** `motif` and `contract` atoms — gates, orderings, flags, output formats — are where the artifact is least replaceable, and they are also the atoms a greedy token-budget shortener discards first (their value is conditional on a failure the sentence does @@ -564,6 +621,7 @@ Things to know before reading a bundle result: - **It is an upper bound.** Injecting the whole bundle measures what the artifact *could* deliver if the agent read all of it. Whether the agent chooses to load `references/x.md` at runtime is the agentic track's question (`skill-agent:`), not this one. + - **The two modes are different cells.** `include_resources` is recorded in `run_config`; an entry-point run and a bundle run of the same skill are not comparable numbers. A run that read half or less of the readable words says so on stdout (the `coverage:` line @@ -574,11 +632,13 @@ Things to know before reading a bundle result: `bundle.measured_share` (for an entry-point run, `1 - measurable_outside_share` to 4 places, which nets out the role decisions below; `outside_share` is the on-disk figure, and the two differ for a skill whose only resources are its evals). + - **A skill's own `evals/` are not content.** They are the question/answer keys for testing the skill, so extracting atoms from them would credit the skill with delivering its own test set. Excluded by default (10 of the 33 corpus skills ship them), with the reason naming the opt-in; `.git/`, `__pycache__/`, `venv/`, hidden files and friends are never walked at all. + - **Exclusions are reported, never silent.** A file over `--max-file-words` is skipped whole rather than truncated, because a cut reference would be attributed as if complete; every skip, role filter, budget overrun and non-text file appears in `bundle.excluded` @@ -588,6 +648,7 @@ Things to know before reading a bundle result: A file that is mostly printable text but does not decode as UTF-8 (latin-1, cp1252, UTF-16) is reported `unreadable` whatever its suffix, and the coverage check then says `measured=False`; only a NUL-bearing non-UTF-16 file, or a blob that is not mostly printable, is silently "not text". + - **One rule decides what an exclusion means.** Every row's `kind` has one class (`metrics/bundle_coverage.py`, `EXCLUSION_CLASS`): `budget` and `not_requested` are a *gap* — the words in `unmeasured_words`, the files a warning calls "excluded"; `policy` is a *decision*, never @@ -595,6 +656,7 @@ Things to know before reading a bundle result: exclusion is a *file* the run did not select; text it selected that no extraction call could read is an *unread span* (`spans.n_unread`, below), which narrows the dead-token figure and not coverage. + - **Coverage has one basis, and the record says which.** The CLI note, the analyzer's warnings, the gate's `artifact_coverage` and the report all read the record through the same `Coverage`, over *readable* words: role decisions are left out of the denominator, so for an @@ -742,6 +804,7 @@ Two numbers, reported side by side and never merged: - **Lexical** — BM25 over each candidate's `name + description`, which is exactly the text a tool schema shows. Free, deterministic, and a floor. + - **Model** (`--model`, optional) — a model picks from the same cards, with the card order a **seeded random permutation per query** (`--seed`), so a model that always picks the first option tends to `1/N` rather than to a figure set by how the query count lines up @@ -876,6 +939,7 @@ Three properties worth knowing before quoting either: says why, and still writes the result with `"status": "no_queries"`. Every rate line prints its `n_queries` (and `triage: self-derived` when it is self-derived), and fewer than five scored queries draw a warning. + - **The model figure is an upper bound on harness behaviour.** A harness whose own skill listing truncates or omits descriptions selects on less text than this command shows — which has been observed. Only a run inside the harness settles it. @@ -954,6 +1018,7 @@ run, so on a two-arm run too `ci95_added` changes (and is null below 5 atoms). - **Placebo-gated `added`** (`added_over_placebo`, needs the `placebo` arm). This counts a probe only when it is right with the artifact, wrong without it, *and* wrong with the placebo text. A gain the placebo also gets is a context effect, not knowledge. + - **Noise floor** (`noise_floor`, needs the `no_artifact_repeat` arm). That arm asks every baseline question again as a new sample (a new cache `draw`). Its flips against the baseline are what `added` and `degraded` would be if the artifact did nothing. @@ -1039,6 +1104,7 @@ The hybrid ships. Two consequences worth remembering when reading a `knowledge` - **Never accept an embedding-only retention score.** Both cheap matchers scored 0.00 recall on the contradiction label: they cannot tell *restates* from *reverses*, so a rewrite that inverted a rule would be reported as faithfully retained. + - **The oracle fails safe.** With no adjudicator configured, the ambiguous band resolves to *absent*, so reported `knowledge` is a floor — a shortening can look worse than it is, never better. @@ -1054,6 +1120,7 @@ reverses to +0.047. Two things not to conclude from that: adjudications, ≤0.0072 even at 0.80 agreement, and replacing per-pair verdicts with a calibrated aggregate collapses to a constant. The escalation *is* the oracle. Shortcut only at `hhem ≥ 0.995 AND top_sim ≥ 0.6 AND the atom carries no number or version`, and never shortcut the "no" end. + - **No cheap matcher may score a protected atom.** Agreement against the band mean falls −0.18 to −0.25 on numeric atoms, −0.07 to −0.10 on version-bound ones, −0.03 to −0.05 on negations, because a similarity score reads topic and not value: `_mm512_reduce_add_epi64` scores mid-scale against an atom @@ -1402,6 +1469,7 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. far from 1 (0.629 one way, 0.851 the other, over 1134 oracle comparisons, and 0.733/0.876 with a third extractor that never saw the reference set). **An atom count is extractor-conditional; an `added` rate is a ratio over whatever inventory the extractor produced and is far more robust.** Quote counts with the extractor named. + - **Tag labelling reproduces; the `type` vocabulary does not.** Holding extraction fixed and having two other models re-label the same 2640 atoms under the same glossary, six of nine tags reach κ 0.52–0.73 and the knowledge-type table above re-derives to within 0.08 under every labeller — so the @@ -1411,6 +1479,7 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. with both labellers moving `contract` and `edge` atoms into `motif`. Gate on the protection tags (`routing`, `negative-constraint`, `version-bound` — the top three by κ); report shape slices without gating on them. + - **Judge calibration — and a 0.15 band on `added` differences.** Signs and rankings, not second decimals. Quantified: the two graders' fine-scale κ is 0.576 (the 0.664 figure is raw agreement), and they differ by +7.7pp under strict grading and −11.6pp under lenient — opposite directions, which is the bias shape @@ -1418,22 +1487,27 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. [0.106, 0.194]**, so **no `added` difference below 0.15 may be quoted**, and for one model that band covers almost the entire between-artifact spread. Narrowing it needs more dual grading (only 25 probes have both arms dual-graded), not more probes. + - **Retention is not monotone in the text you supply.** The matcher retrieves its top chunks before it entails, so a longer text can score *lower* than a subset of itself: atom recovery from a cited span is 0.833, rises to 0.907 with the heading path, then falls to 0.809 when the whole file is supplied. Compare retention only between texts of similar length — which the shortening candidates are, by construction — and never read a retention figure across a large length gap. + - **The need graph has a 14% noise floor** (author/critic path-label agreement was 86%), which propagates into relevance-weighted `value`. + - **Need frequency is assumed uniform.** Nothing here measures how often each need actually arises, so `value` ranks atoms *within* one artifact and must not be compared across artifacts. Real query or issue traffic is what closes that, not another judge. + - **Category boundaries are partly judged.** In the study, where both axes were judged, an independent reviewer agreed with the five-category assignment on 72% of atoms; the disagreements concentrate on useful-vs-inert (±3 points) and known-vs-novel (±2 points). The scaffolding share is the most robust of them. Here useful-vs-inert is still judged and the ±3 applies; known-vs-novel is measured by the paired probes, so it carries the probe grading's error instead. + - **Per-artifact `added` is a profile, not a ranking.** Paired probes per artifact were 12–29 in the study, and subsampling shows the cross-artifact ranking destabilizing below a 0.75 probe budget (Spearman ρ 0.43–0.63 at 0.10). Correlating `added` against the measured task delta says the same @@ -1449,10 +1523,12 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. soft. Two validity checks come with it: `K0` and `degraded` both predict the delta negatively in every group, as designed. All of it under **strict** grading — lenient `added` correlates at 0.126 pooled and 0.000 within-artifact, so partial credit does not survive to task outcomes. + - **A topic-level verdict is a distribution, never a label.** No whole artifact in the study had unanimous atom outcomes under strict grading, on any model; at heading level unanimity was 0.39–0.44 and roughly one atom in four disagreed with its own section's majority. "This section is redundant" is not a statement this instrument can make. + - **Loading a correct artifact raises resistance to a reversed premise.** An earlier reading of the study (45 probes worse, 14 better) had the sign inverted by its analysis script; recomputed on the planted-inversion probes, paired by probe and model, 49 got worse and 367 better @@ -1462,6 +1538,7 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. the artifact, a planted wrong value is stated 0.942 of the time when plausible, 0.846 when absurd, and flagged 3% of the time. `degraded` is the visible tip of that; the channel underneath it transmits at 85–94%. + - **A cell can stop being re-measurable.** Two of the models behind most of these numbers no longer resolve, and one that does no longer accepts `temperature=0`. A novelty figure is quotable only with the served model named and the date attached; a cell whose model is gone is a dead cell, not a cell @@ -1471,4 +1548,5 @@ answer is a failed probe run (its pair is dropped), not a wrong answer. - [skill-novelty-evidence.md](skill-novelty-evidence.md) — the 33-skill study these numbers come from, with its method and its caveats. + - [evaluating-treatments.md](evaluating-treatments.md) — the arm framework this layer consumes. diff --git a/dokimi/dokimi/cli_flags.py b/dokimi/dokimi/cli_flags.py index ab3985679..7edb0d3ef 100644 --- a/dokimi/dokimi/cli_flags.py +++ b/dokimi/dokimi/cli_flags.py @@ -85,6 +85,8 @@ class Concept: canonical=( Flag("--judge", "PROVIDER/MODEL", _MODEL_REF, repeatable=True, meaning="the judge; repeated, it is a panel"), + Flag("--judge-aggregate", "{median,majority,unanimous}", "choice", + meaning="how a judge panel's verdicts become one"), Flag("--extract-model", "PROVIDER/MODEL", _MODEL_REF), Flag("--escalate-model", "PROVIDER/MODEL", _MODEL_REF), Flag("--fill-model", "PROVIDER/MODEL", _MODEL_REF), diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 9a9a326d7..e71a9f44e 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -18,8 +18,9 @@ from typing import TYPE_CHECKING, NoReturn from dokimi.commands.inputs import placebo_text_for_arms_or_exit -from dokimi.commands.novelty_run import (add_judge2_args, add_spend_cap, judge2_client, - exit_if_analyze_partial, stamp) +from dokimi.commands.novelty_run import (add_judge2_args, add_panel_args, add_spend_cap, + attach_panel, exit_if_analyze_partial, + judge2_client, panel_clients, save_panel_verdicts, stamp) from dokimi.commands.novelty_routing_inputs import ( checked_out_path, load_all_vs_all_field, @@ -113,7 +114,8 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: placebo_text = placebo_text_for_arms_or_exit(args.placebo_text, _arms(args.arms)) extractor, answerer, judge = cost.clients(args) - judge2 = judge2_client(args, answerer, judge, cost.client) + panel = panel_clients(args, cost.client) + judge2 = judge2_client(args, answerer, judge, cost.client, panel) config = AnalyzeConfig( artifact_ref=args.artifact, artifact_kind=artifact_bundle.kind, @@ -134,7 +136,8 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: try: result = analyze(artifact_bundle.text, config, extractor=extractor, answerer=answerer, judge=judge, - artifact_bundle=artifact_bundle) + artifact_bundle=artifact_bundle, + panel=panel, aggregate=args.judge_aggregate) except SelfJudgeError as exc: # the judge is the answering model: the flags contradict raise InputError(str(exc)) from exc except (ExtractionError, ValueError) as exc: # a model failure under way @@ -144,6 +147,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: exit_if_analyze_partial(result, out_json) # a run the cap stopped keeps what it paid for stamp(result.artifact, args, result.measure_result, answerer=answerer, judge=judge, judge2=judge2) save_artifact("novelty", result.artifact, out_json) + verdicts_path = save_panel_verdicts(result, out_json) out_md = out_json.with_suffix(".md") out_md.write_text(render_novelty_markdown(result.artifact), encoding="utf-8") title = f"{artifact_bundle.name or args.artifact} × {answerer.model}" @@ -155,6 +159,8 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: print(f"OK Saved novelty artifact: {out_json}") print(f"OK Saved novelty report: {out_md}") print(f"OK Saved annotated view: {view_path}") + if verdicts_path: + print(f"OK Saved judge verdicts: {verdicts_path}") _print_headline(result.artifact) @@ -216,10 +222,12 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None: settings = _settings(args) placebo_text = placebo_text_for_arms_or_exit(args.placebo_text, _arms(args.arms)) extractor, answerer, judge = cost.clients(args) + panel = panel_clients(args, cost.client) warn_answer_key(saved, answerer, judge) try: # before the paraphrase calls below, which are paid - check_self_judge(judge, answerer, args.allow_self_judge) - judge2 = judge2_client(args, answerer, judge, cost.client) + for member in (judge, *panel): + check_self_judge(member, answerer, args.allow_self_judge) + judge2 = judge2_client(args, answerer, judge, cost.client, panel) except SelfJudgeError as exc: raise InputError(str(exc)) from exc paraphrases, reversals = settings.values["paraphrases"], settings.values["reversals"] @@ -239,6 +247,7 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None: arms=_arms(args.arms), concurrency=args.concurrency, allow_self_judge=args.allow_self_judge, seed=args.seed, placebo_text=placebo_text, framing=args.context_framing, + panel=panel, aggregate=args.judge_aggregate, ) except ValueError as exc: # a model failure under way raise SystemExit(str(exc)) from None @@ -249,6 +258,7 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None: probe_set=probe_set, measure_result=result, sampling=sampling_record(args, atoms, saved), framing=args.context_framing, ) + attach_panel(payload, result, args.judge_aggregate, answerer.model) stamp(payload, args, result, answerer=answerer, judge=judge, judge2=judge2) out = Path(args.out) if args.out else _default_dir(args.artifact) / "probes.json" # A run the cap stopped keeps the answers it paid for, as a partial result (and exits). @@ -634,6 +644,7 @@ def _add_model_args(parser: argparse.ArgumentParser, positive_int) -> None: help="Provider for the judge. Without --judge-model, the default judge " "that runs on this provider; if none does, the run exits asking " "for --judge-model") + add_panel_args(parser) parser.add_argument("--extract-model", default=None, dest="extract_model", help="Extractor/adjudicator model (defaults to --judge-model, so the " "answering model never writes its own answer key)") diff --git a/dokimi/dokimi/commands/novelty_budget.py b/dokimi/dokimi/commands/novelty_budget.py index cc70622ab..734b20952 100644 --- a/dokimi/dokimi/commands/novelty_budget.py +++ b/dokimi/dokimi/commands/novelty_budget.py @@ -181,6 +181,7 @@ def run_estimate(args: argparse.Namespace, settings, text: str, estimate_usd) -> ``estimate_usd`` is the command's pricing (``dokimi.novelty.cost.estimate_usd``), looked up by the caller at call time, so a test that patches ``cost.estimate_usd`` takes effect. """ + from dokimi.novelty import cost from dokimi.novelty.measure import parse_arms from dokimi.novelty.spans import segment from dokimi.utils import InputError @@ -199,6 +200,7 @@ def run_estimate(args: argparse.Namespace, settings, text: str, estimate_usd) -> args, n_tokens, estimate["estimate"]["by_role"], settings.tier, context_share=(n_arms - 1 - ("no_artifact_repeat" in args.arms)) / n_arms, ) + cost.add_panel_calls(args, estimate["estimate"]) # after the price, which counts each judge itself return estimate, n_tokens diff --git a/dokimi/dokimi/commands/novelty_run.py b/dokimi/dokimi/commands/novelty_run.py index 2d2bd1b56..d8284aa46 100644 --- a/dokimi/dokimi/commands/novelty_run.py +++ b/dokimi/dokimi/commands/novelty_run.py @@ -11,6 +11,7 @@ import functools import math import sys +from pathlib import Path def stops_at_cap(command): @@ -69,14 +70,14 @@ def add_judge2_args(parser: argparse.ArgumentParser, providers) -> None: help="Provider for --judge2 (default: the judge's provider)") -def judge2_client(args: argparse.Namespace, answerer, judge, make_client): - """The ``--judge2`` client, checked against the answerer before any paid call; ``None`` without it.""" +def judge2_client(args: argparse.Namespace, answerer, judge, make_client, panel=()): + """The ``--judge2`` client, checked against the answerer and every panel judge before any paid call; ``None`` without it.""" from dokimi.novelty.judge2 import check_judge2 if not getattr(args, "judge2", None): return None client = make_client(args.judge2, args.judge2_provider or judge.provider, "judge2", args) - check_judge2(client, answerer, args.judge2_frac, args.allow_self_judge, judge) + check_judge2(client, answerer, args.judge2_frac, args.allow_self_judge, judge, panel) return client @@ -105,3 +106,58 @@ def stamp(payload: dict, args: argparse.Namespace, measure_result, *, answerer, payload["judge2"] = {"model": judge2.model, "stopped": str(exc)} print(f"WARNING second judge stopped: {exc}", file=sys.stderr) payload["judge2"]["cost"] = judge2.cost_summary() + + +def add_panel_args(parser: argparse.ArgumentParser) -> None: + """``--judge`` (repeated, it is a panel) and ``--judge-aggregate``.""" + parser.add_argument("--judge", action="append", default=None, dest="judges", + metavar="PROVIDER/MODEL", + help="A judge, as PROVIDER/MODEL; repeated, it is a panel: every judge " + "grades every answer and --judge-aggregate decides. Pick judges " + "from different vendors (docs/judge-panel-study.md: three from " + "three families cut the judge spread by a third). Replaces " + "--judge-model/--judge-provider; the first judge is the default " + "extractor") + parser.add_argument("--judge-aggregate", default="median", dest="judge_aggregate", + choices=("median", "majority", "unanimous"), + help="How a panel's verdicts become one: median of the verdict scale " + "(default), lenient majority, or unanimous (every judge must agree; " + "not conservative, it can raise the added share)") + + +def panel_clients(args: argparse.Namespace, make_client) -> tuple: + """Clients for the judges after the first ``--judge`` (the first is the judge client).""" + from dokimi.novelty.cost import judge_refs + from dokimi.utils import InputError + + try: + refs = judge_refs(args) + except ValueError as exc: + raise InputError(str(exc)) from exc + return tuple(make_client(model, provider, "judge", args) for model, provider in refs[1:]) + + +def attach_panel(payload: dict, result, aggregate: str, answerer_model: str | None = None) -> None: + """Add the ``judge_panel`` block to a written record and print its warnings.""" + from dokimi.novelty.panel_record import panel_record + from dokimi.novelty.panel import panel_warnings + + judge_panel = panel_record(result, aggregate, answerer_model) + if judge_panel: + payload["judge_panel"] = judge_panel + for note in panel_warnings(judge_panel): + print(f"WARNING: {note}", file=sys.stderr) + + +def save_panel_verdicts(result, out: Path) -> Path | None: + """Write each judge's verdict on every answer beside ``out``, so a panel run can be audited or + re-aggregated without grading again; ``None`` for a single-judge run.""" + import json + + runs = [run.as_dict() for run in result.measure_result.runs if run.judge_verdicts] + dest = out.with_name(f"{out.stem}.judge_verdicts.json") + if not runs: # a single-judge run into the same output: the panel's file is not this run's + dest.unlink(missing_ok=True) + return None + dest.write_text(json.dumps(runs, indent=2), encoding="utf-8") + return dest diff --git a/dokimi/dokimi/novelty/analyze.py b/dokimi/dokimi/novelty/analyze.py index 3b08bd3c4..c3ea3fd29 100644 --- a/dokimi/dokimi/novelty/analyze.py +++ b/dokimi/dokimi/novelty/analyze.py @@ -24,9 +24,11 @@ from . import attribute, bundle as bundle_mod from . import classify, items as items_mod from . import measure, needs, probes, provenance, report, retention, shorten, spans +from . import panel as panel_mod +from . import panel_record as panel_record_mod from .artifact import load_artifact_text # noqa: F401 (public path: novelty.analyze) from .analyze_partial import Progress, stopped_record -from .client import LLMClient, SpendCapReached, roll_up_cost, same_client, warn_extractor_judge +from .client import LLMClient, SpendCapReached, roll_up_cost, warn_extractor_judge from .retention import RetentionOracle from .tiers import DEFAULT_SETTINGS, RunSettings @@ -176,6 +178,8 @@ def analyze( ref_digest: str | None = None, artifact_bundle: "bundle_mod.Bundle | None" = None, tier: dict | None = None, + panel: tuple[LLMClient, ...] = (), + aggregate: str = "median", ) -> AnalyzeResult: """Run the full pipeline over one artifact's text. @@ -203,19 +207,21 @@ def analyze( per-file provenance: every span, atom and outcome can then be traced to the file it came from, which is the cut an author needs to decide what belongs in the body. Without it the run is entry-point-only and says so. + panel: More judges besides ``judge`` (see :func:`measure.run_measurement`). + aggregate: The panel's rule for one verdict per answer. """ progress = Progress() try: return _analyze(text, config, extractor=extractor, answerer=answerer, judge=judge, ref_digest=ref_digest, artifact_bundle=artifact_bundle, tier=tier, - progress=progress) + progress=progress, panel=panel, aggregate=aggregate) except SpendCapReached as stop: if ref_digest is None and artifact_bundle is not None: ref_digest = artifact_bundle.measured_digest() record = stopped_record( progress, stop, artifact_kind=config.artifact_kind, artifact_ref=config.artifact_ref, ref_digest=ref_digest, model=answerer.model, provider=answerer.provider, - judge_model=judge.model, clients=[extractor, answerer, judge], + judge_model=judge.model, clients=[extractor, answerer, judge, *panel], provenance=provenance.provenance( artifact_digest=ref_digest, answerer_model=answerer.model, judge_model=judge.model, context_framing=config.context_framing), @@ -237,10 +243,12 @@ def _analyze( artifact_bundle: "bundle_mod.Bundle | None", tier: dict | None, progress: Progress, + panel: tuple[LLMClient, ...], + aggregate: str, ) -> AnalyzeResult: """The pipeline of :func:`analyze`; ``progress`` records each stage as it starts and what it produced, so a spend stop anywhere can be saved.""" - measure.check_self_judge(judge, answerer, config.allow_self_judge) + measure.check_judges(judge, panel, answerer, config.allow_self_judge, aggregate) relation = warn_extractor_judge(extractor.model, judge, extractor.provider) warnings: list[str] = progress.warnings span_list = progress.spans = spans.segment(text) @@ -327,10 +335,13 @@ def _analyze( answerer=answerer, judge=judge, arms=config.arms, concurrency=config.concurrency, allow_self_judge=config.allow_self_judge, placebo_text=config.placebo_text, seed=config.seed, framing=config.context_framing, + panel=panel, aggregate=aggregate, ) _refuse_cut_short(measure_result) + judge_panel = panel_record_mod.panel_record(measure_result, aggregate, answerer.model) + warnings.extend(panel_mod.panel_warnings(judge_panel)) progress.stage = "classify" - if config.allow_self_judge and same_client(judge, answerer): + if config.allow_self_judge and panel_mod.same_as_answerer((judge, *panel), answerer): warnings.append( "judge and answerer are the same model (--allow-self-judge): scores are inflated" ) @@ -471,7 +482,7 @@ def _analyze( conflicts=conflicts, conflict_coverage=conflict_coverage, extraction=extraction.as_dict(), - cost=roll_up_cost([extractor, answerer, judge]), + cost=roll_up_cost([extractor, answerer, judge, *panel]), warnings=warnings, ) payload["run_config"] = _run_config(config) @@ -482,6 +493,7 @@ def _analyze( provenance.same_vendor_warning(payload["provenance"])])}) payload["run_config"]["tier"] = settings.tier_record(sampled=capped) payload["run_config"]["extractor_judge"] = relation + payload.update(panel_mod.panel_block(judge_panel)) return AnalyzeResult( artifact=payload, attributions=attributions, diff --git a/dokimi/dokimi/novelty/client.py b/dokimi/dokimi/novelty/client.py index 0aa65c2d6..1f9be8ffc 100644 --- a/dokimi/dokimi/novelty/client.py +++ b/dokimi/dokimi/novelty/client.py @@ -71,6 +71,8 @@ class SpendCapReached(BaseException): reply: str | None = None truncated: bool = False + #: With a judge panel: the verdicts of the judges graded before this stop, by model. + panel_votes: dict[str, str] | None = None def __init__(self, message: str, *, reason: str = "spend_cap") -> None: super().__init__(message) @@ -335,6 +337,7 @@ def same_client(a: LLMClient, b: LLMClient) -> bool: ("claude", "anthropic"), ("anthropic", "anthropic"), ("sonnet", "anthropic"), ("opus", "anthropic"), ("haiku", "anthropic"), ("gpt", "openai"), ("openai", "openai"), ("gemini", "google"), ("mistral", "mistral"), + ("grok", "xai"), ("xai", "xai"), ) diff --git a/dokimi/dokimi/novelty/cost.py b/dokimi/dokimi/novelty/cost.py index f0b7f300e..12260e36a 100644 --- a/dokimi/dokimi/novelty/cost.py +++ b/dokimi/dokimi/novelty/cost.py @@ -16,11 +16,50 @@ from dokimi.utils import InputError +def judge_refs(args: argparse.Namespace) -> list[tuple[str, str]]: + """The ``--judge PROVIDER/MODEL`` list as (model, provider) pairs; empty without the flag.""" + from dokimi.llm import ANSWER_PROVIDERS as PROVIDERS + from dokimi.utils import same_model + + refs = getattr(args, "judges", None) or [] + if refs and (getattr(args, "judge_model", None) or getattr(args, "judge_provider", None)): + raise ValueError("--judge replaces --judge-model and --judge-provider; pass one form") + out: list[tuple[str, str]] = [] + for ref in refs: + provider, sep, model = ref.partition("/") + if not sep or not model or provider not in PROVIDERS: + raise ValueError(f"--judge {ref!r} is not PROVIDER/MODEL with PROVIDER one of " + f"{', '.join(PROVIDERS)}") + if any(same_model(m, model) for m, _ in out): + # one model would vote twice, and the panel keys each judge's verdicts by model name + raise ValueError(f"--judge {ref!r}: model {model!r} is given twice") + out.append((model, provider)) + return out + + +def add_panel_usd(args: argparse.Namespace, usd, by_role, n_tokens, tier, context_share, models): + """The USD estimate with each panel judge after the first grading every answer again.""" + from dokimi.novelty.tiers import estimate_usd + + for model, provider in judge_refs(args)[1:]: + if usd is None: + break + extra = estimate_usd({"judge": by_role.get("judge", 0)}, n_tokens, + {"judge": (model, provider)}, tier, context_share) + if extra is None: + models[f"judge {model}"] = (model, provider) + usd = None if extra is None else round(usd + extra, 6) + return usd + + def role_models(args: argparse.Namespace) -> dict[str, tuple[str, str]]: """The (model, provider) of each role, defaulting so roles never silently collapse.""" from dokimi.novelty.client import default_judge - if args.judge_model: + panel = judge_refs(args) + if panel: + judge_model, judge_provider = panel[0] + elif args.judge_model: judge_model, judge_provider = args.judge_model, args.judge_provider or args.provider else: # a CLI answerer with no judge provider judges on the same CLI, another family @@ -38,20 +77,29 @@ def estimate_usd( args: argparse.Namespace, n_tokens: int, by_role: dict[str, int], tier: str | None, context_share: float = 0.5, ) -> float | None: - """The estimate's dollar figure; ``None`` when a role's model cannot be resolved or priced. + """The estimate's dollar figure; ``None`` when a role's model cannot be priced. The reason goes to stderr: a role whose model does not resolve, or a role with no list - price, makes the figure unavailable, and the note names it. + price, makes the figure unavailable, and the note names it. Each panel judge after the + first is priced as grading every answer again. + + Raises: + InputError: A ``--judge`` that is not a usable PROVIDER/MODEL, as the run itself refuses it. """ from dokimi.llm_pricing import list_cost_usd from dokimi.novelty.tiers import estimate_usd as tier_estimate_usd + try: + judge_refs(args) + except ValueError as exc: + raise InputError(str(exc)) from exc try: models = role_models(args) except ValueError as exc: print(f"The USD estimate is unavailable: {exc}", file=sys.stderr) return None - usd = tier_estimate_usd(by_role, n_tokens, models, tier, context_share) + usd = add_panel_usd(args, tier_estimate_usd(by_role, n_tokens, models, tier, context_share), + by_role, n_tokens, tier, context_share, models) if usd is None: unpriced = sorted( f"{role} ({model})" @@ -66,6 +114,22 @@ def estimate_usd( return usd +def add_panel_calls(args: argparse.Namespace, estimate: dict) -> None: + """Count the panel's grading calls in a run estimate: each judge after the first grades every answer again.""" + + try: + extra_judges = max(0, len(judge_refs(args)) - 1) + except ValueError as exc: + raise InputError(str(exc)) from exc + extra = extra_judges * estimate["by_role"]["judge"] + if not extra: + return + estimate["by_role"]["judge"] += extra + for key in ("probe_calls", "total_calls", "total_calls_if_every_pair_escalates", + "expected_calls"): + estimate[key] += extra + + def clients(args: argparse.Namespace): """Build the three role clients, defaulting so roles never silently collapse.""" try: diff --git a/dokimi/dokimi/novelty/judge2.py b/dokimi/dokimi/novelty/judge2.py index 418127e94..c40c2b4bb 100644 --- a/dokimi/dokimi/novelty/judge2.py +++ b/dokimi/dokimi/novelty/judge2.py @@ -22,8 +22,8 @@ def check_judge2(judge2: LLMClient, answerer: LLMClient, frac: float, allow_self_judge: bool, - judge: LLMClient | None = None) -> None: - """Refuse a second judge that is the answering model or the first judge, or a fraction outside (0, 1]. + judge: LLMClient | None = None, panel: tuple[LLMClient, ...] = ()) -> None: + """Refuse a second judge that is the answering model, the first judge or a panel judge, or a fraction outside (0, 1]. The model test is the repo's model-identity rule (``same_client``, which reads ``same_model``): an alias, an equivalent API id and a Bedrock id of one model are one model. @@ -43,11 +43,12 @@ def check_judge2(judge2: LLMClient, answerer: LLMClient, frac: float, allow_self f"--judge2 {judge2.model!r} is the answering model; grading a response with the " "model that produced it inflates the score. Pass another --judge2, or " "--allow-self-judge to do it deliberately.") - if judge is not None and same_client(judge2, judge): - raise InputError( - f"--judge2 {judge2.model!r} is the same model as the first judge {judge.model!r}; " - "its agreement with that judge is not an independent second opinion. Pass another " - "--judge2, or --allow-self-judge to do it deliberately.") + for role, member in (("first judge", judge), *(("panel judge", m) for m in panel)): + if member is not None and same_client(judge2, member): + raise InputError( + f"--judge2 {judge2.model!r} is the same model as the {role} {member.model!r}; " + "its agreement with that judge is not an independent second opinion. Pass another " + "--judge2, or --allow-self-judge to do it deliberately.") def cohen_kappa(a: list[bool], b: list[bool]) -> float | None: diff --git a/dokimi/dokimi/novelty/measure.py b/dokimi/dokimi/novelty/measure.py index 7dbf5ef16..c3b377d4b 100644 --- a/dokimi/dokimi/novelty/measure.py +++ b/dokimi/dokimi/novelty/measure.py @@ -32,6 +32,7 @@ from .client import LLMClient, SpendCapReached, parse_json_object, same_client from .controls import (REPEAT_ARM, clustered_ci, control_fields, reversal_by_arm, reversal_paired) +from .panel import RULES, aggregate as aggregate_votes from .probes import Probe # Judge verdict scale. Deliberately finer than the collapses below so a rater @@ -132,6 +133,7 @@ class ProbeRun: response: str verdict: str confidence: float | None = None # recorded, never scored + judge_verdicts: dict[str, str] | None = None # panel only: each judge's verdict by model #: Set only when the probe carries ``grading``: the judge's own verdict, which ``verdict`` #: may overrule, where the verdict came from (``judge`` or ``check``) and the check results. judge_verdict: str | None = None @@ -158,6 +160,7 @@ def as_dict(self) -> dict: "response": self.response, "verdict": self.verdict, "confidence": self.confidence, + **({"judge_verdicts": self.judge_verdicts} if self.judge_verdicts else {}), **({"judge_verdict": self.judge_verdict, "verdict_source": self.verdict_source, "checks": self.checks} if self.checks else {}), } @@ -295,6 +298,18 @@ def check_arms(arms: tuple[str, ...], placebo_text: str | None) -> None: raise InputError("the placebo arm needs --placebo-text (domain-mismatched text)") +def check_judges(judge: LLMClient, panel, answerer: LLMClient, allow_self_judge: bool, + aggregate: str = "median") -> None: + """Refuse an unknown rule, a judge that is the answerer, or one model twice (it would vote twice).""" + if aggregate not in RULES: + raise InputError(f"unknown judge aggregation rule {aggregate!r}; known: {list(RULES)}") + members = (judge, *panel) + for i, member in enumerate(members): + check_self_judge(member, answerer, allow_self_judge) + if any(same_client(member, other) for other in members[:i]): + raise InputError(f"judge {member.model!r} is the same model as another judge") + + def _unpriced_answer_run(probe: Probe, arm: str, exc: SpendCapReached, stops: list[SpendCapReached], ungraded: list[ProbeRun]) -> ProbeRun: """The paid answer of a call with no price, kept ungraded (no datum) as the stop's run; a reply @@ -331,6 +346,8 @@ def run_measurement( placebo_text: str | None = None, seed: int = 0, framing: str = "reference", + panel: tuple[LLMClient, ...] = (), + aggregate: str = "median", ) -> MeasureResult: """Answer every probe in every arm and grade the answers. @@ -349,8 +366,11 @@ def run_measurement( irrelevant context actively harms them. seed: Seed for the ``shuffled`` control, so the control is reproducible. framing: One of :data:`FRAMINGS`; how the context is introduced in the prompt. + panel: More judges besides ``judge``, each grading every answer. + aggregate: The rule that makes one verdict of the panel's (:func:`~.panel.aggregate`). """ - check_self_judge(judge, answerer, allow_self_judge) + judges = (judge, *panel) + check_judges(judge, panel, answerer, allow_self_judge, aggregate) check_arms(arms, placebo_text) if framing not in FRAMINGS: raise InputError(f"unknown context framing {framing!r}; known: {list(FRAMINGS)}") @@ -369,6 +389,7 @@ def run_measurement( def one(job: tuple[Probe, str]) -> ProbeRun: probe, arm = job + by_judge: dict[str, str] | None = None answered = True try: response, confidence = _answer( @@ -383,13 +404,13 @@ def one(job: tuple[Probe, str]) -> ProbeRun: answered = False else: try: - verdict = _grade(probe, response, judge) + verdict, by_judge = _grade_panel(probe, response, judges, aggregate) except SpendCapReached as exc: return _capped_grade_run(probe, arm, response, confidence, exc, stops, ungraded) run = ProbeRun( atom_id=probe.atom_id, probe_kind=probe.kind, arm=arm, question=probe.question, response=response, verdict=verdict, - confidence=confidence, + confidence=confidence, judge_verdicts=by_judge, ) if probe.grading and answered: _apply_checks(run, probe, response, verdict) @@ -411,18 +432,35 @@ def guarded(job: tuple[Probe, str]) -> ProbeRun | None: probes=list(probes)) +def _grade_panel(probe: Probe, response: str, judges: tuple[LLMClient, ...], + aggregate: str) -> tuple[str, dict[str, str] | None]: + """One answer's verdict and, with a panel, each judge's verdict by model; a failed judge is no vote.""" + if len(judges) == 1: + return _grade(probe, response, judges[0]), None + by_judge: dict[str, str] = {} + try: + for m in judges: + by_judge[m.model] = _grade(probe, response, m) + except SpendCapReached as exc: + exc.panel_votes = by_judge + raise + votes = [v for v in by_judge.values() if v != ERRORED] + return aggregate_votes(votes, aggregate) or ERRORED, by_judge + + def _capped_grade_run(probe: Probe, arm: str, response: str, confidence: float | None, exc: SpendCapReached, stops: list[SpendCapReached], ungraded: list[ProbeRun]) -> ProbeRun: """The answer was paid for and the judge hit the cap: keep the verdict a refused judge reply - names, else keep the run ungraded (no datum). The checks grade the run either way, and the + names (not a truncated one, not one judge's of a panel), else keep the run ungraded. The checks grade the run either way, and the measurement stops here.""" stops.append(exc) - named = _reply_verdict(exc.reply) + panel_votes = getattr(exc, "panel_votes", None) + named = None if exc.truncated or panel_votes is not None else _reply_verdict(exc.reply) run = ProbeRun( atom_id=probe.atom_id, probe_kind=probe.kind, arm=arm, question=probe.question, response=response, verdict=named or ERRORED, - confidence=confidence, + confidence=confidence, judge_verdicts=panel_votes or None, ) if probe.grading: # the veto runs on the answer whatever the refused judge reply said _apply_checks(run, probe, response, named or ERRORED) @@ -760,4 +798,3 @@ def _pearson(xs: list[float], ys: list[float]) -> float | None: if den == 0: return None return round(sum(a * b for a, b in zip(dx, dy, strict=True)) / den, 4) - diff --git a/dokimi/dokimi/novelty/panel.py b/dokimi/dokimi/novelty/panel.py new file mode 100644 index 000000000..371998845 --- /dev/null +++ b/dokimi/dokimi/novelty/panel.py @@ -0,0 +1,117 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A judge panel for the paired probes: several judges grade each answer, one rule decides. + +The judge-panel study (docs/judge-panel-study.md) re-graded the skill-novelty sample with +seven judges from seven families. The paired added share of one answer model moved by up to +0.23 with the judge, while the order of the extreme models did not move. Majority, median +and Dawid-Skene agreed on 99.3% of rows; unanimous and any-correct moved the headline by up +to 0.20. Three judges from three families cut the spread across panels from SD 0.039 to +0.025. Hence the defaults here: the median rule, a warning when two judges share a vendor, +and a per-judge 2×2 next to the panel's, so a judge-sensitive headline is visible. +""" + +from __future__ import annotations + +from collections import Counter + +from .client import same_client, vendor_of + +# The order the median walks: best first. The three wrong verdicts score 0 and tie. +_SCORE = { + "correct_complete": 1.0, + "correct_missing_qualifier": 0.7, + "partially_correct": 0.4, + "wrong_other": 0.0, + "wrong_reversed": 0.0, + "abstained": 0.0, +} +LENIENT = frozenset({"correct_complete", "correct_missing_qualifier", "partially_correct"}) +RULES = ("median", "majority", "unanimous") +# A judge range of the lenient added share above this marks the headline judge-sensitive. +# The study's noise floor (a fresh no-skill draw against the first) was 0.00-0.10. +SENSITIVE_RANGE = 0.10 + + +def _most_common(votes: list[str]) -> str: + """The most common verdict; a tie goes to the lower score, then to the scale's order.""" + counts = Counter(votes) + order = list(_SCORE) + return min(counts, key=lambda v: (-counts[v], _SCORE[v], order.index(v))) + + +def aggregate(votes: list[str], rule: str) -> str | None: + """One verdict from a panel's verdicts, or ``None`` when no judge returned one. + + ``votes`` holds only verdicts on the scale (failed grades are left out by the caller). + + * ``median``: the lower median of the verdict scores (an even panel rounds toward + wrong); among the 0-score verdicts the most common one, so ``wrong_reversed`` survives. + * ``majority``: correct (lenient) when more than half the judges say so, and then the + median of the correct votes; otherwise the most common of the other votes. + * ``unanimous``: correct only when every judge says so, then the lowest correct vote. + """ + if rule not in RULES: + raise ValueError(f"unknown judge aggregation rule {rule!r}; known: {list(RULES)}") + if not votes: + return None + good = [v for v in votes if v in LENIENT] + bad = [v for v in votes if v not in LENIENT] + if rule == "median": + ranked = sorted(votes, key=lambda v: -_SCORE[v]) + pick = ranked[len(ranked) // 2] + if _SCORE[pick] == 0.0: + return _most_common([v for v in votes if _SCORE[v] == 0.0]) + return pick + if rule == "majority": + if len(good) > len(votes) / 2: + return aggregate(good, "median") + return _most_common(bad) if bad else _most_common(good) + # unanimous + if not bad: + return min(good, key=lambda v: _SCORE[v]) + return _most_common(bad) + + +def shared_vendors(models: list[str]) -> dict[str, list[str]]: + """Vendor -> the panel's judges from it, for every vendor with more than one judge.""" + by_vendor: dict[str, list[str]] = {} + for model in models: + vendor = vendor_of(model) + if vendor: + by_vendor.setdefault(vendor, []).append(model) + return {v: ms for v, ms in by_vendor.items() if len(ms) > 1} + + +def panel_warnings(record: dict | None) -> list[str]: + """Warnings a panel record carries into the artifact's ``warnings``.""" + if not record: + return [] + out = [] + for vendor, models in sorted(record["shared_vendors"].items()): + out.append(f"judge panel has {len(models)} judges from one vendor ({vendor}: " + f"{', '.join(models)}); judges of one family agree with each other more " + "than with the rest, so the panel counts them nearly once") + for model in record.get("same_vendor_as_answerer", []): + out.append(f"judge {model} is of the answerer's vendor: a judge can favour answers of its " + "own vendor; pass a --judge of another vendor") + for model, failed in sorted(record.get("failed_grades", {}).items()): + if failed: + out.append(f"judge {model} failed to grade {failed} answer(s); its vote is left out " + "of those verdicts") + if record["judge_sensitive"]: + out.append(f"judge-sensitive: the lenient added share varies by " + f"{record['added_share_range']['lenient']:.2f} across the panel's judges " + f"(more than {SENSITIVE_RANGE:.2f}); report the range, not one judge's number") + return out + + +def same_as_answerer(judges, answerer) -> bool: + """Whether any judge is the answerer's model (the self-judge warning).""" + return any(same_client(j, answerer) for j in judges) + + +def panel_block(judge_panel: dict | None) -> dict: + """The ``judge_panel`` key of a record: empty for a single-judge run.""" + return {"judge_panel": judge_panel} if judge_panel else {} diff --git a/dokimi/dokimi/novelty/panel_record.py b/dokimi/dokimi/novelty/panel_record.py new file mode 100644 index 000000000..eb9169fac --- /dev/null +++ b/dokimi/dokimi/novelty/panel_record.py @@ -0,0 +1,96 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The ``judge_panel`` block of a probe or novelty record, built from a panel's measured runs.""" + +from __future__ import annotations + +from .measure import BASELINE_ARM, ERRORED, ProbeRun, paired_outcomes, resolve_verdict +from .client import vendor_of +from .panel import SENSITIVE_RANGE, shared_vendors + +SCORINGS = ("lenient", "strict") + + +def _own_verdict(run: ProbeRun, model: str) -> str: + """The verdict ``model`` gave ``run``, with the run's program checks applied as the headline + applies them; a judge that cast no vote (a spend stop, a failed call) is a missing datum.""" + raw = (run.judge_verdicts or {}).get(model) + if raw is None: + return ERRORED + return resolve_verdict(raw, run.checks)[0] if run.checks else raw + + +def _own_runs(runs: list[ProbeRun], model: str) -> list[ProbeRun]: + """``runs`` with each verdict replaced by the verdict ``model`` gave it.""" + return [ + ProbeRun(atom_id=r.atom_id, probe_kind=r.probe_kind, arm=r.arm, question=r.question, + response=r.response, verdict=_own_verdict(r, model)) + for r in runs + ] + + +def _graded_pairs(own: list[ProbeRun], with_arm: str) -> set[tuple[str, str, str]]: + """The probes one judge graded in both the baseline and ``with_arm``.""" + def keys(arm: str) -> set[tuple[str, str, str]]: + return {(r.atom_id, r.probe_kind, r.question) for r in own + if r.arm == arm and r.probe_kind != "reversal" and not r.errored} + return keys(BASELINE_ARM) & keys(with_arm) + + +def _added_range(per_judge_runs: dict[str, list[ProbeRun]], with_arm: str, + scoring: str) -> float | None: + """The spread of the added share across judges on the pairs every judge graded. + + A judge whose grade failed on one arm of a pair drops that pair from its own 2×2; the + range is taken on the pairs graded by all, so a missing grade is not read as a judge + that disagrees. ``None`` when no pair is graded by every judge. + """ + shared = set.intersection(*(_graded_pairs(own, with_arm) for own in per_judge_runs.values())) + if not shared: + return None + shares = [] + for own in per_judge_runs.values(): + kept = [r for r in own + if r.probe_kind == "reversal" or (r.atom_id, r.probe_kind, r.question) in shared] + shares.append(paired_outcomes(kept, with_arm=with_arm, scoring=scoring)["added"]) + return round(max(shares) - min(shares), 4) + + +def panel_record(measure_result, rule: str, answerer_model: str | None = None) -> dict | None: + """The ``judge_panel`` block of a probe or novelty record: who judged, the rule, each + judge's own paired 2×2, the judge range of the added share, and a sensitivity flag. + + The range is over the pairs every judge graded in both arms; each judge's own 2×2 keeps + its own sample (``n_unpaired_dropped`` says what it lost). ``None`` for a single-judge + run, which records no panel. + """ + runs = measure_result.runs + judges: list[str] = [] + for run in runs: + for model in run.judge_verdicts or {}: + if model not in judges: + judges.append(model) + if len(judges) < 2: + return None + own_runs = {model: _own_runs(runs, model) for model in judges} + with_arm = measure_result.with_arm + per_judge = { + model: {s: paired_outcomes(own, with_arm=with_arm, scoring=s) for s in SCORINGS} + for model, own in own_runs.items() + } + added_range = {s: _added_range(own_runs, with_arm, s) for s in SCORINGS} + lenient_range = added_range["lenient"] + return { + "judges": judges, + "rule": rule, + "shared_vendors": shared_vendors(judges), + "same_vendor_as_answerer": [m for m in judges if answerer_model and vendor_of(m) + and vendor_of(m) == vendor_of(answerer_model)], + "failed_grades": {m: sum((r.judge_verdicts or {}).get(m) == ERRORED for r in runs) + for m in judges}, + "per_judge_paired": per_judge, + "added_share_range": added_range, + "judge_sensitive": lenient_range is not None and lenient_range > SENSITIVE_RANGE, + "sensitive_threshold": SENSITIVE_RANGE, + } diff --git a/dokimi/dokimi/novelty/report.py b/dokimi/dokimi/novelty/report.py index e96f810b1..5eb233d85 100644 --- a/dokimi/dokimi/novelty/report.py +++ b/dokimi/dokimi/novelty/report.py @@ -496,6 +496,15 @@ def _render_categories(artifact: dict, uncleared: str | None) -> list[str]: return [*lines, ""] +def _judged_by(artifact: dict) -> str: + """The judge line: the one judge, or a panel's judges and its rule.""" + panel = artifact.get("judge_panel") + if not panel: + return f"**{artifact['judge_model']}**" + return (f"a panel of {len(panel['judges'])} judges (**{'**, **'.join(panel['judges'])}**; " + f"{panel['rule']} rule)") + + def render_novelty_markdown(artifact: dict) -> str: """Render the summary report, in the order an author should act on it.""" lines: list[str] = [] @@ -504,7 +513,7 @@ def render_novelty_markdown(artifact: dict) -> str: lines.append("") lines.append( f"Model **{artifact['model']}** ({artifact['provider']}), judged by " - f"**{artifact['judge_model']}**. Artifact: {artifact['artifact']['tokens']} tokens, " + f"{_judged_by(artifact)}. Artifact: {artifact['artifact']['tokens']} tokens, " f"{artifact['atoms']['n']} atoms ({artifact['atoms']['sampling']}), " f"{artifact['atoms']['n_probed']} probed.{_capped_at(artifact)}" ) diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index e6b963f81..2d62e417c 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -126,6 +126,7 @@ nav: - skill-novelty.md - skill-novelty-walkthrough.md - skill-novelty-evidence.md + - judge-panel-study.md - grading-tiers.md - Repository evaluation: - repo-evaluation-contract.md diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index 7dc5535ca..cc614b1ad 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -136,6 +136,8 @@ extend_exclude = [ "scripts/conda_forge_specs.py", # Run by fuzz.yml, which installs atheris from requirements-fuzz.lock.txt. "fuzz", + # Study scripts that import their sibling llm and run_probes from a study checkout. + "research/judge-panel", ] # Tools in the dev extra (pytest and plugins, ruff, mypy, scriv), never imported by # the package. A dev dep imported from the package is DEP004. diff --git a/dokimi/research/judge-panel/analyze.py b/dokimi/research/judge-panel/analyze.py new file mode 100755 index 000000000..c86a3fc11 --- /dev/null +++ b/dokimi/research/judge-panel/analyze.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""Analyse judge-panel grades: per-judge headline numbers, kappa, self-preference, aggregation. + +Writes OUT/summary.json and CSV tables in OUT/tables/. Pure stdlib + numpy. +""" + +from __future__ import annotations + +import csv +import hashlib +import itertools +import json +import random +import sys +from collections import Counter, defaultdict +from pathlib import Path + +import numpy as np + +OUT = Path(sys.argv[1] if len(sys.argv) > 1 else "judge-panel-out") +T = OUT / "tables" +T.mkdir(exist_ok=True) +V6 = [ + "correct_complete", + "correct_missing_qualifier", + "partially_correct", + "wrong_reversed", + "wrong_other", + "abstain", +] +SCORE = { + "correct_complete": 1.0, + "correct_missing_qualifier": 0.7, + "partially_correct": 0.4, + "wrong_reversed": 0.0, + "wrong_other": 0.0, + "abstain": 0.0, +} +LEN = set(V6[:3]) +STR = {"correct_complete"} +FAM = { + "claude-sonnet-5": "anthropic", + "gpt-6-luna": "openai", + "gpt-5.6-luna": "openai", + "grok-4.6": "xai", + "kimi-k3": "moonshot", + "deepseek-v3.2": "deepseek", + "glm-5": "zai", + "qwen3-vl-235b": "alibaba", +} +AFAM = { + "claude-opus-5": "anthropic", + "claude-haiku-4-5": "anthropic", + "gpt-5.6-luna": "openai", + "grok-4.6": "xai", + "nemotron-nano-12b": "nvidia", +} +ORIG = "gpt-5.6-luna" + +rows = {} +for f in ("sample.jsonl", "repeats.jsonl"): + p = OUT / f + if p.exists(): + for l in p.read_text().splitlines(): + r = json.loads(l) + rows[f"{r['probe_id']}|{r['arm']}|{r['model']}"] = r +# The study is pinned to the judges of FAM: grades from the generation experiment's extra +# judges stay out of its tables and of the consensus. gen.py sets PANEL_ONLY = False to load them. +PANEL_ONLY = globals().get("PANEL_ONLY", True) +V = defaultdict(dict) # key -> judge -> verdict +for k, r in rows.items(): + if r.get("verdict") in SCORE: + V[k][ORIG] = r["verdict"] + +def answer_sha(r): + """Fingerprint of a row's answer, as judge_panel.py stamps the grades it makes on it.""" + return hashlib.sha256((r.get("answer") or "").encode()).hexdigest()[:16] + + +for l in (OUT / "grades.jsonl").read_text().splitlines(): + g = json.loads(l) + # A grade made on an answer the row no longer holds (its regrade failed) is not this row's. + current = g["key"] in rows and g.get("answer_sha") in (None, answer_sha(rows[g["key"]])) + if g.get("verdict") in SCORE and current and (g["judge"] in FAM or not PANEL_ONLY): + V[g["key"]][g["judge"]] = g["verdict"] +JUDGES = [ORIG] + sorted({j for d in V.values() for j in d} - {ORIG}) +NEW = [j for j in JUDGES if j != ORIG] +MODELS = list(AFAM) +cells = defaultdict(dict) # (probe, model) -> arm -> key +for k, r in rows.items(): + cells[(r["probe_id"], r["model"])][r["arm"]] = k + + +def wcsv(name, header, data): + with (T / name).open("w", newline="") as fh: + w = csv.writer(fh) + w.writerow(header) + w.writerows(data) + + +# ---------- generic "rater" = function key -> verdict-or-binary ---------- +def headline(get, model, good, gated_needs_placebo=True): + """get(key) -> bool|None (correct under `good`). Returns dict of shares over non-adversarial.""" + t = Counter() + n = 0 + nf = Counter() + nn = 0 + for (_p, m), arms in cells.items(): + if m != model or rows[arms["no_skill"]]["adversarial"]: + continue + b, w, pl = get(arms["no_skill"]), get(arms["full"]), get(arms["placebo"]) + if None in (b, w, pl): + continue + n += 1 + t["known"] += b and w + t["degraded"] += b and not w + t["added"] += w and not b + t["gated"] += w and not b and not pl + if "no_skill_repeat" in arms: + rr = get(arms["no_skill_repeat"]) + if rr is not None: + nn += 1 + nf["nf_added"] += rr and not b + nf["nf_degraded"] += b and not rr + out = {k: t[k] / n if n else None for k in ("added", "gated", "known", "degraded")} + out["n"] = n + out["nf_added"] = nf["nf_added"] / nn if nn else None + out["nf_degraded"] = nf["nf_degraded"] / nn if nn else None + out["nf_n"] = nn + return out + + +def judge_get(j, good): + return lambda k: (V[k][j] in good) if j in V[k] else None + + +summary = { + "judges": JUDGES, + "n_rows": len(rows), + "coverage": {j: sum(j in d for d in V.values()) for j in JUDGES}, +} + +# ---------- 1. per-judge headline numbers ---------- +hl = {} +tab = [] +for mode, good in (("lenient", LEN), ("strict", STR)): + for m in MODELS: + for j in JUDGES: + h = headline(judge_get(j, good), m, good) + hl[(mode, m, j)] = h + tab.append( + [ + mode, + m, + j, + h["n"], + *( + round(h[x], 3) if h[x] is not None else "" + for x in ("added", "gated", "known", "degraded", "nf_added", "nf_degraded") + ), + ] + ) +wcsv( + "per_judge_headline.csv", + [ + "mode", + "answer_model", + "judge", + "n", + "added", + "gated", + "known", + "degraded", + "noise_added", + "noise_degraded", + ], + tab, +) + +# range across judges per (mode, model) +rng = [] +for mode in ("lenient", "strict"): + for m in MODELS: + for x in ("added", "gated", "known", "degraded"): + vals = [hl[(mode, m, j)][x] for j in JUDGES if hl[(mode, m, j)][x] is not None] + rng.append( + [ + mode, + m, + x, + round(min(vals), 3), + round(max(vals), 3), + round(float(np.median(vals)), 3), + round(hl[(mode, m, ORIG)][x], 3), + ] + ) +wcsv("headline_range.csv", ["mode", "answer_model", "metric", "min", "max", "median", "orig"], rng) + +# model ranking per judge (gated lenient) -> does the order change? +rank = {} +for mode in ("lenient", "strict"): + for j in JUDGES: + order = sorted(MODELS, key=lambda m: -(hl[(mode, m, j)]["gated"] or 0)) + rank[f"{mode}|{j}"] = order +summary["gated_rank_by_judge"] = rank + + +# ---------- 2. pairwise agreement ---------- +def rounded(x, nd=3): + """``x`` rounded, or an empty cell when the statistic is undefined.""" + return "" if x is None else round(x, nd) + + +def kappa(a, b, labels): + """Cohen's kappa, or None when the judges share no row or chance agreement is 1.""" + n = len(a) + if n == 0: + return None + po = sum(x == y for x, y in zip(a, b, strict=True)) / n + ca, cb = Counter(a), Counter(b) + pe = sum(ca[l] * cb[l] for l in labels) / n / n + return (po - pe) / (1 - pe) if pe < 1 else None + + +kt = [] +kap = {} +for j1, j2 in itertools.combinations(JUDGES, 2): + ks = [k for k in V if j1 in V[k] and j2 in V[k]] + a6 = [V[k][j1] for k in ks] + b6 = [V[k][j2] for k in ks] + al = [x in LEN for x in a6] + bl = [x in LEN for x in b6] + as_ = [x in STR for x in a6] + bs = [x in STR for x in b6] + r = [ + j1, + j2, + len(ks), + rounded(kappa(a6, b6, V6)), + rounded(kappa(al, bl, [True, False])), + rounded(kappa(as_, bs, [True, False])), + rounded(sum(x == y for x, y in zip(al, bl, strict=True)) / len(ks)) if ks else "", + ] + kap[(j1, j2)] = kap[(j2, j1)] = kappa(al, bl, [True, False]) + kt.append(r) +wcsv( + "pairwise_kappa.csv", + ["judge_a", "judge_b", "n", "kappa_6way", "kappa_lenient", "kappa_strict", "agree_lenient"], + kt, +) +def mean_kappa(j): + """The mean lenient kappa of ``j`` against the other judges, over the pairs it is defined on.""" + vals = [kap[(j, o)] for o in JUDGES if o != j and kap[(j, o)] is not None] + return rounded(float(np.mean(vals))) if vals else None + + +summary["mean_kappa_lenient"] = {j: mean_kappa(j) for j in JUDGES} + +# leniency of each judge: overall lenient / strict rate +summary["judge_rates"] = { + j: { + "lenient": round(np.mean([V[k][j] in LEN for k in V if j in V[k]]), 3), + "strict": round(np.mean([V[k][j] in STR for k in V if j in V[k]]), 3), + "dist": dict(Counter(V[k][j] for k in V if j in V[k])), + } + for j in JUDGES +} + +# ---------- 3. self-preference ---------- +# Judge j's lenient (strict) rate on model m, minus the other judges' mean on the same keys. +sp = [] +for j in JUDGES: + for m in MODELS: + for mode, good in (("lenient", LEN), ("strict", STR)): + ks = [k for k in V if rows[k]["model"] == m and j in V[k] and len(V[k]) >= 4] + if not ks: + continue + own = np.mean([V[k][j] in good for k in ks]) + oth = np.mean([np.mean([V[k][o] in good for o in V[k] if o != j]) for k in ks]) + rel = "self" if j == m else ("family" if FAM.get(j) == AFAM[m] else "other") + sp.append([j, m, mode, rel, len(ks), round(own, 3), round(oth, 3), round(own - oth, 3)]) +wcsv( + "self_preference.csv", + ["judge", "answer_model", "mode", "relation", "n", "judge_rate", "others_rate", "delta"], + sp, +) +# pooled: per judge, delta on own family minus mean delta on other families +pool = {} +for mode in ("lenient", "strict"): + for j in JUDGES: + same = [r[7] for r in sp if r[0] == j and r[2] == mode and r[3] in ("self", "family")] + oth = [r[7] for r in sp if r[0] == j and r[2] == mode and r[3] == "other"] + if same: + pool[f"{mode}|{j}"] = { + "same_family_delta": round(float(np.mean(same)), 3), + "other_delta": round(float(np.mean(oth)), 3), + "self_preference": round(float(np.mean(same) - np.mean(oth)), 3), + } +summary["self_preference"] = pool + + +# ---------- 4. aggregation rules over the panel ---------- +def dawid_skene(keys, judges, iters=50): + """Binary Dawid-Skene on lenient labels. Returns key -> P(correct), judge -> (sens, spec).""" + lab = {k: {j: V[k][j] in LEN for j in judges if j in V[k]} for k in keys} + p = {k: np.mean(list(d.values())) if d else 0.5 for k, d in lab.items()} + for _ in range(iters): + sens, spec = {}, {} + for j in judges: + num1 = sum(p[k] * d[j] for k, d in lab.items() if j in d) + den1 = sum(p[k] for k, d in lab.items() if j in d) + num0 = sum((1 - p[k]) * (not d[j]) for k, d in lab.items() if j in d) + den0 = sum((1 - p[k]) for k, d in lab.items() if j in d) + sens[j] = min(max((num1 + 1) / (den1 + 2), 0.01), 0.99) + spec[j] = min(max((num0 + 1) / (den0 + 2), 0.01), 0.99) + prior = np.mean(list(p.values())) + for k, d in lab.items(): + l1, l0 = np.log(prior), np.log(1 - prior) + for j, y in d.items(): + l1 += np.log(sens[j] if y else 1 - sens[j]) + l0 += np.log(1 - spec[j] if y else spec[j]) + p[k] = 1 / (1 + np.exp(l0 - l1)) + return p, sens, spec + + +def rules(judges, keys): + """Return {rule_name: get(key)->bool|None} for lenient-binary panel decisions.""" + ds_p, sens, spec = dawid_skene(keys, judges) + + def votes(k): + return [V[k][j] for j in judges if j in V[k]] + + def mk(f): + def g(k): + v = votes(k) + return f(v, k) if len(v) >= max(1, len(judges) // 2 + 1) else None + + return g + + out = { + "majority": mk(lambda v, k: sum(x in LEN for x in v) > len(v) / 2), + "mean_score>=0.4": mk(lambda v, k: np.mean([SCORE[x] for x in v]) >= 0.4), + "median_score>=0.4": mk(lambda v, k: np.median([SCORE[x] for x in v]) >= 0.4), + "unanimous": mk(lambda v, k: all(x in LEN for x in v)), + "any_correct": mk(lambda v, k: any(x in LEN for x in v)), + "dawid_skene": mk(lambda v, k: ds_p[k] >= 0.5), + "majority_strict": mk(lambda v, k: sum(x in STR for x in v) > len(v) / 2), + } + return out, sens, spec + + +allkeys = list(V) +panel_rules, sens, spec = rules(NEW, allkeys) +summary["dawid_skene_new_judges"] = { + j: {"sens": round(sens[j], 3), "spec": round(spec[j], 3)} for j in NEW +} +agg = [] +for name, g in panel_rules.items(): + for m in MODELS: + h = headline(g, m, None) + agg.append( + [ + name, + m, + h["n"], + *( + round(h[x], 3) if h[x] is not None else "" + for x in ("added", "gated", "known", "degraded", "nf_added", "nf_degraded") + ), + ] + ) +wcsv( + "aggregation_rules.csv", + [ + "rule", + "answer_model", + "n", + "added", + "gated", + "known", + "degraded", + "noise_added", + "noise_degraded", + ], + agg, +) + +# agreement of each single judge and each rule with the DS consensus (on NEW judges) +ref = panel_rules["dawid_skene"] +agree = [] +for j in JUDGES: + ks = [k for k in allkeys if j in V[k] and ref(k) is not None] + agree.append( + ["judge:" + j, len(ks), round(np.mean([(V[k][j] in LEN) == ref(k) for k in ks]), 3)] + ) +for name, g in panel_rules.items(): + ks = [k for k in allkeys if g(k) is not None and ref(k) is not None] + agree.append(["rule:" + name, len(ks), round(np.mean([g(k) == ref(k) for k in ks]), 3)]) +wcsv("agreement_with_ds.csv", ["rater", "n", "agree_with_ds_consensus"], agree) + +# ---------- 5. how many judges are enough ---------- +# Panel size s: random subsets of NEW judges, majority vote; agreement with the DS consensus. +rnd = random.Random(3) +sz = [] +for s in range(1, len(NEW) + 1): + subs = list(itertools.combinations(NEW, s)) + if len(subs) > 35: + subs = rnd.sample(subs, 35) + accs, spreads = [], defaultdict(list) + for sub in subs: + + def g(k, sub=sub): + v = [V[k][j] in LEN for j in sub if j in V[k]] + return (sum(v) > len(v) / 2) if v else None + + ks = [k for k in allkeys if g(k) is not None and ref(k) is not None] + accs.append(np.mean([g(k) == ref(k) for k in ks])) + for m in MODELS: + share = headline(g, m, None)["gated"] + if share is not None: + spreads[m].append(share) + sd = np.mean([np.std(v) for v in spreads.values()]) + rg = np.mean([max(v) - min(v) for v in spreads.values()]) + sz.append( + [ + s, + len(subs), + round(float(np.mean(accs)), 3), + round(float(np.min(accs)), 3), + round(float(sd), 4), + round(float(rg), 3), + ] + ) +wcsv( + "panel_size.csv", + [ + "panel_size", + "subsets", + "mean_agree_with_full_ds", + "worst_agree", + "sd_gated_added_across_subsets", + "range_gated_added_across_subsets", + ], + sz, +) + +# ---------- 6. which arms/probe types do judges disagree on ---------- +dis = [] +for by in ("arm", "style", "adversarial", "model"): + grp = defaultdict(list) + for k in allkeys: + v = [V[k][j] in LEN for j in NEW if j in V[k]] + if len(v) >= 5: + frac = sum(v) / len(v) + grp[str(rows[k][by])].append(min(frac, 1 - frac) > 0) # any disagreement + for gname, xs in sorted(grp.items()): + dis.append([by, gname, len(xs), round(float(np.mean(xs)), 3)]) +wcsv("disagreement_by_group.csv", ["by", "group", "n", "share_items_not_unanimous"], dis) + +(OUT / "summary.json").write_text(json.dumps(summary, indent=1, default=str)) +print(json.dumps({k: summary[k] for k in ("n_rows", "coverage", "mean_kappa_lenient")}, indent=1)) +for f in sorted(T.glob("*.csv")): + print("==", f.name) + print(f.read_text()[:2500]) diff --git a/dokimi/research/judge-panel/fig.py b/dokimi/research/judge-panel/fig.py new file mode 100644 index 000000000..55c112129 --- /dev/null +++ b/dokimi/research/judge-panel/fig.py @@ -0,0 +1,86 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Dot plot: placebo-gated added share per answer model, one dot per judge, panel rules marked.""" + +import csv +import sys +from pathlib import Path + +T = Path(sys.argv[1]) +out = Path(sys.argv[2]) +mode = sys.argv[3] if len(sys.argv) > 3 else "lenient" +MODELS = ["claude-opus-5", "gpt-5.6-luna", "claude-haiku-4-5", "grok-4.6", "nemotron-nano-12b"] +COL = { + "gpt-5.6-luna": "#d1495b", + "claude-sonnet-5": "#c47a2c", + "gpt-6-luna": "#8e44ad", + "grok-4.6": "#2a9d8f", + "kimi-k3": "#3b6fb6", + "deepseek-v3.2": "#6c757d", + "glm-5": "#7fb069", + "qwen3-vl-235b": "#e9a03b", +} +pj = [r for r in csv.DictReader((T / "per_judge_headline.csv").open()) if r["mode"] == mode] +ag = { + (r["rule"], r["answer_model"]): float(r["gated"]) + for r in csv.DictReader((T / "aggregation_rules.csv").open()) + if r["gated"] != "" +} +W, H, L, R, TOP, ROW = 760, 70 + 46 * len(MODELS) + 60, 170, 30, 40, 46 +x0, x1 = 0.0, 0.6 + + +def X(v): + return L + (W - L - R) * (v - x0) / (x1 - x0) + + +s = [ + f'', + f'', + f'Placebo-gated added share ({mode}) by judge', +] +for t in range(7): + v = t / 10 + s.append( + f'' + ) + s.append( + f'{v:.1f}' + ) +for i, m in enumerate(MODELS): + y = TOP + ROW * i + ROW / 2 + s.append(f'{m}') + vals = [(r["judge"], float(r["gated"])) for r in pj if r["answer_model"] == m and r["gated"] != ""] + if not vals: + continue + lo, hi = min(v for _, v in vals), max(v for _, v in vals) + s.append( + f'' + ) + for k, (j, v) in enumerate(vals): + dy = (k - len(vals) / 2) * 2.2 + shape = ( + f'' + if j == "gpt-5.6-luna" + else f'' + ) + s.append(shape) + if mode == "lenient" and ("majority", m) in ag: + v = ag[("majority", m)] + s.append(f'') +ly = TOP + ROW * len(MODELS) + 32 +lx = 20 +for j, c in COL.items(): + mark = "square = original judge" if j == "gpt-5.6-luna" else "" + s.append( + f'{j}' + ) + lx += 92 +if mode == "lenient": + s.append( + f'square: original judge (gpt-5.6-luna); triangle: 7-judge majority; grey bar: judge range. n = 60-62 paired non-reversal probes per model.' + ) +s.append("") +out.write_text("\n".join(s)) +print("wrote", out) diff --git a/dokimi/research/judge-panel/gen.py b/dokimi/research/judge-panel/gen.py new file mode 100755 index 000000000..0681e3197 --- /dev/null +++ b/dokimi/research/judge-panel/gen.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""Old vs new judge generations on the same rows: leniency, headline shift, agreement, panels. + +Reuses analyze.py's loading and headline() (run with its tables dir redirected), writes +OUT/tables_gen/*.csv and OUT/gen_summary.json. +""" +import json +import sys +from pathlib import Path + +HERE = Path(__file__).parent +src = (HERE / "analyze.py").read_text().split("# ---------- 2. pairwise agreement")[0] +src = src.replace('T = OUT / "tables"', 'T = OUT / "tables_gen"') +g = {"__name__": "gen", "sys": sys, "PANEL_ONLY": False} # all judges, not the study's seven +exec(compile(src, "analyze.py", "exec"), g) # nosec B102 - runs this directory's own analyze.py +V, rows, cells, headline, wcsv = g["V"], g["rows"], g["cells"], g["headline"], g["wcsv"] +LEN, STR, SCORE, V6, MODELS, OUT = g["LEN"], g["STR"], g["SCORE"], g["V6"], g["MODELS"], g["OUT"] + +PAIRS = [ # family, old -> new + ("openai", "gpt-5.6-luna", "gpt-6-luna"), + ("openai", "gpt-5.6-luna", "gpt-6-sol"), + ("moonshot", "kimi-k2.5", "kimi-k3"), + ("anthropic", "claude-haiku-4-5", "claude-sonnet-5"), + ("anthropic", "claude-sonnet-5", "claude-opus-5-5"), + ("anthropic", "claude-haiku-4-5", "claude-opus-5-5"), + ("deepseek", "deepseek-r1", "deepseek-v3.2"), +] +OLD_PANEL = ["gpt-5.6-luna", "kimi-k2.5", "claude-haiku-4-5"] +NEW_PANEL = ["gpt-6-luna", "kimi-k3", "claude-opus-5-5"] +ALL = sorted({j for d in V.values() for j in d}) + + +def kappa(a, b): + """Cohen's kappa of two judges on the answers both graded, and that count; kappa is None + when they share none or both put every shared answer in one class (chance agreement 1).""" + keys = [k for k in V if a in V[k] and b in V[k]] + n = len(keys) + if not n: + return None, 0 + po = sum(V[k][a] == V[k][b] for k in keys) / n + pe = sum((sum(V[k][a] == l for k in keys) / n) * (sum(V[k][b] == l for k in keys) / n) + for l in V6) + return ((po - pe) / (1 - pe) if pe < 1 else None), n + + +def rounded(x, digits=3): + return None if x is None else round(x, digits) + + +def lenient_rate(j, keys=None): + """Lenient and strict rate of judge ``j`` over its grades, or over ``keys`` only.""" + vs = [d[j] for k, d in V.items() if j in d and (keys is None or k in keys)] + if not vs: + return None, None, 0 + return sum(v in LEN for v in vs) / len(vs), sum(v in STR for v in vs) / len(vs), len(vs) + + +def rate_delta(old, new): + """Lenient-rate change from ``old`` to ``new`` on the answers both graded, so a judge that + also graded other rows (the repeats) is not compared on a different population.""" + shared = {k for k, d in V.items() if old in d and new in d} + lo, ln = lenient_rate(old, shared)[0], lenient_rate(new, shared)[0] + return None if lo is None else ln - lo + + +def median(votes): + r = sorted(votes, key=lambda v: -SCORE[v]) + return r[len(r) // 2] + + +def panel_get(panel, good, also=()): + """The panel's median verdict, None unless every judge of it and of ``also`` graded the cell.""" + def get(k): + vs = [V[k][j] for j in panel if j in V[k]] + return (median(vs) in good) if len(vs) == len(panel) and all(a in V[k] for a in also) else None + return get + + +def jget(j, good, also=()): + """``j``'s correctness per grade, None where it has no grade or one of ``also`` has none: a + cell counts only when every judge in the comparison graded every arm, so both shares share one sample.""" + return lambda k: (V[k][j] in good) if j in V[k] and all(a in V[k] for a in also) else None + + +out = {"judges": ALL, "per_judge": {}, "pairs": [], "panels": {}} +tab = [] +for j in ALL: + le, st, n = lenient_rate(j) + out["per_judge"][j] = {"lenient_rate": rounded(le, 4), "strict_rate": rounded(st, 4), "n": n} + tab.append([j, n, rounded(le, 4), rounded(st, 4)]) +wcsv("judge_leniency.csv", ["judge", "n", "lenient_rate", "strict_rate"], tab) + +tab, ktab = [], [] +for fam, old, new in PAIRS: + k, n = kappa(old, new) + row = {"family": fam, "old": old, "new": new, "kappa_old_new": rounded(k), "n": n, + "lenient_rate_delta": rounded(rate_delta(old, new), 4), + "added": {}} + for mode, good in (("lenient", LEN), ("strict", STR)): + for m in MODELS: + ho = headline(jget(old, good, (new,)), m, good) + hn = headline(jget(new, good, (old,)), m, good) + d = None if ho["added"] is None or hn["added"] is None else hn["added"] - ho["added"] + row["added"][f"{mode}:{m}"] = [ho["added"], hn["added"]] + tab.append([fam, old, new, mode, m, rounded(ho["added"]), rounded(hn["added"]), + round(d, 3) if d is not None else ""]) + out["pairs"].append(row) + ktab.append([fam, old, new, n, rounded(k), row["lenient_rate_delta"]]) +wcsv("generation_kappa.csv", ["family", "old_judge", "new_judge", "n", "kappa", "lenient_rate_delta"], + ktab) +wcsv("generation_added_share.csv", ["family", "old_judge", "new_judge", "mode", "answer_model", + "added_old", "added_new", "delta"], tab) + +# agreement with the rest of the panel: does a newer judge sit closer to the consensus? +tab = [] +for j in ALL: + ks = [k for k in (kappa(j, o)[0] for o in ALL if o != j) if k is not None] + mean_k = rounded(sum(ks) / len(ks)) if ks else None # None: no other judge shares an answer + out["per_judge"][j]["mean_kappa_vs_others"] = mean_k + tab.append([j, mean_k]) +wcsv("judge_mean_kappa.csv", ["judge", "mean_kappa_vs_others"], tab) + +tab = [] +for name, panel, other in (("old", OLD_PANEL, NEW_PANEL), ("new", NEW_PANEL, OLD_PANEL)): + out["panels"][name] = {"judges": panel, "added": {}} + for mode, good in (("lenient", LEN), ("strict", STR)): + for m in MODELS: + h = headline(panel_get(panel, good, other), m, good) + out["panels"][name]["added"][f"{mode}:{m}"] = h["added"] + tab.append([name, "+".join(panel), mode, m, h["n"], rounded(h["added"])]) +wcsv("generation_panels.csv", ["panel", "judges", "mode", "answer_model", "n", "added"], tab) +(OUT / "gen_summary.json").write_text(json.dumps(out, indent=1)) + +print("judge leniency strict mean_kappa") +for j in ALL: + p = out["per_judge"][j] + print(f" {j:24s} {p['lenient_rate']:.3f} {p['strict_rate']:.3f} {p['mean_kappa_vs_others'] or float('nan'):.3f}") +print("pairs (kappa old-new, lenient-rate delta, lenient added old->new per answer model)") +def shown(x): + return "nan" if x is None else f"{x:.2f}" + + +for r in out["pairs"]: + adds = " ".join(f"{k.split(':')[1][:10]}={shown(v[0])}->{shown(v[1])}" + for k, v in r["added"].items() if k.startswith("lenient")) + print(f" {r['old']}->{r['new']}: k={r['kappa_old_new']} dlen={r['lenient_rate_delta'] or float('nan'):+.3f} {adds}") +print("panels median, lenient added") +for name, p in out["panels"].items(): + print(" ", name, {k.split(':')[1]: rounded(v) for k, v in p["added"].items() if k.startswith("lenient")}) + print(" ", name, "strict", {k.split(':')[1]: rounded(v) for k, v in p["added"].items() if k.startswith("strict")}) diff --git a/dokimi/research/judge-panel/judge_panel.py b/dokimi/research/judge-panel/judge_panel.py new file mode 100755 index 000000000..dd2f93d96 --- /dev/null +++ b/dokimi/research/judge-panel/judge_panel.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""Judge-panel study: re-grade a stratified sample of Dokimi probe answers with several judges. + +Steps (each resumable; LLM calls cached in OUT/cache, never in the study cache): + sample -> OUT/sample.jsonl (all arms for sampled (probe, model) cells + orig luna verdict) + repeat -> OUT/repeats.jsonl (fresh no_skill answer, cache bypassed: the noise-floor arm) + grade -> OUT/grades.jsonl (one row per (row_key, judge)) +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import sys +from collections import defaultdict +from pathlib import Path + +STUDY = Path(os.environ["JP_STUDY"]) # the skill-novelty study checkout +sys.path.insert(0, str(STUDY / "tools")) +import llm # noqa: E402 +import run_probes as rp # noqa: E402 + +OUT = Path(os.environ.get("JP_OUT", "judge-panel-out")) +OUT.mkdir(parents=True, exist_ok=True) +llm.CACHE = OUT / "cache" +llm.CACHE.mkdir(exist_ok=True) +llm.LOG = OUT / "llm_calls.jsonl" +llm.NO_TEMPERATURE.update( + {"gpt-6-luna", "kimi-k3", "gpt-6-sol", "claude-opus-5-5"} +) # these reject temperature=0 and run at their default + +ANSWER_MODELS = [ + "claude-opus-5", + "gpt-5.6-luna", + "grok-4.6", + "claude-haiku-4-5", + "nemotron-nano-12b", +] +ARMS = ["no_skill", "root", "routed", "full", "placebo", "shuffled"] +JUDGES = [ + "claude-sonnet-5", + "gpt-6-luna", + "grok-4.6", + "kimi-k3", + "deepseek-v3.2", + "glm-5", + "qwen3-vl-235b", +] +# gen.py's re-grade used --judges gpt-6-sol,kimi-k2.5,claude-haiku-4-5,claude-opus-5-5, +# deepseek-r1,nemotron-super-3-120b,minimax-m2.5 + + +def key(r): + return f"{r['probe_id']}|{r['arm']}|{r['model']}" + + +def sha(r): + """Fingerprint of the answer a grade was made on.""" + return hashlib.sha256((r["answer"] or "").encode()).hexdigest()[:16] + + +def do_sample(per_model: int): + rows = [json.loads(l) for l in (STUDY / "data/probe_results.jsonl").open()] + cell = defaultdict(dict) + for r in rows: + if r.get("verdict") and r["verdict"] != "ungraded": + cell[(r["probe_id"], r["model"])][r["arm"]] = r + rnd = random.Random(20261108) + out = [] + for m in ANSWER_MODELS: + full = [(p, c) for (p, mm), c in cell.items() if mm == m and all(a in c for a in ARMS)] + strata = defaultdict(list) + for _p, c in full: + strata[(c["no_skill"]["skill"], c["no_skill"]["adversarial"])].append(c) + tot = len(full) + picked = [] + for _s, cs in sorted(strata.items()): + k = max(1, round(per_model * len(cs) / tot)) + picked += rnd.sample(cs, min(k, len(cs))) + for c in picked: + for a in ARMS: + r = c[a] + out.append( + { + k: r.get(k) + for k in ( + "probe_id", + "atom_id", + "skill", + "arm", + "model", + "model_family", + "adversarial", + "style", + "question", + "answer", + "verdict", + "verdict2", + "grader2", + ) + } + ) + print(m, "eligible", tot, "picked", len(picked)) + with (OUT / "sample.jsonl").open("w") as fh: + for r in out: + fh.write(json.dumps(r) + "\n") + print("rows", len(out)) + + +def do_repeat(workers: int): + rows = [json.loads(l) for l in (OUT / "sample.jsonl").open()] + base = [r for r in rows if r["arm"] == "no_skill"] + + def run(r): + rec = llm.chat( + r["model"], + [{"role": "system", "content": rp.SYSTEM}, {"role": "user", "content": r["question"]}], + temperature=0.0, + max_tokens=1400, + use_cache=False, + tag=f"repeat|{r['model']}|{r['probe_id']}", + ) + return { + **r, + "arm": "no_skill_repeat", + "answer": rec["text"], + "verdict": None, + "verdict2": None, + "grader2": None, + } + + # Resuming keeps the repeat answers already drawn: grades are keyed by probe, arm, model and + # judge, so a fresh draw under an old key would inherit the old answer's verdicts. + kept = {} + if (OUT / "repeats.jsonl").exists(): + kept = {key(r): r for r in map(json.loads, (OUT / "repeats.jsonl").open())} + todo = [r for r in base if key({**r, "arm": "no_skill_repeat"}) not in kept] + res = llm.pmap(run, todo, workers=workers, desc="repeats") + ok = [x for x in res if isinstance(x, dict)] + ok = [kept[k] for k in kept if k in {key({**r, "arm": "no_skill_repeat"}) for r in base}] + ok + print("repeat ok", len(ok), "failed", len(res) - len(ok)) + with (OUT / "repeats.jsonl").open("w") as fh: + for r in ok: + fh.write(json.dumps(r) + "\n") + + +def do_grade(workers: int, judges: list[str]): + probes = {} + for l in (STUDY / "data/probes.jsonl").open(): + p = json.loads(l) + probes[p["probe_id"]] = p + rows = [json.loads(l) for l in (OUT / "sample.jsonl").open()] + if (OUT / "repeats.jsonl").exists(): + rows += [json.loads(l) for l in (OUT / "repeats.jsonl").open()] + done = set() + gp = OUT / "grades.jsonl" + if gp.exists(): + for l in gp.open(): + g = json.loads(l) + if g.get("verdict"): + done.add((g["key"], g["judge"], g.get("answer_sha"))) + # A grade counts only for the answer it graded: one made before answers were fingerprinted + # (no answer_sha) is kept, any other is redone when the row's answer has changed. + jobs = [(r, j) for j in judges for r in rows + if (key(r), j, sha(r)) not in done and (key(r), j, None) not in done] + random.Random(1).shuffle(jobs) # interleave judges so one slow route does not serialize + print("grade jobs", len(jobs)) + + def grade(job): + r, j = job + p = probes[r["probe_id"]] + g = llm.ask_json( + j, + rp.GRADE_PROMPT.format( + question=r["question"], + key=p["atom_statement"], + quals=", ".join(p["atom_qualifiers"]) or "none", + core=p.get("expected_core", ""), + ptype="adversarial" if r["adversarial"] else r["style"], + answer=(r["answer"] or "")[:6000], + ), + tag=f"grade|{j}|{key(r)}", + max_tokens=1200, + repair_with="claude-haiku-4-5", + ) + v = g.get("verdict") if isinstance(g, dict) else None + if v not in rp.SCORE: + v = None + rec = { + "key": key(r), + "judge": j, + "answer_sha": sha(r), + "verdict": v, + "reason": (g.get("reason") if isinstance(g, dict) else None), + } + with lock: + with gp.open("a") as fh: + fh.write(json.dumps(rec) + "\n") + return rec + + import threading + + lock = threading.Lock() + res = llm.pmap(grade, jobs, workers=workers, desc="grading") + bad = sum(1 for x in res if not isinstance(x, dict) or not x.get("verdict")) + print("graded", len(res), "bad", bad, llm.stats()) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("step", choices=["sample", "repeat", "grade"]) + ap.add_argument("--per-model", type=int, default=90) + ap.add_argument("--workers", type=int, default=32) + ap.add_argument("--judges", default=",".join(JUDGES)) + a = ap.parse_args() + if a.step == "sample": + do_sample(a.per_model) + elif a.step == "repeat": + do_repeat(a.workers) + else: + do_grade(a.workers, a.judges.split(",")) diff --git a/dokimi/research/records/experiment/X031.md b/dokimi/research/records/experiment/X031.md new file mode 100644 index 000000000..ed9af895d --- /dev/null +++ b/dokimi/research/records/experiment/X031.md @@ -0,0 +1,31 @@ +--- +id: X031 +type: experiment +title: Eight-judge panel on the skill-novelty headline numbers +date: '2026-11-08' +idea: I005 +hypothesis: The per-model added, known and placebo-gated shares do not depend on which LLM judge grades the answers. +outcome: refuted +--- + +## Setup +2,664 existing study answers (444 probe-model cells, all six arms, five answer models, +stratified by skill and reversal probe) plus 268 fresh no-skill repeats, re-graded with the +study's prompt by seven judges from seven families. The original judge's grades were kept. +The aggregation rules compared were majority, median, mean, unanimous, any-correct and +Dawid-Skene, over panel sizes 1-7. + +## Result +The lenient placebo-gated added share moves by 0.05-0.23 per answer model across judges, and +the original judge is the most lenient, at the low end. The extremes of the answer-model ranking +(lenient placebo-gated added share: nemotron first, opus last) are stable under every judge; +the models in between swap. Pairwise lenient kappa is 0.70-0.88, and highest within one family. No +self-preference above 0.04 was seen. Majority, median and Dawid-Skene agree on 99.3% of rows. +The rule moves numbers as much as the judge does. Three judges cut the spread from 0.039 to +0.025 SD. Details: docs/judge-panel-study.md. + +## Evidence +- local run, 2,932 answers x 7 judges, 20,524 grading calls diff --git a/dokimi/research/records/finding/F021.md b/dokimi/research/records/finding/F021.md new file mode 100644 index 000000000..edf12837b --- /dev/null +++ b/dokimi/research/records/finding/F021.md @@ -0,0 +1,22 @@ +--- +id: F021 +type: finding +title: The panel aggregation rule is as large a knob as the judge +date: '2026-11-08' +experiments: +- X031 +confidence: med +--- + +## Statement +On the same verdicts, unanimous, mean-score and any-correct rules move the placebo-gated added +share by up to 0.20 against majority, because one rule applies to both the baseline and the +treatment arm. Majority, median and binary Dawid-Skene are equivalent here. Judges disagree +most on the no-skill and placebo arms (30% and 21% of rows not unanimous, against 17% with +the skill). + +## Applies to +Judge panels, arms run, eval judge agreement diff --git a/dokimi/research/records/finding/F022.md b/dokimi/research/records/finding/F022.md new file mode 100644 index 000000000..fdcc3ed47 --- /dev/null +++ b/dokimi/research/records/finding/F022.md @@ -0,0 +1,27 @@ +--- +id: F022 +type: finding +title: A newer judge generation is a judge change, not a re-run +date: '2026-11-08' +experiments: +- X031 +confidence: med +--- + +## Statement +On 2,932 rows, judge generations within a family (openai, moonshot, anthropic, deepseek) are +compared: +- They agree at κ 0.61–0.72, about the same as judges from different vendors (mean κ + 0.51–0.64). +- Their lenient rates shift by up to ±0.12, with no common direction. +- One answer model's lenient added share moves by up to 0.15. +- A newer judge sits no closer to the panel consensus. + +A three-family median panel cuts the generation shift to ≤0.07 lenient (≤0.11 strict) and +keeps the order of the extreme answer models. + +## Applies to +Judge panels, judge upgrades, longitudinal comparisons of graded results diff --git a/dokimi/research/records/finding/F023.md b/dokimi/research/records/finding/F023.md new file mode 100644 index 000000000..353ae84c4 --- /dev/null +++ b/dokimi/research/records/finding/F023.md @@ -0,0 +1,22 @@ +--- +id: F023 +type: finding +title: A single lenient judge understates the added share; model ranking survives any judge +date: '2026-11-08' +experiments: +- X031 +confidence: med +--- + +## Statement +Per-model added and known shares depend on the judge by up to 0.23 (lenient), and the +original study judge is at the lenient, low-added end. The order of answer models by the lenient +placebo-gated added share keeps nemotron first and opus last under all eight judges; the +models in between swap, and the ungated share does not keep opus last. Report shares with a +judge range, or a 3+ judge panel from different families, and keep claims about the extremes. + +## Applies to +Skill novelty, arms reports diff --git a/dokimi/tests/test_judge_panel_study_scripts.py b/dokimi/tests/test_judge_panel_study_scripts.py new file mode 100644 index 000000000..19910ee29 --- /dev/null +++ b/dokimi/tests/test_judge_panel_study_scripts.py @@ -0,0 +1,105 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""research/judge-panel/analyze.py stays pinned to the study's judges when the generation +experiment adds grades from others.""" + +import hashlib +import json +import sys +from pathlib import Path + +HERE = Path(__file__).parent.parent / "research" / "judge-panel" +CC = "correct_complete" +MODELS = ("claude-opus-5", "claude-haiku-4-5", "gpt-5.6-luna", "grok-4.6", "nemotron-nano-12b") +ARMS = ("no_skill", "full", "placebo") + + +def _load(tmp_path, monkeypatch, **extra): + monkeypatch.setattr(sys, "argv", ["analyze.py", str(tmp_path)]) + rows = [ + {"probe_id": "p1", "arm": arm, "model": model, "adversarial": False, "verdict": CC} + for model in MODELS for arm in ARMS + ] + (tmp_path / "sample.jsonl").write_text("\n".join(map(json.dumps, rows)), encoding="utf-8") + grades = [{"key": f"p1|{arm}|{model}", "judge": judge, "verdict": CC} + for model in MODELS for arm in ARMS for judge in ("kimi-k3", "gpt-6-sol")] + (tmp_path / "grades.jsonl").write_text("\n".join(map(json.dumps, grades)), encoding="utf-8") + src = (HERE / "analyze.py").read_text(encoding="utf-8").split("# ---------- 2. pairwise")[0] + scope = {"__name__": "analyze", **extra} + exec(compile(src, "analyze.py", "exec"), scope) # nosec B102 - this directory's own script + return scope + + +def test_analyze_ignores_grades_from_judges_outside_its_panel(tmp_path, monkeypatch): + scope = _load(tmp_path, monkeypatch) + assert "gpt-6-sol" not in scope["JUDGES"] + assert scope["JUDGES"] == ["gpt-5.6-luna", "kimi-k3"] + + +def test_the_generation_study_can_still_load_every_judge(tmp_path, monkeypatch): + scope = _load(tmp_path, monkeypatch, PANEL_ONLY=False) + assert "gpt-6-sol" in {j for d in scope["V"].values() for j in d} + + +def test_a_grade_on_a_replaced_answer_is_not_loaded(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "argv", ["analyze.py", str(tmp_path)]) + fp = lambda text: hashlib.sha256(text.encode()).hexdigest()[:16] # noqa: E731 - as judge_panel.py + key = "p1|full|claude-opus-5" + rows = [{"probe_id": "p1", "arm": arm, "model": model, "adversarial": False, "verdict": CC, + "answer": "the replacement answer"} for model in MODELS for arm in ARMS] + (tmp_path / "sample.jsonl").write_text("\n".join(map(json.dumps, rows)), encoding="utf-8") + row = rows[ARMS.index("full") + len(ARMS) * MODELS.index("claude-opus-5")] + grades = [{"key": key, "judge": "kimi-k3", "verdict": "wrong_other", "answer_sha": fp("the old answer")}, + {"key": key, "judge": "gpt-5.6-luna", "verdict": CC, "answer_sha": fp(row["answer"])}] + (tmp_path / "grades.jsonl").write_text("\n".join(map(json.dumps, grades)), encoding="utf-8") + src = (HERE / "analyze.py").read_text(encoding="utf-8").split("# ---------- 2. pairwise")[0] + scope = {"__name__": "analyze"} + exec(compile(src, "analyze.py", "exec"), scope) # nosec B102 - this directory's own script + assert "kimi-k3" not in scope["V"][key] # its regrade of the old answer is stale + assert scope["V"][key]["gpt-5.6-luna"] == CC + + +def _write_disjoint_judges(tmp_path): + """Two judges with no row in common, and the original judge on every row.""" + rows, grades = [], [] + for probe, judge in (("p1", "claude-sonnet-5"), ("p2", "gpt-6-luna")): # no row has both + for model in MODELS: + for arm in ARMS: + key = f"{probe}|{arm}|{model}" + rows.append({"probe_id": probe, "arm": arm, "model": model, "adversarial": False, + "verdict": CC, "answer": key}) + grades.append({"key": key, "judge": judge, "verdict": CC}) + (tmp_path / "sample.jsonl").write_text("\n".join(map(json.dumps, rows)), encoding="utf-8") + (tmp_path / "grades.jsonl").write_text("\n".join(map(json.dumps, grades)), encoding="utf-8") + + +def test_judges_with_disjoint_rows_give_an_empty_kappa_cell_not_a_crash(tmp_path): + import subprocess + + _write_disjoint_judges(tmp_path) + (tmp_path / "tables").mkdir() + subprocess.run([sys.executable, str(HERE / "analyze.py"), str(tmp_path)], check=True, + capture_output=True, timeout=300) # nosec B603 - this directory's own script + assert (tmp_path / "tables" / "pairwise_kappa.csv").exists() + # both judges grade every row the same way: chance agreement is 1, kappa is undefined + rows = [json.loads(line) for line in (tmp_path / "sample.jsonl").read_text(encoding="utf-8").splitlines()] + grades = [{"key": r["answer"], "judge": j, "verdict": CC} for r in rows + for j in ("claude-sonnet-5", "gpt-6-luna")] + (tmp_path / "grades.jsonl").write_text("\n".join(map(json.dumps, grades)), encoding="utf-8") + subprocess.run([sys.executable, str(HERE / "analyze.py"), str(tmp_path)], check=True, + capture_output=True, timeout=300) # nosec B603 - this directory's own script + cells = (tmp_path / "tables" / "pairwise_kappa.csv").read_text(encoding="utf-8").splitlines() + assert any(c.startswith("claude-sonnet-5,gpt-6-luna,") and c.split(",")[3:6] == ["", "", ""] + for c in cells) + + +def test_the_generation_study_runs_when_a_judge_has_no_shared_rows(tmp_path): + import subprocess + + _write_disjoint_judges(tmp_path) + (tmp_path / "tables_gen").mkdir() + subprocess.run([sys.executable, str(HERE / "gen.py"), str(tmp_path)], check=True, + capture_output=True, timeout=300) # nosec B603 - this directory's own script + assert (tmp_path / "gen_summary.json").exists() + diff --git a/dokimi/tests/test_novelty_panel.py b/dokimi/tests/test_novelty_panel.py new file mode 100644 index 000000000..619df6870 --- /dev/null +++ b/dokimi/tests/test_novelty_panel.py @@ -0,0 +1,394 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The judge panel: aggregation rules, per-judge 2×2, vendor and sensitivity warnings, CLI.""" + +from __future__ import annotations + +import argparse +import json + +import pytest + +from dokimi.novelty.atoms import Atom +from dokimi.novelty.client import LLMClient +from dokimi.novelty.cost import judge_refs as _judge_refs +from dokimi.novelty.cost import role_models as _role_models +from dokimi.novelty.judge2 import check_judge2 +from dokimi.novelty.measure import ( + BASELINE_ARM, + MeasureResult, + ProbeRun, + SelfJudgeError, + run_measurement, +) +from dokimi.novelty.panel import aggregate, panel_warnings, shared_vendors +from dokimi.novelty.panel_record import panel_record +from dokimi.novelty.probes import Probe +from dokimi.utils import InputError + +CC, CMQ, PC = "correct_complete", "correct_missing_qualifier", "partially_correct" +WO, WR, AB = "wrong_other", "wrong_reversed", "abstained" + + +@pytest.mark.parametrize("votes, rule, want", [ + ([CC, CMQ, WO], "median", CMQ), + ([CC, WO, WO], "median", WO), + ([CC, CC, WO, WO], "median", WO), # an even panel rounds toward wrong + ([WR, WR, WO, CC, CC], "median", WR), # the most common wrong verdict survives + ([CC, PC, WO], "majority", PC), # correct majority: lower median of the correct votes + ([CC, WO, AB, AB], "majority", AB), + ([CC, CMQ, CC], "unanimous", CMQ), # all correct: the lowest correct vote + ([CC, CC, WR], "unanimous", WR), + ([], "median", None), +]) +def test_aggregate_rules(votes, rule, want): + assert aggregate(votes, rule) == want + + +def test_aggregate_rejects_unknown_rule(): + with pytest.raises(ValueError, match="rule"): + aggregate([CC], "mean") + + +def test_shared_vendors_names_only_repeated_families(): + assert shared_vendors(["gpt-6-luna", "gpt-6-sol", "claude-sonnet-5", "kimi-k3"]) == { + "openai": ["gpt-6-luna", "gpt-6-sol"]} + + +def _judge(model: str, verdict_for): + def transport(prompt, system): + return json.dumps({"verdict": verdict_for(prompt)}), None + return LLMClient(model=model, transport=transport) + + +def _setup(): + atoms = [Atom(statement=f"fact {i}", question=f"Q{i}?", answer="yes", span_ids=("s0001",)) + for i in range(4)] + probes = [Probe(atom_id=a.id, kind="direct", question=a.question, expected=a.answer) + for a in atoms] + + def answer(prompt, system): + return ("ARTIFACT" if "Reference material" in prompt else "BARE") + "\nconfidence: 0.5", None + + return atoms, probes, LLMClient(model="answerer", transport=answer) + + +def test_panel_grades_with_every_judge_and_keeps_each_verdict(): + atoms, probes, answerer = _setup() + # lenient judge: everything correct; strict judges: only the with-artifact answer + lenient = _judge("gpt-x", lambda p: CC) + strict_a = _judge("claude-y", lambda p: CC if "RESPONSE: ARTIFACT" in p else WO) + strict_b = _judge("kimi-z", lambda p: CC if "RESPONSE: ARTIFACT" in p else WO) + result = run_measurement(atoms, probes, "the artifact", answerer=answerer, judge=lenient, + panel=(strict_a, strict_b), aggregate="median", concurrency=1) + assert all(set(r.judge_verdicts) == {"gpt-x", "claude-y", "kimi-z"} for r in result.runs) + assert result.paired("lenient")["added"] == 1.0 # the panel sides with the two strict judges + rec = panel_record(result, "median") + assert rec["judges"] == ["gpt-x", "claude-y", "kimi-z"] + assert rec["per_judge_paired"]["gpt-x"]["lenient"]["added"] == 0.0 + assert rec["per_judge_paired"]["kimi-z"]["lenient"]["added"] == 1.0 + assert rec["added_share_range"]["lenient"] == 1.0 + assert rec["judge_sensitive"] is True + assert any("judge-sensitive" in w for w in panel_warnings(rec)) + assert "judge_verdicts" in result.runs[0].as_dict() + + +def test_single_judge_run_records_no_panel(): + atoms, probes, answerer = _setup() + result = run_measurement(atoms, probes, "t", answerer=answerer, + judge=_judge("j", lambda p: CC), concurrency=1) + assert result.runs[0].judge_verdicts is None + assert "judge_verdicts" not in result.runs[0].as_dict() + assert panel_record(result, "median") is None + + +def test_a_judge_that_fails_is_left_out_of_the_vote(): + atoms, probes, answerer = _setup() + broken = _judge("broken", lambda p: None) + result = run_measurement(atoms[:1], probes[:1], "t", answerer=answerer, + judge=_judge("a", lambda p: WO), panel=(broken, _judge("b", lambda p: WO)), + concurrency=1) + base = next(r for r in result.runs if r.arm == BASELINE_ARM) + assert base.verdict == WO + assert base.judge_verdicts["broken"] == "error" + + +def test_every_panel_judge_is_checked_against_the_answerer(): + atoms, probes, answerer = _setup() + with pytest.raises(SelfJudgeError): + run_measurement(atoms, probes, "t", answerer=answerer, judge=_judge("j", lambda p: CC), + panel=(answerer,)) + + +def _args(**kw): + base = dict(judges=None, judge_model=None, judge_provider=None, model="m", provider="litellm", + extract_model=None, extract_provider=None) + base.update(kw) + return argparse.Namespace(**base) + + +def test_judge_flag_parses_provider_model_and_sets_the_extractor(): + args = _args(judges=["openrouter/gpt-6-luna", "openrouter/claude-sonnet-5"]) + assert _judge_refs(args) == [("gpt-6-luna", "openrouter"), ("claude-sonnet-5", "openrouter")] + roles = _role_models(args) + assert roles["judge"] == ("gpt-6-luna", "openrouter") + assert roles["extractor"] == ("gpt-6-luna", "openrouter") + + +@pytest.mark.parametrize("kw, match", [ + ({"judges": ["gpt-6-luna"]}, "PROVIDER/MODEL"), + ({"judges": ["nope/x"]}, "PROVIDER/MODEL"), + ({"judges": ["openrouter/a", "openrouter/a"]}, "twice"), + ({"judges": ["openrouter/a", "openai/a"]}, "twice"), + ({"judges": ["openrouter/a"], "judge_model": "b"}, "one form"), +]) +def test_judge_flag_rejects_bad_refs(kw, match): + with pytest.raises(ValueError, match=match): + _judge_refs(_args(**kw)) + + +def test_two_judges_of_one_vendor_the_vendor_map_lacks_are_one_family(): + grok = ["x-ai/grok-4.6", "x-ai/grok-4"] + assert shared_vendors(grok) == {"xai": grok} + assert shared_vendors(["acme/one", "acme/two"]) == {} # two unrecognised judges share nothing + + +def test_judge_flag_rejects_one_model_under_two_spellings(): + args = _args(judges=["openai/gpt-4o", "openrouter/openai/gpt-4o", "anthropic/claude-sonnet-5"]) + with pytest.raises(ValueError, match="twice"): + _judge_refs(args) + + +def test_run_measurement_rejects_one_model_given_twice_before_any_call(): + atoms, probes, answerer = _setup() + calls = [] + a = _judge("gpt-4o", lambda p: calls.append(p) or CC) + b = _judge("openai/gpt-4o", lambda p: calls.append(p) or CC) + with pytest.raises(InputError, match="same model as another judge"): + run_measurement(atoms, probes, "t", answerer=answerer, judge=a, panel=(b,)) + assert calls == [] + + +def test_judge2_is_checked_against_every_panel_judge(): + _, _, answerer = _setup() + first, second = _judge("j-one", lambda p: CC), _judge("j-two", lambda p: CC) + with pytest.raises(InputError, match="panel judge"): + check_judge2(_judge("j-two", lambda p: CC), answerer, 0.5, False, first, (second,)) + check_judge2(_judge("j-three", lambda p: CC), answerer, 0.5, False, first, (second,)) + + +def _run(atom: str, arm: str, verdicts: dict[str, str]) -> ProbeRun: + return ProbeRun(atom_id=atom, probe_kind="direct", arm=arm, question=f"Q{atom}?", + response="r", verdict=next(iter(verdicts.values())), judge_verdicts=verdicts) + + +def test_added_range_is_taken_on_the_pairs_every_judge_graded(): + err = "error" + runs = [ + _run("a", BASELINE_ARM, {"j1": WO, "j2": WO}), _run("a", "full", {"j1": CC, "j2": err}), + _run("b", BASELINE_ARM, {"j1": CC, "j2": CC}), _run("b", "full", {"j1": CC, "j2": CC}), + ] + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + assert rec["per_judge_paired"]["j1"]["lenient"]["added"] == 0.5 + assert rec["per_judge_paired"]["j2"]["lenient"]["added"] == 0.0 # its own sample lost pair a + assert rec["added_share_range"]["lenient"] == 0.0 + assert rec["judge_sensitive"] is False + + +def test_a_judge_sees_the_veto_its_own_verdict_meets_and_a_missing_vote_is_no_datum(): + veto = [{"role": "veto", "passed": False}] + runs = [ + _run("a", BASELINE_ARM, {"j1": WO, "j2": WO}), + _run("a", "full", {"j1": CC, "j2": CC}), # the veto fails: both judges' CC is wrong_other + _run("b", BASELINE_ARM, {"j1": WO, "j2": WO}), + _run("b", "full", {"j1": CC}), # j2 cast no vote on this answer (a spend stop) + ] + runs[1].checks = veto + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + assert rec["per_judge_paired"]["j1"]["lenient"]["added"] == 0.5 # pair a: the veto makes it redundant + j2 = rec["per_judge_paired"]["j2"]["lenient"] + assert (j2["n_paired"], j2["n_unpaired_dropped"], j2["added"]) == (1, 1, 0.0) + + +def test_panel_warns_for_each_judge_of_the_answerers_vendor_and_for_a_failing_judge(): + err = "error" + runs = [_run("a", BASELINE_ARM, {"claude-opus-5": WO, "gpt-6-luna": err}), + _run("a", "full", {"claude-opus-5": CC, "gpt-6-luna": err})] + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median", + "claude-sonnet-5") + assert rec["same_vendor_as_answerer"] == ["claude-opus-5"] + assert rec["failed_grades"] == {"claude-opus-5": 0, "gpt-6-luna": 2} + warnings = panel_warnings(rec) + assert any("claude-opus-5 is of the answerer's vendor" in w for w in warnings) + assert any("gpt-6-luna failed to grade 2" in w for w in warnings) + + +def test_a_range_of_exactly_the_threshold_is_not_judge_sensitive(): + runs = [] + for i in range(10): # j1 sees one probe gained, j2 none: a range of exactly 0.10 + runs += [_run(str(i), BASELINE_ARM, {"j1": WO, "j2": WO}), + _run(str(i), "full", {"j1": CC if i == 0 else WO, "j2": WO})] + rec = panel_record(MeasureResult(runs=runs, arms=(BASELINE_ARM, "full")), "median") + assert rec["added_share_range"]["lenient"] == 0.1 + assert rec["judge_sensitive"] is False + + +def test_estimate_counts_each_panel_judges_grading_calls(): + from dokimi.novelty.cost import add_panel_calls + + estimate = {"by_role": {"extractor": 3, "answerer": 8, "judge": 8}, "probe_calls": 32, + "total_calls": 40, "total_calls_if_every_pair_escalates": 44, + "expected_calls": 19} + add_panel_calls(_args(judges=["openai/a", "openai/b", "anthropic/c"]), estimate) + assert estimate["by_role"]["judge"] == 24 + assert (estimate["probe_calls"], estimate["total_calls"], estimate["expected_calls"]) == (48, 56, 35) + assert estimate["total_calls_if_every_pair_escalates"] == 60 + single = dict(estimate, by_role=dict(estimate["by_role"])) + add_panel_calls(_args(judges=["openai/a"]), single) + assert single == estimate + + +def test_estimate_only_counts_each_panel_judge_at_the_command(tmp_path, capsys): + from dokimi.cli import build_parser + + (tmp_path / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + def estimate(*judges): + flags = [f for j in judges for f in ("--judge", j)] + args = build_parser().parse_args(["skill", "novelty", "analyze", "--artifact", + f"skill:{tmp_path}", "--estimate-only", *flags]) + args.func(args) + out = json.loads(capsys.readouterr().out) + return dict(out["estimate"], expected_usd=out["expected_usd"]) + one = estimate("openai/gpt-5.6-luna") + three = estimate("openai/gpt-5.6-luna", "anthropic/claude-sonnet-5", "openai/gpt-6-luna") + assert three["by_role"]["judge"] == 3 * one["by_role"]["judge"] + assert three["probe_calls"] > one["probe_calls"] + assert three["expected_usd"] > one["expected_usd"] + + +def test_estimate_only_refuses_an_unusable_judge(tmp_path): + from dokimi.cli import build_parser + + (tmp_path / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + args = build_parser().parse_args(["skill", "novelty", "analyze", "--artifact", + f"skill:{tmp_path}", "--estimate-only", "--judge", "nope/x"]) + with pytest.raises(InputError, match="PROVIDER/MODEL"): + args.func(args) + + +def test_the_report_names_the_panel_and_analyze_saves_each_verdict(tmp_path): + from dokimi.commands.novelty_run import save_panel_verdicts + from dokimi.novelty.report import _judged_by + + runs = [_run("a", BASELINE_ARM, {"j1": WO, "j2": CC}), _run("a", "full", {"j1": CC, "j2": CC})] + result = type("R", (), {"measure_result": MeasureResult(runs=runs)})() + out = tmp_path / "novelty.json" + dest = save_panel_verdicts(result, out) + assert dest == tmp_path / "novelty.judge_verdicts.json" + assert json.loads(dest.read_text())[0]["judge_verdicts"] == {"j1": WO, "j2": CC} + assert "panel of 2 judges" in _judged_by({"judge_model": "j1", "judge_panel": + {"judges": ["j1", "j2"], "rule": "median"}}) + assert _judged_by({"judge_model": "j1"}) == "**j1**" + single = type("R", (), {"measure_result": MeasureResult( + runs=[ProbeRun("a", "direct", "full", "q", "r", WO)])})() + assert save_panel_verdicts(single, out) is None + assert not dest.exists() # a one-judge run into the same output does not leave the panel's file + + +def _capped_judge(model: str, verdict: str, *, priced: bool, truncated: bool = False): + from dokimi.metrics.usage import UsageRecord + + def transport(prompt, system): + usage = UsageRecord(cost_usd=1.0 if priced else None, truncated=truncated) + return json.dumps({"verdict": verdict}), usage + + from dokimi.novelty.client import SpendCap + return LLMClient(model=model, transport=transport, cap=SpendCap(limit_usd=100.0)) + + +def _one_probe(): + atoms, probes, answerer = _setup() + return atoms[:1], probes[:1], answerer + + +def test_an_unpriced_judge_stopping_a_panel_does_not_decide_the_verdict(): + atoms, probes, answerer = _one_probe() + first = _capped_judge("j1", WO, priced=True) + second = _capped_judge("j2", CC, priced=False) + result = run_measurement(atoms, probes, "t", answerer=answerer, judge=first, panel=(second,), + aggregate="median", concurrency=1) + stopped = [r for r in result.runs if r.judge_verdicts and "j2" not in r.judge_verdicts] + assert stopped and all(r.verdict == "error" for r in stopped) + assert result.ungraded == len(stopped) + assert result.paired("lenient")["added"] == 0.0 + + +def test_a_truncated_judge_reply_under_a_cap_is_a_failed_grade(): + atoms, probes, answerer = _one_probe() + judge = _capped_judge("j", CC, priced=False, truncated=True) + result = run_measurement(atoms, probes, "t", answerer=answerer, judge=judge, concurrency=1) + assert all(r.verdict == "error" for r in result.runs) + assert result.ungraded == len(result.runs) + + +def test_analyze_refuses_a_panel_member_that_answers_before_any_paid_call(tmp_path, monkeypatch): + from dokimi.cli import build_parser + from dokimi.novelty import cost + from dokimi.novelty.client import LLMClient + + (tmp_path / "SKILL.md").write_text("---\nname: s\ndescription: d\n---\n\n# S\n\nText.\n", + encoding="utf-8") + base = ["skill", "novelty", "analyze", "--artifact", f"skill:{tmp_path}"] + answer_model, answer_provider = cost.role_models(build_parser().parse_args(base))["answerer"] + calls: list[str] = [] + + def transport(prompt, system): + calls.append(prompt) # any call here is a paid one that the refusal should have stopped + raise AssertionError("a paid call happened before the self-judge refusal") + + monkeypatch.setattr(cost, "client", + lambda model, provider, role, args: LLMClient(model=model, transport=transport)) + args = build_parser().parse_args( + [*base, "--judge", f"{answer_provider}/{answer_model}", "--judge", "openai/gpt-6-luna"]) + with pytest.raises(InputError, match=answer_model): + args.func(args) + assert calls == [] + + +@pytest.mark.parametrize("sub", ["probe", "analyze"]) +def test_the_answerer_reaches_the_panel_record_so_the_vendor_warning_shows( + sub, tmp_path, monkeypatch, capsys): + from dokimi.cli import build_parser + from dokimi.novelty import cost + + monkeypatch.chdir(tmp_path) # analyze also writes its report under ./results + skill = tmp_path / "s" / "SKILL.md" + skill.parent.mkdir(parents=True) + skill.write_text("---\nname: s\ndescription: d\n---\n\n# H\n\nDo not enable the legacy driver.\n", + encoding="utf-8") + rows = [{"statement": "The legacy driver must not be enabled.", "question": "Enable it?", + "answer": "No.", "kind": "contract", "tags": [], "span_ids": ["s0002"]}] + + def grader(prompt, system=None): + if "Extract the checkable knowledge" in prompt: + return json.dumps(rows), None + return json.dumps({"verdict": CC}), None + + def answer(prompt, system=None): + return "No.\nconfidence: 0.5", None + + monkeypatch.setattr(cost, "client", lambda model, provider, role, args: LLMClient( + model=model, transport=answer if role == "answerer" else grader)) + atoms, out = tmp_path / "atoms.json", tmp_path / "p.json" + run = [sub, "--out", str(out), "--no-paraphrase", "--no-reversal", "--model", "claude-sonnet-5", + "--judge", "anthropic/claude-opus-5", "--judge", "openai/gpt-6-luna"] + for argv in ([["atoms", "--out", str(atoms)], [*run, "--atoms", str(atoms)]] if sub == "probe" + else [run]): + args = build_parser().parse_args(["skill", "novelty", argv[0], "--artifact", f"skill:{skill}", + *argv[1:]]) + args.func(args) + assert json.loads(out.read_text())["judge_panel"]["same_vendor_as_answerer"] == ["claude-opus-5"] + assert "claude-opus-5" in capsys.readouterr().err