diff --git a/app/analysis.js b/app/analysis.js index b98bced..6cf082e 100644 --- a/app/analysis.js +++ b/app/analysis.js @@ -241,19 +241,38 @@ function sameCoverage(a,b,config=null,matching='strict'){return a.length===b.len // Public, serializable analysis boundary used by Python parity tests and the UI. function measurementId(r){return JSON.stringify([r.checkpoint,...['task','metric','filter','n_shot','harness','backend'].map(k=>r[k])]);} const diagnosticTitles={config_caveat:'Config caveat',no_config:'No config',not_used:'Not used',unknown_language:'Unknown language',invalid_sample_count:'Invalid sample count',strict_shot_setting:'Few-shot mismatch excluded',relaxed_shot_setting:'Few-shot mismatch allowed',ambiguous_shot_setting:'Ambiguous few-shot setting',inconsistent_scoring_settings:'Inconsistent scoring settings',missing_scoring_field:'Missing scoring field',missing_scoring_setting:'Missing scoring setting',no_selected_score:'No selected score',missing_suite_data:'Missing suite data',incomplete_components:'Incomplete components',comparison_coverage:'Comparison coverage',sample_count_mismatch:'Sample-count mismatch',no_category_weight:'No category weight'}; -function diagnostic(code,model,evalName,rows,detail,effect='included',tasks=null){ +diagnosticTitles.intentional_exclusion='Intentional exclusions'; +function diagnostic(code,model,evalName,rows,detail,effect='included',tasks=null,severity='warning'){ const names=[...new Set(tasks??rows.map(r=>r.task))].sort(compareText); - return {code,type:diagnosticTitles[code],model,eval:evalName,name:evalName||names[0]||model,tasks:names,measurement_ids:rows.map(measurementId).sort(compareText),effect,detail,variants:names.length?[{settings:detail,tasks:names}]:[]}; + return {code,severity,type:diagnosticTitles[code],model,eval:evalName,name:evalName||names[0]||model,tasks:names,measurement_ids:rows.map(measurementId).sort(compareText),effect,detail,variants:names.length?[{settings:detail,tasks:names}]:[]}; } function reportDiagnostics(audits,config,included,comparison=false,matchingMode='strict',resolved=null){ const {profile,catalogue,suite,scheme}=resolved||resolveInputs(config.catalogue,config.suite,config.profile),out=[]; + const exclusions=new Map(),metadata=new Map(catalogue.languages.flatMap(g=>g.tasks.map(t=>[t,g]))),localLanguages=new Map((suite.evals||[]).map(e=>[e.name,e.exclude_languages||[]])); + function recordExclusion(row){ + const reasons=[]; + if((suite.exclude||[]).includes(row.eval))reasons.push(['eval',row.eval,'']); + else{ + const group=metadata.get(row.task)||{},codes=new Set(['language','source_language','target_language'].map(k=>group[k]).filter(Boolean)); + for(const language of codes){ + if((suite.exclude_languages||[]).includes(language))reasons.push(['language',language,'']); + else if((localLanguages.get(row.eval)||[]).includes(language))reasons.push(['language',language,row.eval]); + } + } + for(const [kind,name,scope] of reasons){ + const key=JSON.stringify([kind,name,scope]); + if(!exclusions.has(key))exclusions.set(key,{kind,name,scope,records:new Map()}); + exclusions.get(key).records.set(measurementId(row),row); + } + return reasons.length>0; + } const used=new Set([...included.values()].flat().map(r=>r.eval)); for(const e of catalogue.evals)if(e.warning&&used.has(e.name))out.push(diagnostic('config_caveat','Selected comparison',e.name,[],e.warning)); for(const [model,rows] of audits){ const scope=scopeRows(rows,suite),explainedShots=new Set(),accepted=included.get(model)||[],acceptedIds=new Set(accepted.map(measurementId)); for(const task of [...new Set(rows.filter(r=>!r.eval).map(r=>r.task))].sort(compareText))out.push(diagnostic('no_config',model,null,rows.filter(r=>r.task===task),'No eval configuration; excluded from scoring.','excluded')); for(const e of catalogue.evals){ - const all=rows.filter(r=>r.eval===e.name),outside=all.filter(r=>(!e.select||matchTask(e.select,r.task))&&!inSuite(r,suite)),matching=all.filter(r=>inSuite(r,suite)),selected=matching.filter(r=>r.selected); + const all=rows.filter(r=>r.eval===e.name),outside=all.filter(r=>(!e.select||matchTask(e.select,r.task))&&!inSuite(r,suite)).filter(r=>!recordExclusion(r)),matching=all.filter(r=>inSuite(r,suite)),selected=matching.filter(r=>r.selected); const add=(code,rr,detail,effect='included',tasks=null)=>out.push(diagnostic(code,model,e.name,rr,detail,effect,tasks)); if(matchingMode==='relaxed'&&'shots'in e){ const actuals=[...new Set(selected.filter(r=>acceptedIds.has(measurementId(r))&&Number(r.n_shot)!==e.shots).map(r=>Number(r.n_shot)))].sort((a,b)=>a-b); @@ -295,6 +314,17 @@ function reportDiagnostics(audits,config,included,comparison=false,matchingMode= if(comparison){const match=r=>matchingKey(r,scheme,matchingMode),entries=[...included],a=entries[0]?.[1]||[],b=entries[1]?.[1]||a,bm=new Map(b.map(r=>[match(r),r])); for(const e of catalogue.evals){const rr=a.filter(r=>r.eval===e.name&&bm.has(match(r))&&sampleCount(r)!==null&&sampleCount(bm.get(match(r)))!==null&&sampleCount(r)!==sampleCount(bm.get(match(r))));if(rr.length)out.push(diagnostic('sample_count_mismatch','Selected comparison',e.name,rr.concat(rr.map(r=>bm.get(match(r)))),'Matched results have different sample counts. Scores remain included.'));} } + if(exclusions.size){ + const groups=[],affected=new Map(),variants=[],labels=[]; + for(const {kind,name,scope,records} of [...exclusions.values()].sort((a,b)=>compareText(a.kind,b.kind)||compareText(a.name,b.name)||compareText(a.scope,b.scope))){ + for(const [id,row] of records)affected.set(id,row); + const rows=[...records.values()],tasks=[...new Set(rows.map(r=>r.task))].sort(compareText),models=[...new Set(rows.map(r=>r.checkpoint))].sort(compareText); + groups.push({kind,name,eval:scope||null,models,tasks,measurement_ids:[...records.keys()].sort(compareText)}); + const label=(kind==='eval'?'Eval ':'Language ')+name+(kind==='language'?(scope?' in '+scope:' across the set'):''); + labels.push(label);variants.push({settings:label+' · '+models.join(', '),tasks}); + } + out.push({...diagnostic('intentional_exclusion','Selected models',null,[...affected.values()],'Intentionally excluded by '+suite.name+': '+labels.join('; ')+'.','excluded',null,'info'),name:suite.name,exclusions:groups,variants}); + } return out; } function allocationTree(rows,config,allocation,model){ diff --git a/app/app.js b/app/app.js index bbef8e2..60ba05c 100644 --- a/app/app.js +++ b/app/app.js @@ -104,7 +104,7 @@ function start(){ } function activeWarnings(){ if(!models.size)return []; - const coverage=selected(),warnings=coverage.result.diagnostics.filter(w=>(!isDemoModel(w.model)||w.code==='relaxed_shot_setting')&&(w.code!=='no_category_weight'||weights[w.category]===0)); + const coverage=selected(),warnings=coverage.result.diagnostics.filter(w=>w.severity==='warning'&&(!isDemoModel(w.model)||w.code==='relaxed_shot_setting')&&(w.code!=='no_category_weight'||weights[w.category]===0)); if(state.aggregate!=='standard')for(const c of totals(coverage.a,scheme,weights,state.aggregate,englishWeights,metadata).categories)if(c.issue)warnings.push({type:'English split unavailable',name:c.name,model:'Selected comparison',detail:c.issue}); return warnings; } @@ -194,7 +194,11 @@ function start(){ const total=a.components.reduce((sum,c)=>sum+c.relative_weight,0); return '
Component aggregation

group score = sum(relative_weight × normalized score) / '+esc(total)+'

'+table(['Component','Task match','Relative weight','Share of group'],a.components.map(c=>''+td(c.name)+td(c.match.name??c.match.regex)+num(c.relative_weight,3)+''+fmt(c.relative_weight/total*100,2)+'%'),[2,3])+'

All components are required within the same language and scoring protocol. Missing, ambiguous, or duplicate components exclude that group from both comparison scores. Complete groups are averaged within the eval, with the selected English balance applied afterward.

'+(a.note?'

'+esc(a.note)+'

':'')+(a.sources?.length?'

'+a.sources.map((u,i)=>'Aggregation source '+(i+1)+'').join(' · ')+'

':'')+'
'; } - function renderWarnings(){const warnings=activeWarnings();return '

Warnings

Coverage for the selected comparison, scoring consistency, and config caveats. A named eval set checks required measurements; unused global catalogue rules are allowed.

'+(warnings.length?table(['Warning','Eval / task','Model','Details'],warnings.map(w=>''+td(w.type)+td(w.name)+td(w.model)+''+esc(w.detail)+(w.variants?'
All affected variants'+w.variants.map(v=>'

'+esc(v.settings)+'
'+v.tasks.map(esc).join('
')+'

').join('')+'
':'')+'')):'

No warnings. The selected comparison has no detected data or configuration issues.

');} + function diagnosticTable(items,label){return table([label,'Eval / scope','Model','Details'],items.map(w=>''+td(w.type)+td(w.name)+td(w.model)+''+esc(w.detail)+(w.variants?.length?'
All affected variants'+w.variants.map(v=>'

'+esc(v.settings)+'
'+v.tasks.map(esc).join('
')+'

').join('')+'
':'')+''));} + function renderWarnings(){ + const warnings=activeWarnings(),information=models.size?selected().result.diagnostics.filter(d=>d.severity==='info'):[]; + return '

Warnings

Coverage for the selected comparison, scoring consistency, and config caveats. Intentional exclusions are listed separately as information.

'+(warnings.length?diagnosticTable(warnings,'Warning'):'

No warnings. The selected comparison has no detected data or configuration issues.

')+(information.length?'

Information

Explicit exclusions do not count as warnings. Their data remains available in Eval configuration.

'+diagnosticTable(information,'Information')+'
':''); + } function scoringOptions(e,rows){ const used=rows.filter(r=>r.selected),values=(rr,key)=>[...new Set(rr.map(r=>String(r[key])))].sort((a,b)=>key==='n_shot'?Number(a)-Number(b):a.localeCompare(b)); const shots=values(used,'n_shot'),otherMetrics=values(rows,'metric').filter(m=>m!==e.metric),otherShots=values(rows,'n_shot').filter(n=>!shots.includes(n)),otherFilters=values(rows,'filter').filter(f=>f!==e.metric_filter); diff --git a/app/suite_config.js b/app/suite_config.js index f83ee98..b65aaf4 100644 --- a/app/suite_config.js +++ b/app/suite_config.js @@ -12,7 +12,7 @@ const SuiteConfig=(()=>{ if(s.version!==1||!text(s.name))throw Error('Suite requires version 1 and a name'); if(!['available','fixed'].includes(s.mode))throw Error('Suite mode must be available or fixed'); if('notes'in s&&(!Array.isArray(s.notes)||s.notes.some(n=>typeof n!=='string')))throw Error('Suite notes must be strings'); - if('exclude'in s&&(s.mode!=='available'||!Array.isArray(s.exclude)||s.exclude.some(n=>!text(n))||new Set(s.exclude).size!==s.exclude.length))throw Error('exclude must be a unique list of eval names in available mode'); + if('exclude'in s&&(!Array.isArray(s.exclude)||s.exclude.some(n=>!text(n))||new Set(s.exclude).size!==s.exclude.length))throw Error('exclude must be a unique list of eval names'); validateLanguageExclusions(s); if(s.mode==='available'){if('evals'in s)throw Error('Available mode does not declare required evals');return s;} if(!Array.isArray(s.evals)||!s.evals.length)throw Error('Fixed suite needs required evals'); @@ -23,6 +23,7 @@ const SuiteConfig=(()=>{ if('metric_filter'in e&&typeof e.metric_filter!=='string')throw Error('metric_filter must be text'); if('shots'in e&&(!Number.isSafeInteger(e.shots)||e.shots<0))throw Error('shots must be a nonnegative integer'); if(!text(e.name)||names.has(e.name))throw Error('Suite eval names must be unique');names.add(e.name); + if((s.exclude||[]).includes(e.name))throw Error('Eval cannot be both required and excluded: '+e.name); if(!('variants'in e))continue; if(!Array.isArray(e.variants)||!e.variants.length)throw Error('Required variants must be a nonempty list'); const seen=new Map(); @@ -71,8 +72,8 @@ const SuiteConfig=(()=>{ return new Set(tasks.filter(t=>codes(t).some(c=>languages.includes(c)))); } const removed=excluded(suite.exclude_languages||[],[...new Set([...taskSets.values()].flat())],'catalogue'),result=structuredClone(suite); + for(const name of suite.exclude||[])if(!byName.has(name))throw Error('Excluded eval has no catalogue rule: '+name); if(suite.mode==='available'){ - for(const name of suite.exclude||[])if(!byName.has(name))throw Error('Excluded eval has no catalogue rule: '+name); result._excluded_tasks=[...removed].sort(api.compareText);return result; } for(const required of result.evals){ diff --git a/configs/README.md b/configs/README.md index 752e2bc..0ef6cb3 100644 --- a/configs/README.md +++ b/configs/README.md @@ -27,6 +27,8 @@ python3 -m app.build --results-dir results --sample-csv examples/sample-evals.cs The build validates every offered combination before replacing output. In the dashboard, inspect **Warnings** for the comparison you intend to use. Named-set scores with missing requirements are explicitly incomplete; extras are excluded but remain inspectable. +Record intentional eval exclusions in the set's `exclude` list and explain each eval/language exclusion with an inline YAML comment. Explicit exclusions appear together under **Information**, outside the warning count. Keep the authored YAML in version control because exported YAML does not preserve comments. + The generated `output/shared/catalogue.yaml` contains the complete catalogue, including every language assignment, with no file references. Browser catalogue export produces the same portable format. Import that complete file when moving settings between dashboards; the repository manifest alone is not a browser import. For temporary changes, use the separate load/export controls under **Eval configuration**. Files stay in the browser. Weight exports save edited weights and the active calculation; catalogue and eval-set exports are independent. None of these controls modify the repository. diff --git a/configs/sets/any-available.yaml b/configs/sets/any-available.yaml index 34e877a..da41051 100644 --- a/configs/sets/any-available.yaml +++ b/configs/sets/any-available.yaml @@ -2,6 +2,6 @@ version: 1 name: Any available mode: available exclude: - - global_piqa_prompted + - global_piqa_prompted # Scoring and normalization await validation. notes: - Compare recognized measurements shared by both models; warn about differences. diff --git a/configs/sets/flagship-1.yaml b/configs/sets/flagship-1.yaml index 4278395..2b0e2a5 100644 --- a/configs/sets/flagship-1.yaml +++ b/configs/sets/flagship-1.yaml @@ -1,11 +1,25 @@ version: 1 name: flagship-1 mode: fixed -exclude_languages: [kat_Geor] +exclude_languages: + - kat_Geor # Not targeted or evaluated for this model generation because Georgian was omitted from tokenizer training. +exclude: + - AIME24 # Replaced by aime24_cot with corrected CoT evaluation settings. + - AIME25 # Replaced by aime25_cot with corrected CoT evaluation settings. + - AMC23 # Replaced by amc23_cot with corrected CoT evaluation settings. + - GPQADiamond # Replaced by gpqa_diamond_cot with corrected CoT evaluation settings. + - JEEBench # Replaced by jeebench_cot with corrected CoT evaluation settings. + - MATH500 # Replaced by math500_cot with corrected CoT evaluation settings. + - polymath # Replaced by polymath_cot; all four corrected difficulty levels are required. + - HumanEval # Replaced by humaneval_cont with the corrected code-continuation protocol. + - LiveCodeBench # Replaced by livecodebench_cont with the corrected code-continuation protocol. + - mbpp # Replaced by mbpp_cont with the corrected code-continuation protocol. + - global_piqa_prompted # Scoring and normalization await validation; use completion-based global_piqa. notes: - Use corrected CoT reasoning and code-continuation protocols; original-protocol results are excluded without fallback. - Require the catalogue tasks for each listed eval, inheriting its scoring defaults. - - Exclude Georgian results because of a vocabulary error. Exclude English X-CSQA to avoid overlap with CommonsenseQA. + - The omission of Georgian from tokenizer training was discovered too late to correct for this model generation, so Georgian is not targeted or evaluated. + - Exclude English X-CSQA to avoid overlap with CommonsenseQA. evals: - name: bigbench_cs_algorithms - name: humaneval_cont @@ -42,7 +56,7 @@ evals: - name: winogrande - name: xcsqa exclude_languages: - - eng_Latn + - eng_Latn # Avoid overlap with CommonsenseQA. - name: xcopa - name: belebele - name: boolq diff --git a/docs/configuration.md b/docs/configuration.md index 9460000..53b3178 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -29,9 +29,9 @@ Both profiles default to the standard calculation and store an English share of **flagship-1** is the startup eval set. The **Weighting profile** and **Eval set** selectors operate independently. Switching a profile resets category weights, English shares, and the calculation to that profile's values, leaving the eval set unchanged. Switching eval sets preserves your current weights and calculation. Export edits before switching profiles if you want to keep them. -**Any available** uses recognized selected measurements shared by A and B. Measurements present on only one side generate comparison warnings and are excluded from both scores. Catalogue entries absent from both models do not generate warnings. The supplied freeform set explicitly excludes prompted Global PIQA pending validation; present data for it generates a **Not used** warning. +**Any available** uses recognized selected measurements shared by A and B. Measurements present on only one side generate comparison warnings and are excluded from both scores. Catalogue entries absent from both models do not generate warnings. The supplied freeform set explicitly excludes prompted Global PIQA pending validation; present data for it appears in the **Intentional exclusions** informational summary. -**flagship-1** requires the catalogue's known tasks for each listed eval, excluding Georgian throughout the set because of a vocabulary error and English specifically for X-CSQA. Original and translated tasks stay under their existing eval group (for example `arc_challenge`). A missing required result generates a warning even if other languages for that eval are present. The score is labelled **INCOMPLETE** and uses the shared subset with redistributed weights. Excluded tasks are not requirements; present excluded data produces **Not used** warnings and remains inspectable. +**flagship-1** requires the catalogue's known tasks for each listed eval. Georgian is excluded throughout the set because it was omitted from tokenizer training, and this was discovered too late to correct for this model generation. Georgian is therefore neither targeted nor evaluated for this generation. English is excluded specifically for X-CSQA to avoid overlap with CommonsenseQA. Original and translated tasks stay under their existing eval group (for example `arc_challenge`). A missing required result generates a warning even if other languages for that eval are present. The score is labelled **INCOMPLETE** and uses the shared subset with redistributed weights. Excluded tasks are not requirements; present explicitly excluded data appears in one informational summary and remains inspectable. The set uses the corrected CoT reasoning and code-continuation protocols from the `flag-evals-471` exports. Each has its own catalogue entry, so the original and corrected runs retain distinct task identities: @@ -79,10 +79,29 @@ Available-mode sets can exclude entire evals or languages without creating requi version: 1 name: Any available mode: available -exclude: [global_piqa_prompted] +exclude: + - global_piqa_prompted # Scoring and normalization await validation. ``` -`exclude` is allowed only in available mode and contains unique, exact catalogue eval names. All names must exist even if no corresponding model results are loaded. Fixed sets exclude unlisted evals automatically. Exclusions warn when eligible task data is present, including rows with only the wrong metric. Alternate metric rows or summary children of a selected eval do not warn merely because another field or summary was selected. +`exclude` works in both modes and contains unique, exact catalogue eval names. All names must exist even if no corresponding model results are loaded. An eval cannot be both required under `evals` and listed in `exclude`. Fixed sets exclude unlisted evals automatically; listing an intentional exclusion explicitly explains why its presence should not warn. + +Document each exclusion with an inline YAML comment explaining the replacement or known issue. For example: + +```yaml +version: 1 +name: Corrected AIME +mode: fixed +evals: + - name: aime25_cot +exclude: + - AIME25 # Replaced by aime25_cot with corrected CoT evaluation settings. +``` + +An excluded eval is outside this set even if its results arrive later. To require it again, remove it from `exclude` and add it to `evals`. An intentionally missing eval therefore does not become an unsatisfied requirement; genuinely missing required evals still warn and make coverage incomplete. This mechanism does not waive missing components or protocols of an included eval. + +Explicit eval and language exclusions produce one **Intentional exclusions** entry under **Information** in the Warnings view when eligible data is present, including rows with only the wrong metric. It combines the affected models and evals, with expandable task lists for each exclusion. It does not increase the warning badge, emit Python warnings, or fail strict diagnostic checks. If no excluded data is present, no informational entry is needed. Unlisted evals and unselected variants without an explicit exclusion still produce **Not used** warnings. Alternate metric rows or summary children of a selected eval do not warn merely because another field or summary was selected. + +YAML comments document the source file; the parsers and browser exports do not preserve comments. Exclusion names survive import/export. Keep the authored YAML in version control as the record of why each exclusion exists. For an explicitly pinned task inventory, a fixed entry may still use `variants: [{task: example_en}]`, optionally with a hard `n_shot` requirement. Each task must be known and selected by its catalogue rule. These are membership constraints, not setting overrides; a pinned `n_shot` must agree with the effective eval settings and is not loosened by relaxed matching. Ordinary named sets need no `variants` list. @@ -412,7 +431,7 @@ The referenced [OpenSubtitles task](https://github.com/OpenEuroLLM/oellm-eval/bl Both translation evals retain native 0–100 points (`score.scale: 100`, `normalize: {min: 0, max: 1}`), without chance correction. This accepted policy is documented in their notes rather than flagged as an unresolved caveat. A shared numerical range does not imply equal difficulty across metrics or language pairs. -Completion-based PIQA retains `acc_norm` and a 0.5 baseline. **Global PIQA (prompted)** has a separate interpretation rule for `exact_match` / `strict_match`, a provisional zero floor, and a warning that normalization and metric selection have not been validated. It is excluded by the supplied Any available set and omitted from flagship-1. Its data generates a **Not used** notice while excluded. Including it in a custom set surfaces its scoring caveat; configuration inspection always shows the attached warning. +Completion-based PIQA retains `acc_norm` and a 0.5 baseline. **Global PIQA (prompted)** has a separate interpretation rule for `exact_match` / `strict_match`, a provisional zero floor, and a warning that normalization and metric selection have not been validated. It is excluded by the supplied Any available set and omitted from flagship-1. Its data appears in the **Intentional exclusions** informational summary while explicitly excluded. Including it in a custom set surfaces its scoring caveat; configuration inspection always shows the attached warning. With these defaults, MultiBlimp's Croatian/Serbian pooling is the only configured caveat for included evals. Runtime warnings for coverage, inconsistent settings, missing fields, and unused data still apply. @@ -485,7 +504,8 @@ Builds and browser imports use the same CSV parser. Required columns are `checkp | Task has no eval config, or lacks the configured metric/filter/shots | Warn and exclude from scoring. Alternate metrics remain inspectable and never silently substitute for the configured metric. | | Measurements match only one of the compared models | Warn; use only shared measurements and redistribute weights. | | Named set requirement is missing from either or both models | Warn and mark the set incomplete; compare the shared subset. | -| Eval/task data is not selected by a named set or freeform exclusion | Show **Not used** and exclude it, even if only an alternate metric exists; keep it in the audit. | +| Eval/task data is covered by an explicit eval or language exclusion | Combine it into one **Intentional exclusions** informational entry across models and evals; exclude it from scoring and retain it in the audit. | +| Eval/task data is outside the selected set without an explicit exclusion | Show **Not used**, even if only an alternate metric exists; keep it in the audit. | | Catalogue rule has no results in either model | No warning unless required by the selected named set. | | Shared category has no profile weight | Warn; zero contribution until a weight is assigned. | | Selected task has no explicit language assignment | For component evals, warn and exclude the group. Otherwise warn and retain the score. Language views show Unknown; English-balance modes use the English fallback. Known mixed-language pools use the documented fallback without claiming a resolved single language. | @@ -494,7 +514,7 @@ Builds and browser imports use the same CSV parser. Required columns are `checkp | Valid component selection has missing data or multiple exported results for a component | Warn and exclude the whole language/protocol group from both scores. Keep raw data inspectable; never average only the remaining components. | | Matched A/B measurements report different positive `n_samples` | Warn and retain scores; sample count does not determine score weights. Review whether dataset coverage is comparable. | | Supplied `n_samples` is not a positive integer | Warn and retain scores; omit it from sample-count comparisons. Absent/blank sample counts are allowed. | -| Eval has a YAML `warning` | Show the caveat in Warnings when the eval has shared comparison data. Always show it in its configuration details. Excluded evals get **Not used**, not an active scoring caveat. | +| Eval has a YAML `warning` | Show the caveat in Warnings when the eval has shared comparison data. Always show it in its configuration details. Explicitly excluded evals appear under **Information**; other unselected evals get **Not used**. Neither produces an active scoring caveat. | | No shared data with positive category weight | Show an unavailable composite (`—`), never an invented zero. | Invalid model/config imports leave the active models, settings, and scores unchanged, including multi-model files where a later model is invalid. Build input validation completes before existing output files are replaced. Warnings are calculated from the current config and loaded results; fixing or removing the underlying issue removes its warning. Fields not used for scoring, such as source paths and standard errors, remain audit information; their presence is not a guarantee that dataset revisions or prompts match. Invalid values in excluded alternate metrics remain visible but are not normalized using the selected metric's scale. diff --git a/docs/python-api.md b/docs/python-api.md index 19e4c21..6cdd931 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -124,7 +124,7 @@ requirements or exclusions; missing data for a valid requirement warns instead. ## Surface warnings -By default, `analyze()` and `compare()` emit a `QuickdashWarning` through Python's standard `warnings` mechanism for each grouped diagnostic. Warnings normally appear on stderr. Each warning object has a `.diagnostic` attribute containing its structured record. The same diagnostics are retained in `report.diagnostics`, separately from the tree. +By default, `analyze()` and `compare()` emit a `QuickdashWarning` through Python's standard `warnings` mechanism for each grouped diagnostic with `severity: "warning"`. Informational diagnostics remain in the report without emitting a Python warning. Warnings normally appear on stderr. Each warning object has a `.diagnostic` attribute containing its structured record. The same diagnostics are retained in `report.diagnostics`, separately from the tree. Applications that display diagnostics themselves can explicitly collect them: @@ -134,14 +134,15 @@ for diagnostic in report.diagnostics: print(diagnostic["code"], diagnostic["model"], diagnostic["tasks"]) ``` -For strict automation, use `diagnostics="error"`. If any diagnostic occurs, it raises `DiagnosticError`, whose `.diagnostics` contains all warnings; it returns no report. Invalid configurations, ambiguous selected matches, duplicate selected measurements, and invalid selected scores raise `ValueError` regardless of diagnostic policy. File access failures raise the usual `OSError` subclasses. +For strict automation, use `diagnostics="error"`. If any warning occurs, it raises `DiagnosticError`, whose `.diagnostics` contains only warnings; it returns no report. Invalid configurations, ambiguous selected matches, duplicate selected measurements, and invalid selected scores raise `ValueError` regardless of diagnostic policy. File access failures raise the usual `OSError` subclasses. -The diagnostic contract is `code`, `model`, `eval`, `tasks`, `measurement_ids`, and `effect`. `effect` describes the warning's policy (`excluded`, `included`, or `zero_weight`); the measurement's `included` field is authoritative when several conditions apply. Config-wide caveats have no individual measurement IDs. `type`, `name`, `detail`, and `variants` support dashboard presentation and are not wording contracts. +The diagnostic contract is `code`, `severity` (`warning` or `info`), `model`, `eval`, `tasks`, `measurement_ids`, and `effect`. `effect` describes the condition's policy (`excluded`, `included`, or `zero_weight`); the measurement's `included` field is authoritative when several conditions apply. Config-wide caveats have no individual measurement IDs. `type`, `name`, `detail`, and `variants` support dashboard presentation and are not wording contracts. | Codes | Condition and action | | --- | --- | | `config_caveat` | An included eval has a configured warning. | -| `no_config`, `not_used` | Unknown eval or data outside the selected set; excluded. | +| `no_config`, `not_used` | Unknown eval or data outside the selected set without an explicit exclusion; excluded. | +| `intentional_exclusion` | One informational record for data explicitly excluded by eval or language, across all affected models. | | `missing_scoring_field`, `missing_scoring_setting`, `no_selected_score` | Required metric/protocol unavailable; no alternate substitution. | | `missing_suite_data` | A named set has missing requirements; available shared results are reweighted. | | `strict_shot_setting` | Tasks excluded for a shot mismatch, grouped by eval/model/expected/actual count. Includes expected/actual counts and all affected task/measurement IDs. | @@ -154,6 +155,8 @@ The diagnostic contract is `code`, `model`, `eval`, `tasks`, `measurement_ids`, | `invalid_sample_count`, `sample_count_mismatch` | Sample-count metadata needs review; it does not determine weights. | | `no_category_weight` | An included category has no profile weight and contributes zero. | +`intentional_exclusion` has `severity: "info"` and an `exclusions` list. Each group records `kind` (`eval` or `language`), `name`, `eval` (the scope for a local language exclusion, otherwise null), `models`, `tasks`, and `measurement_ids`. The top-level measurement list deduplicates data covered by multiple exclusions. An explicitly excluded eval is not a required result. Missing data for included evals still warns. + Strict shot mismatches produce one diagnostic per eval, model, and expected/actual shot-count pair. Covered tasks do not also generate `missing_scoring_setting`, `no_selected_score`, or `missing_suite_data` diagnostics for the same cause. Coverage still records those unsatisfied requirements and remains incomplete. Genuinely absent tasks and other missing settings retain their diagnostics. Unused catalogue entries do not warn merely because no data exists for them. Declare an expected eval set when absence should warn. Alternate metrics remain auditable without creating warnings when the selected metric is present. @@ -171,6 +174,6 @@ quickdash examples/scores.csv \ Add `--compare 'Example A' 'Example B'` for shared-coverage scores, or `--format json` for the complete machine-readable report. `python -m quickdash` provides the same command. Warnings go to stderr; stdout contains only the selected result format. -Exit status is 0 for a successful calculation, including recoverable warnings. `--strict` prints warnings and exits 1 without writing a result. Invalid input exits 2. A score can be unavailable when no valid weighted data remains; inspect diagnostics and coverage or use strict mode when warnings must block a workflow. +Exit status is 0 for a successful calculation, including recoverable warnings. `--strict` prints diagnostics and exits 1 without writing a result when warnings exist. Informational entries print as `info [...]` on stderr and do not fail strict mode. Invalid input exits 2. A score can be unavailable when no valid weighted data remains; inspect diagnostics and coverage or use strict mode when warnings must block a workflow. -Use `--matching relaxed` to allow few-shot differences. This is separate from `--strict`, which makes the CLI fail on diagnostics; it does not choose the matching mode. +Use `--matching relaxed` to allow few-shot differences. This is separate from `--strict`, which makes the CLI fail on warnings; it does not choose the matching mode. diff --git a/quickdash/analysis.py b/quickdash/analysis.py index 739bc27..aa98e68 100644 --- a/quickdash/analysis.py +++ b/quickdash/analysis.py @@ -119,7 +119,7 @@ def __init__(self, diagnostic): class DiagnosticError(ValueError): - """Strict mode rejected the result. All diagnostics remain inspectable.""" + """Strict mode rejected the result; warning diagnostics remain inspectable.""" def __init__(self, diagnostics): self.diagnostics = diagnostics @@ -129,10 +129,11 @@ def __init__(self, diagnostics): def finish(report, policy): if policy not in ("warn", "collect", "error"): raise ValueError("diagnostics must be warn, collect, or error") - if policy == "error" and report["diagnostics"]: - raise DiagnosticError(report["diagnostics"]) + actionable = [d for d in report["diagnostics"] if d["severity"] == "warning"] + if policy == "error" and actionable: + raise DiagnosticError(actionable) if policy == "warn": - for diagnostic in report["diagnostics"]: + for diagnostic in actionable: warnings.warn(QuickdashWarning(diagnostic), stacklevel=3) return Report(report) @@ -419,6 +420,7 @@ def protocol_inconsistent(rows): TITLES = dict( + intentional_exclusion="Intentional exclusions", config_caveat="Config caveat", no_config="No config", not_used="Not used", @@ -439,10 +441,11 @@ def protocol_inconsistent(rows): ) -def diagnostic(code, model, eval_name, rows, detail, effect="included", tasks=None): +def diagnostic(code, model, eval_name, rows, detail, effect="included", tasks=None, severity="warning"): names = sorted(set(tasks if tasks is not None else [r["task"] for r in rows])) return dict( code=code, + severity=severity, type=TITLES[code], model=model, eval=eval_name, @@ -465,6 +468,26 @@ def report_diagnostics( resolved[k] for k in ("catalogue", "suite", "scheme", "profile") ) out = [] + exclusions = {} + metadata = {t: g for g in catalogue["languages"] for t in g["tasks"]} + local_languages = {e["name"]: e.get("exclude_languages", []) for e in suite.get("evals", [])} + + def record_exclusion(row): + reasons = [] + if row["eval"] in suite.get("exclude", []): + reasons.append(("eval", row["eval"], "")) + else: + group = metadata.get(row["task"], {}) + codes = {group.get(k) for k in ("language", "source_language", "target_language")} - {None} + for language in codes: + if language in suite.get("exclude_languages", []): + reasons.append(("language", language, "")) + elif language in local_languages.get(row["eval"], []): + reasons.append(("language", language, row["eval"])) + for reason in reasons: + exclusions.setdefault(reason, {})[measurement_id(row)] = row + return bool(reasons) + used = {r["eval"] for rr in included.values() for r in rr} for e in catalogue["evals"]: if e.get("warning") and e["name"] in used: @@ -496,6 +519,7 @@ def report_diagnostics( if ("select" not in e or match_task(e["select"], r["task"]) is not None) and not in_suite(r, suite) ] + outside = [r for r in outside if not record_exclusion(r)] matching = [r for r in all_rows if in_suite(r, suite)] selected = [r for r in matching if r["selected"]] @@ -729,6 +753,25 @@ def match(r): "Matched results have different sample counts. Scores remain included.", ) ) + if exclusions: + groups, affected, variants, labels = [], {}, [], [] + for (kind, name, scope), records in sorted(exclusions.items()): + affected.update(records) + rows = list(records.values()) + tasks = sorted({r["task"] for r in rows}) + models = sorted({r["checkpoint"] for r in rows}) + groups.append(dict(kind=kind, name=name, eval=scope or None, models=models, + tasks=tasks, measurement_ids=sorted(records))) + label = ("Eval " if kind == "eval" else "Language ") + name + if kind == "language": + label += " in " + scope if scope else " across the set" + labels.append(label) + variants.append(dict(settings=label + " · " + ", ".join(models), tasks=tasks)) + info = diagnostic("intentional_exclusion", "Selected models", None, list(affected.values()), + "Intentionally excluded by " + suite["name"] + ": " + "; ".join(labels) + ".", + "excluded", severity="info") + info.update(name=suite["name"], exclusions=groups, variants=variants) + out.append(info) return out diff --git a/quickdash/cli.py b/quickdash/cli.py index 1966e20..a67e608 100644 --- a/quickdash/cli.py +++ b/quickdash/cli.py @@ -76,11 +76,11 @@ def main(argv=None): ) for d in report.diagnostics: print( - f"warning [{d['code']}] {d['model']} / {d['name']}: {d['detail']}" + f"{d['severity']} [{d['code']}] {d['model']} / {d['name']}: {d['detail']}" + (f" Tasks: {', '.join(d['tasks'])}" if d["tasks"] else ""), file=sys.stderr, ) - if args.strict and report.diagnostics: + if args.strict and any(d['severity'] == 'warning' for d in report.diagnostics): return 1 if args.format == "json": print(json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False)) diff --git a/quickdash/config.py b/quickdash/config.py index 4872cf0..56806db 100644 --- a/quickdash/config.py +++ b/quickdash/config.py @@ -727,8 +727,7 @@ def validate_suite(s): ): raise ValueError("Notes must be strings") if "exclude" in s and ( - s["mode"] != "available" - or not isinstance(s["exclude"], list) + not isinstance(s["exclude"], list) or any(not isinstance(n, str) or not n.strip() for n in s["exclude"]) or len(set(s["exclude"])) != len(s["exclude"]) ): @@ -774,6 +773,8 @@ def validate_suite(s): ): raise ValueError("Suite eval names must be unique") names.add(e["name"]) + if e["name"] in s.get("exclude", []): + raise ValueError("Eval cannot be both required and excluded: " + e["name"]) if "variants" not in e: continue if not isinstance(e["variants"], list) or not e["variants"]: @@ -906,10 +907,10 @@ def codes(task): all_tasks = set(t for tasks in task_sets.values() for t in tasks) removed = excluded(suite.get("exclude_languages", []), all_tasks, "catalogue") result = deepcopy(suite) + for name in suite.get("exclude", []): + if name not in by_name: + raise ValueError("Excluded eval has no catalogue rule: " + name) if suite["mode"] == "available": - for name in suite.get("exclude", []): - if name not in by_name: - raise ValueError("Excluded eval has no catalogue rule: " + name) result["_excluded_tasks"] = sorted(removed) return result for required in result["evals"]: diff --git a/tests/test_engines.py b/tests/test_engines.py index 13de995..e48806b 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -113,6 +113,106 @@ def semantics(value): class Engines(unittest.TestCase): + def test_intentional_language_exclusions_coalesce_across_models_and_evals(self): + c = fixture() + c['catalogue']['evals'].append({**deepcopy(c['catalogue']['evals'][0]), + 'name': 'Translation', 'match': {'regex': 't_.+'}}) + c['catalogue']['languages'] += [ + dict(tasks=[task], scope='translation', source_language=source, target_language=target) + for task, source, target in [('t_fr_en', 'fra_Latn', 'eng_Latn'), + ('t_en_fr', 'eng_Latn', 'fra_Latn'), + ('t_en_de', 'eng_Latn', 'deu_Latn')] + ] + c['suite'] = dict(version=1, name='Required', mode='fixed', + exclude_languages=['fra_Latn'], + evals=[dict(name='E'), dict(name='Translation', exclude_languages=['deu_Latn'])]) + rows = paired([row(t) for t in ['e_en', 'e_fr', 't_fr_en', 't_en_fr', 't_en_de']]) + for output in self.both([dict(config=c, rows=rows, operation='compare')])[0]: + report = output['value'] + self.close([report['a']['score'], report['b']['score']], [50, 20]) + self.assertTrue(report['coverage']['complete']) + self.assertEqual(report['coverage']['required'], 1) + self.assertEqual(len(report['diagnostics']), 1) + info = report['diagnostics'][0] + self.assertEqual((info['code'], info['severity']), ('intentional_exclusion', 'info')) + self.assertEqual(info['tasks'], ['e_fr', 't_en_de', 't_en_fr', 't_fr_en']) + self.assertEqual(len(info['measurement_ids']), 8) + self.assertEqual([(g['kind'], g['name'], g['eval']) for g in info['exclusions']], + [('language', 'deu_Latn', 'Translation'), ('language', 'fra_Latn', None)]) + self.assertTrue(all(g['models'] == ['A', 'B'] for g in info['exclusions'])) + self.assertEqual([len(g['measurement_ids']) for g in info['exclusions']], [2, 6]) + # Both translation endpoints can match, but each measurement is counted once. + c['suite']['exclude_languages'].append('eng_Latn') + c['suite']['evals'][1]['exclude_languages'].append('fra_Latn') + for output in self.both([dict(config=c, rows=rows, operation='compare')])[0]: + info = output['value']['diagnostics'][0] + self.assertEqual(len(output['value']['diagnostics']), 1) + self.assertEqual(len(info['measurement_ids']), 10) + self.assertEqual(len(set(info['measurement_ids'])), 10) + self.assertEqual(len(info['exclusions']), 3) + + def test_fixed_exclusions_explain_unused_data_without_hiding_missing_requirements(self): + c = fixture() + for name in ('Old', 'Missing'): + c['catalogue']['evals'].append({**deepcopy(c['catalogue']['evals'][0]), + 'name': name, 'match': {'name': name.lower()}, 'warning': 'Review this eval'}) + c['catalogue']['languages'].append(dict(tasks=[name.lower()], scope='single', language='eng_Latn')) + c['suite'] = dict(version=1, name='Required', mode='fixed', exclude=['Old'], + evals=[dict(name='E'), dict(name='Missing')]) + rows = paired([row(), row('e_fr'), row('old', metric='wrong_metric')]) + unmarked = deepcopy(c) + unmarked['suite'].pop('exclude') + reference = native(dict(config=unmarked, rows=rows, operation='compare'))['value'] + for output in self.both([dict(config=c, rows=rows, operation='compare')])[0]: + report = output['value'] + self.assertFalse(report['coverage']['complete']) + self.close([report['a']['score'], report['b']['score']], [50, 20]) + self.close({k: v for k, v in report.items() if k != 'diagnostics'}, + {k: v for k, v in reference.items() if k != 'diagnostics'}) + warnings = [d for d in report['diagnostics'] if d['severity'] == 'warning'] + self.assertEqual([(d['code'], d['model'], d['eval']) for d in warnings], + [('missing_suite_data', 'A', 'Missing'), ('missing_suite_data', 'B', 'Missing')]) + infos = [d for d in report['diagnostics'] if d['severity'] == 'info'] + self.assertEqual(len(infos), 1) + self.assertEqual(infos[0]['exclusions'][0]['name'], 'Old') + self.assertEqual(infos[0]['exclusions'][0]['kind'], 'eval') + self.assertEqual(len(infos[0]['measurement_ids']), 2) + # An intentionally absent eval is not required and creates no data notice. + c['suite']['evals'] = [dict(name='E')] + for output in self.both([dict(config=c, rows=paired([row(), row('e_fr')]), operation='compare')])[0]: + self.assertTrue(output['value']['coverage']['complete']) + self.assertEqual(output['value']['diagnostics'], []) + + def test_exclusions_reject_unknown_names_and_required_excluded_conflicts(self): + cases = [] + for mode in ('available', 'fixed'): + for exclusion in (None, 'E', [1], [''], ['E', 'E'], ['Unknown eval']): + c = fixture() + c['suite'] = dict(version=1, name='Bad', mode=mode, exclude=exclusion) + if mode == 'fixed': + c['suite']['evals'] = [dict(name='E')] + cases.append(dict(config=c, rows=[row()])) + c = fixture() + c['suite'] = dict(version=1, name='Conflict', mode='fixed', exclude=['E'], evals=[dict(name='E')]) + cases.append(dict(config=c, rows=[row()])) + for outputs in self.both(cases): + for output in outputs: + self.assertIn('error', output) + + def test_information_is_retained_without_emitting_warnings_or_failing_strict_mode(self): + c = fixture() + c['suite']['exclude_languages'] = ['fra_Latn'] + rows = [row(), row('e_fr')] + with warnings.catch_warnings(record=True) as seen: + warnings.simplefilter('always') + report = analyze(rows, c) + self.assertFalse(seen) + self.assertEqual(report.diagnostics[0]['severity'], 'info') + self.assertEqual(analyze(rows, c, diagnostics='error'), report) + with self.assertRaises(DiagnosticError) as raised: + analyze(rows + [row('unknown')], c, diagnostics='error') + self.assertEqual([d['code'] for d in raised.exception.diagnostics], ['no_config']) + def close(self, a, b, path="result"): if isinstance(a, dict): self.assertEqual(a.keys(), b.keys(), path) @@ -459,7 +559,7 @@ def test_strict_fewshot_warnings_are_grouped_without_duplicate_coverage_warnings ds = result["value"]["diagnostics"] shot = next(d for d in ds if d["code"] == "strict_shot_setting") self.assertEqual(shot["tasks"], ["e_en"]) - self.assertIn("not_used", {d["code"] for d in ds}) + self.assertIn("intentional_exclusion", {d["code"] for d in ds}) # Counts are grouped independently for each model. for result in self.both([dict(config=c, rows=paired(rr))])[0]: ds = result["value"]["diagnostics"] @@ -643,7 +743,7 @@ def test_eval_set_defaults_overrides_and_language_exclusions(self): {m["task"] for m in r["a"]["measurements"] if m["included"]}, {"e_en", "e_fr"}, ) - self.assertEqual({d["code"] for d in r["diagnostics"]}, {"not_used"}) + self.assertEqual({d["code"] for d in r["diagnostics"]}, {"intentional_exclusion"}) self.assertEqual(c, before, "Resolution must not mutate inputs") c["suite"]["evals"][0]["exclude_languages"] = ["eng_Latn"] cases = [dict(config=deepcopy(c), rows=rr, operation="compare")] @@ -839,7 +939,7 @@ def test_eval_set_inheritance_and_component_language_exclusions(self): self.assertEqual(r["coverage"]["required"], 2) self.assertTrue(r["coverage"]["complete"]) self.assertAlmostEqual(r["a"]["score"], 200 / 3) - self.assertEqual({d["code"] for d in r["diagnostics"]}, {"not_used"}) + self.assertEqual({d["code"] for d in r["diagnostics"]}, {"intentional_exclusion"}) # Explicit membership may not slice a required component group. c["suite"]["evals"][0]["variants"] = [dict(task="e_en_low")] for result in self.both([dict(config=c, rows=rr)])[0]: @@ -875,7 +975,7 @@ def test_flagship_exclusions_are_set_policy(self): self.assertTrue(excluded_tasks) self.assertIn("xcsqa_eng_Latn", excluded_tasks) warned = { - t for d in report.diagnostics if d["code"] == "not_used" for t in d["tasks"] + t for d in report.diagnostics if d["code"] == "intentional_exclusion" for t in d["tasks"] } self.assertTrue(excluded_tasks <= warned) self.assertFalse( @@ -1078,7 +1178,7 @@ def add(config, rows, codes, operation="compare"): add(c, paired([row()]), ["config_caveat"]) c = fixture() c["suite"]["exclude"] = ["E"] - add(c, paired([row()]), ["not_used"]) + add(c, paired([row()]), ["intentional_exclusion"]) c = fixture() c["suite"] = dict( version=1, @@ -1547,6 +1647,27 @@ def test_python_runtime_never_calls_node_and_cli_streams(self): self.assertEqual(strict.returncode, 1) self.assertEqual(strict.stdout, "") self.assertIn("no_config", strict.stderr) + # Information remains visible on stderr/JSON without failing --strict. + suite = {**fixture()['suite'], 'exclude_languages': ['fra_Latn']} + (folder / 'suite.yaml').write_text(json.dumps(suite)) + stream = io.StringIO() + writer = csv.DictWriter(stream, fieldnames=row()) + writer.writeheader() + writer.writerows([row(), row('e_fr')]) + (folder / 'scores.csv').write_text(stream.getvalue()) + info_command = command + ['--eval-set', str(folder / 'suite.yaml'), '--strict'] + info = subprocess.run(info_command, capture_output=True, text=True, env=env, cwd=ROOT) + self.assertEqual(info.returncode, 0, info.stderr) + self.assertIn('info [intentional_exclusion]', info.stderr) + self.assertNotIn('warning [', info.stderr) + self.assertEqual(json.loads(info.stdout)['diagnostics'][0]['severity'], 'info') + writer.writerow(row('unknown')) + (folder / 'scores.csv').write_text(stream.getvalue()) + mixed = subprocess.run(info_command, capture_output=True, text=True, env=env, cwd=ROOT) + self.assertEqual(mixed.returncode, 1) + self.assertEqual(mixed.stdout, '') + self.assertIn('info [intentional_exclusion]', mixed.stderr) + self.assertIn('warning [no_config]', mixed.stderr) def test_diagnostic_context_and_suppression(self): c = fixture() diff --git a/tests/test_public_browser.mjs b/tests/test_public_browser.mjs index 7de3761..278f641 100644 --- a/tests/test_public_browser.mjs +++ b/tests/test_public_browser.mjs @@ -473,12 +473,35 @@ reconcile(Path(sys.argv[3]), source) await evaluate('URL.createObjectURL=originalCreate;HTMLAnchorElement.prototype.click=originalClick'); await upload('#suiteFile',serializeSuite({version:1,name:'Any example',mode:'available'}),'any.yaml'); assert.equal(await evaluate("document.querySelector('#cards').textContent"),noShared); - // A present excluded language warns; absent excluded languages are not requirements. + // Explicit language/eval exclusions are one informational entry, not warnings. const frenchCSV='checkpoint,task,metric,filter,n_shot,harness,backend,value\nShots A,example_reasoning_fr,acc_norm,none,5,test,cpu,0.5\nShots B,example_reasoning_fr,acc_norm,none,5,test,cpu,0.5'; await click('#clearModels');await upload('#modelFile',shotCSV()+'\n'+frenchCSV.split('\n').slice(1).join('\n'),'languages.csv');await click('[data-view=config]'); assert.equal(await evaluate("document.querySelector('#error').textContent"),''); await upload('#suiteFile',serializeSuite({...shotSet,exclude_languages:['fra_Latn']}),'exclusions.yaml'); - await click('[data-view=warnings]');assert.match(await evaluate("document.querySelector('#view').textContent"),/Not used.*example_reasoning_fr/s); + await click('[data-view=warnings]');assert.match(await evaluate("document.querySelector('#information').textContent"),/Intentional exclusions.*example_reasoning_fr/s); + const knownExcludedCSV=shotCSV().replace('acc_norm,none,5','acc_norm,none,0')+'\n'+frenchCSV.split('\n').slice(1).join('\n')+'\nShots A,example_math_en,exact_match,none,0,test,cpu,0.2\nShots B,example_math_en,exact_match,none,0,test,cpu,0.3'; + await click('#clearModels');await upload('#modelFile',knownExcludedCSV,'known-exclusions.csv'); + const beforeExclusion=await evaluate("document.querySelector('#cards').textContent"); + assert.equal(await evaluate("document.querySelector('#warningCount').textContent"),'2'); + const markedSet={...shotSet,exclude:['Example math'],exclude_languages:['fra_Latn']}; + await click('[data-view=config]');await upload('#suiteFile',serializeSuite(markedSet).replace('- Example math','- Example math # Intentionally outside this fixture.'),'intentional-exclusions.yaml'); + assert.equal(await evaluate("document.querySelector('#error').textContent"),''); + assert.equal(await evaluate("document.querySelector('#cards').textContent"),beforeExclusion); + assert.equal(await evaluate("document.querySelector('#warningCount').textContent"),'0'); + assert.equal(await evaluate("document.querySelector('#warningCount').classList.contains('has-warnings')"),false); + await click('[data-view=warnings]'); + assert.match(await evaluate("document.querySelector('#view').textContent"),/No warnings/); + assert.equal(await evaluate("document.querySelectorAll('#information tbody tr').length"),1); + assert.match(await evaluate("document.querySelector('#information').textContent"),/Example math.*fra_Latn/s); + assert.equal(await evaluate("document.querySelectorAll('#information details p').length"),2); + await click('[data-view=config]'); + await evaluate(`window.originalCreate=URL.createObjectURL;window.originalClick=HTMLAnchorElement.prototype.click;URL.createObjectURL=b=>{window.exportBlob=b;return 'blob:test'};HTMLAnchorElement.prototype.click=function(){};`); + await click('#exportSuite');assert.deepEqual(await evaluate('exportBlob.text().then(parseSuite)'),markedSet); + await evaluate('URL.createObjectURL=originalCreate;HTMLAnchorElement.prototype.click=originalClick'); + await upload('#suiteFile',JSON.stringify({...markedSet,exclude:['Example reasoning']}),'conflicting-exclusion.yaml'); + assert.match(await evaluate("document.querySelector('#error').textContent"),/both required and excluded/); + assert.equal(await evaluate("document.querySelector('#cards').textContent"),beforeExclusion); + assert.equal(await evaluate("document.querySelector('#warningCount').textContent"),'0'); await click('[data-view=config]');await upload('#suiteFile',serializeSuite({version:1,name:'Any example',mode:'available'}),'any.yaml'); await click('#clearModels');await upload('#modelFile',shotCSV('bad'),'invalid-alternative.csv'); const beforeRelax=await evaluate("document.querySelector('#cards').textContent"); diff --git a/tests/test_warning_policy.cjs b/tests/test_warning_policy.cjs index f4a4edc..aa93bfb 100644 --- a/tests/test_warning_policy.cjs +++ b/tests/test_warning_policy.cjs @@ -18,7 +18,9 @@ test('displayed evals emit their caveats; unused catalogue rules and excluded ev const r=run([row()],[row()]);assert.deepEqual(r.warnings.filter(w=>w.type==='Config caveat').map(w=>w.name),['E']); const excluded=run([row(),row('f')],[row(),row('f')],{...available,exclude:['F']}); assert.deepEqual(excluded.warnings.filter(w=>w.type==='Config caveat').map(w=>w.name),['E']); - assert.equal(excluded.warnings.filter(w=>w.type==='Not used'&&w.name==='F').length,2); + assert.equal(excluded.warnings.filter(w=>w.type==='Not used').length,0); + const info=excluded.warnings.filter(w=>w.severity==='info');assert.equal(info.length,1); + assert.equal(info[0].code,'intentional_exclusion');assert.deepEqual(info[0].exclusions[0].models,['A','B']); assert.equal(excluded.coverage.pairs.length,1); }); test('unselected eval data warns even if it only contains the wrong metric',()=>{ @@ -60,5 +62,6 @@ test('available-mode exclusions reject unknown catalogue names',()=>{ assert.throws(()=>resolveConfig(catalogue(),{...available,exclude:['Absent from this catalogue']},profile),/no catalogue rule/); const r=run([row()],[row()],{...available,exclude:['E','F']}); assert.equal(r.score,null);assert.equal(r.coverage.pairs.length,0); - assert.equal(r.warnings.filter(w=>w.type==='Not used').length,2); + assert.equal(r.warnings.filter(w=>w.type==='Not used').length,0); + assert.equal(r.warnings.filter(w=>w.severity==='info').length,1); });