Coalesce intentional exclusions into information instead of warnings

app/analysis.js

@@ -241,19 +241,38 @@ function sameCoverage(a,b,config=null,matching='strict'){return a.length===b.len
 // Public, serializable analysis boundary used by Python parity tests and the UI.
 function measurementId(r){return JSON.stringify([r.checkpoint,...['task','metric','filter','n_shot','harness','backend'].map(k=>r[k])]);}
 const diagnosticTitles={config_caveat:'Config caveat',no_config:'No config',not_used:'Not used',unknown_language:'Unknown language',invalid_sample_count:'Invalid sample count',strict_shot_setting:'Few-shot mismatch excluded',relaxed_shot_setting:'Few-shot mismatch allowed',ambiguous_shot_setting:'Ambiguous few-shot setting',inconsistent_scoring_settings:'Inconsistent scoring settings',missing_scoring_field:'Missing scoring field',missing_scoring_setting:'Missing scoring setting',no_selected_score:'No selected score',missing_suite_data:'Missing suite data',incomplete_components:'Incomplete components',comparison_coverage:'Comparison coverage',sample_count_mismatch:'Sample-count mismatch',no_category_weight:'No category weight'};
-function diagnostic(code,model,evalName,rows,detail,effect='included',tasks=null){
+diagnosticTitles.intentional_exclusion='Intentional exclusions';
+function diagnostic(code,model,evalName,rows,detail,effect='included',tasks=null,severity='warning'){
  const names=[...new Set(tasks??rows.map(r=>r.task))].sort(compareText);
- return {code,type:diagnosticTitles[code],model,eval:evalName,name:evalName||names[0]||model,tasks:names,measurement_ids:rows.map(measurementId).sort(compareText),effect,detail,variants:names.length?[{settings:detail,tasks:names}]:[]};
+ return {code,severity,type:diagnosticTitles[code],model,eval:evalName,name:evalName||names[0]||model,tasks:names,measurement_ids:rows.map(measurementId).sort(compareText),effect,detail,variants:names.length?[{settings:detail,tasks:names}]:[]};
 }
 function reportDiagnostics(audits,config,included,comparison=false,matchingMode='strict',resolved=null){
  const {profile,catalogue,suite,scheme}=resolved||resolveInputs(config.catalogue,config.suite,config.profile),out=[];
+ const exclusions=new Map(),metadata=new Map(catalogue.languages.flatMap(g=>g.tasks.map(t=>[t,g]))),localLanguages=new Map((suite.evals||[]).map(e=>[e.name,e.exclude_languages||[]]));
+ function recordExclusion(row){
+  const reasons=[];
+  if((suite.exclude||[]).includes(row.eval))reasons.push(['eval',row.eval,'']);
+  else{
+   const group=metadata.get(row.task)||{},codes=new Set(['language','source_language','target_language'].map(k=>group[k]).filter(Boolean));
+   for(const language of codes){
+    if((suite.exclude_languages||[]).includes(language))reasons.push(['language',language,'']);
+    else if((localLanguages.get(row.eval)||[]).includes(language))reasons.push(['language',language,row.eval]);
+   }
+  }
+  for(const [kind,name,scope] of reasons){
+   const key=JSON.stringify([kind,name,scope]);
+   if(!exclusions.has(key))exclusions.set(key,{kind,name,scope,records:new Map()});
+   exclusions.get(key).records.set(measurementId(row),row);
+  }
+  return reasons.length>0;
+ }
  const used=new Set([...included.values()].flat().map(r=>r.eval));
  for(const e of catalogue.evals)if(e.warning&&used.has(e.name))out.push(diagnostic('config_caveat','Selected comparison',e.name,[],e.warning));
  for(const [model,rows] of audits){
   const scope=scopeRows(rows,suite),explainedShots=new Set(),accepted=included.get(model)||[],acceptedIds=new Set(accepted.map(measurementId));
   for(const task of [...new Set(rows.filter(r=>!r.eval).map(r=>r.task))].sort(compareText))out.push(diagnostic('no_config',model,null,rows.filter(r=>r.task===task),'No eval configuration; excluded from scoring.','excluded'));
   for(const e of catalogue.evals){
-   const all=rows.filter(r=>r.eval===e.name),outside=all.filter(r=>(!e.select||matchTask(e.select,r.task))&&!inSuite(r,suite)),matching=all.filter(r=>inSuite(r,suite)),selected=matching.filter(r=>r.selected);
+   const all=rows.filter(r=>r.eval===e.name),outside=all.filter(r=>(!e.select||matchTask(e.select,r.task))&&!inSuite(r,suite)).filter(r=>!recordExclusion(r)),matching=all.filter(r=>inSuite(r,suite)),selected=matching.filter(r=>r.selected);
    const add=(code,rr,detail,effect='included',tasks=null)=>out.push(diagnostic(code,model,e.name,rr,detail,effect,tasks));
    if(matchingMode==='relaxed'&&'shots'in e){
     const actuals=[...new Set(selected.filter(r=>acceptedIds.has(measurementId(r))&&Number(r.n_shot)!==e.shots).map(r=>Number(r.n_shot)))].sort((a,b)=>a-b);
@@ -295,6 +314,17 @@ function reportDiagnostics(audits,config,included,comparison=false,matchingMode=
  if(comparison){const match=r=>matchingKey(r,scheme,matchingMode),entries=[...included],a=entries[0]?.[1]||[],b=entries[1]?.[1]||a,bm=new Map(b.map(r=>[match(r),r]));
   for(const e of catalogue.evals){const rr=a.filter(r=>r.eval===e.name&&bm.has(match(r))&&sampleCount(r)!==null&&sampleCount(bm.get(match(r)))!==null&&sampleCount(r)!==sampleCount(bm.get(match(r))));if(rr.length)out.push(diagnostic('sample_count_mismatch','Selected comparison',e.name,rr.concat(rr.map(r=>bm.get(match(r)))),'Matched results have different sample counts. Scores remain included.'));}
  }
+ if(exclusions.size){
+  const groups=[],affected=new Map(),variants=[],labels=[];
+  for(const {kind,name,scope,records} of [...exclusions.values()].sort((a,b)=>compareText(a.kind,b.kind)||compareText(a.name,b.name)||compareText(a.scope,b.scope))){
+   for(const [id,row] of records)affected.set(id,row);
+   const rows=[...records.values()],tasks=[...new Set(rows.map(r=>r.task))].sort(compareText),models=[...new Set(rows.map(r=>r.checkpoint))].sort(compareText);
+   groups.push({kind,name,eval:scope||null,models,tasks,measurement_ids:[...records.keys()].sort(compareText)});
+   const label=(kind==='eval'?'Eval ':'Language ')+name+(kind==='language'?(scope?' in '+scope:' across the set'):'');
+   labels.push(label);variants.push({settings:label+' · '+models.join(', '),tasks});
+  }
+  out.push({...diagnostic('intentional_exclusion','Selected models',null,[...affected.values()],'Intentionally excluded by '+suite.name+': '+labels.join('; ')+'.','excluded',null,'info'),name:suite.name,exclusions:groups,variants});
+ }
  return out;
 }
 function allocationTree(rows,config,allocation,model){

app/suite_config.js

@@ -12,7 +12,7 @@ const SuiteConfig=(()=>{
   if(s.version!==1||!text(s.name))throw Error('Suite requires version 1 and a name');
   if(!['available','fixed'].includes(s.mode))throw Error('Suite mode must be available or fixed');
   if('notes'in s&&(!Array.isArray(s.notes)||s.notes.some(n=>typeof n!=='string')))throw Error('Suite notes must be strings');
-  if('exclude'in s&&(s.mode!=='available'||!Array.isArray(s.exclude)||s.exclude.some(n=>!text(n))||new Set(s.exclude).size!==s.exclude.length))throw Error('exclude must be a unique list of eval names in available mode');
+  if('exclude'in s&&(!Array.isArray(s.exclude)||s.exclude.some(n=>!text(n))||new Set(s.exclude).size!==s.exclude.length))throw Error('exclude must be a unique list of eval names');
   validateLanguageExclusions(s);
   if(s.mode==='available'){if('evals'in s)throw Error('Available mode does not declare required evals');return s;}
   if(!Array.isArray(s.evals)||!s.evals.length)throw Error('Fixed suite needs required evals');
@@ -23,6 +23,7 @@ const SuiteConfig=(()=>{
    if('metric_filter'in e&&typeof e.metric_filter!=='string')throw Error('metric_filter must be text');
    if('shots'in e&&(!Number.isSafeInteger(e.shots)||e.shots<0))throw Error('shots must be a nonnegative integer');
    if(!text(e.name)||names.has(e.name))throw Error('Suite eval names must be unique');names.add(e.name);
+   if((s.exclude||[]).includes(e.name))throw Error('Eval cannot be both required and excluded: '+e.name);
    if(!('variants'in e))continue;
    if(!Array.isArray(e.variants)||!e.variants.length)throw Error('Required variants must be a nonempty list');
    const seen=new Map();
@@ -71,8 +72,8 @@ const SuiteConfig=(()=>{
    return new Set(tasks.filter(t=>codes(t).some(c=>languages.includes(c))));
   }
   const removed=excluded(suite.exclude_languages||[],[...new Set([...taskSets.values()].flat())],'catalogue'),result=structuredClone(suite);
+  for(const name of suite.exclude||[])if(!byName.has(name))throw Error('Excluded eval has no catalogue rule: '+name);
   if(suite.mode==='available'){
-   for(const name of suite.exclude||[])if(!byName.has(name))throw Error('Excluded eval has no catalogue rule: '+name);
    result._excluded_tasks=[...removed].sort(api.compareText);return result;
   }
   for(const required of result.evals){

configs/README.md

@@ -27,6 +27,8 @@ python3 -m app.build --results-dir results --sample-csv examples/sample-evals.cs
 
 The build validates every offered combination before replacing output. In the dashboard, inspect **Warnings** for the comparison you intend to use. Named-set scores with missing requirements are explicitly incomplete; extras are excluded but remain inspectable.
 
+Record intentional eval exclusions in the set's `exclude` list and explain each eval/language exclusion with an inline YAML comment. Explicit exclusions appear together under **Information**, outside the warning count. Keep the authored YAML in version control because exported YAML does not preserve comments.
+
 The generated `output/shared/catalogue.yaml` contains the complete catalogue, including every language assignment, with no file references. Browser catalogue export produces the same portable format. Import that complete file when moving settings between dashboards; the repository manifest alone is not a browser import.
 
 For temporary changes, use the separate load/export controls under **Eval configuration**. Files stay in the browser. Weight exports save edited weights and the active calculation; catalogue and eval-set exports are independent. None of these controls modify the repository.

configs/sets/any-available.yaml

@@ -2,6 +2,6 @@ version: 1
 name: Any available
 mode: available
 exclude:
-  - global_piqa_prompted
+  - global_piqa_prompted # Scoring and normalization await validation.
 notes:
   - Compare recognized measurements shared by both models; warn about differences.

configs/sets/flagship-1.yaml

@@ -1,11 +1,25 @@
 version: 1
 name: flagship-1
 mode: fixed
-exclude_languages: [kat_Geor]
+exclude_languages:
+  - kat_Geor # Not targeted or evaluated for this model generation because Georgian was omitted from tokenizer training.
+exclude:
+  - AIME24 # Replaced by aime24_cot with corrected CoT evaluation settings.
+  - AIME25 # Replaced by aime25_cot with corrected CoT evaluation settings.
+  - AMC23 # Replaced by amc23_cot with corrected CoT evaluation settings.
+  - GPQADiamond # Replaced by gpqa_diamond_cot with corrected CoT evaluation settings.
+  - JEEBench # Replaced by jeebench_cot with corrected CoT evaluation settings.
+  - MATH500 # Replaced by math500_cot with corrected CoT evaluation settings.
+  - polymath # Replaced by polymath_cot; all four corrected difficulty levels are required.
+  - HumanEval # Replaced by humaneval_cont with the corrected code-continuation protocol.
+  - LiveCodeBench # Replaced by livecodebench_cont with the corrected code-continuation protocol.
+  - mbpp # Replaced by mbpp_cont with the corrected code-continuation protocol.
+  - global_piqa_prompted # Scoring and normalization await validation; use completion-based global_piqa.
 notes:
   - Use corrected CoT reasoning and code-continuation protocols; original-protocol results are excluded without fallback.
   - Require the catalogue tasks for each listed eval, inheriting its scoring defaults.
-  - Exclude Georgian results because of a vocabulary error. Exclude English X-CSQA to avoid overlap with CommonsenseQA.
+  - The omission of Georgian from tokenizer training was discovered too late to correct for this model generation, so Georgian is not targeted or evaluated.
+  - Exclude English X-CSQA to avoid overlap with CommonsenseQA.
 evals:
   - name: bigbench_cs_algorithms
   - name: humaneval_cont
@@ -42,7 +56,7 @@ evals:
   - name: winogrande
   - name: xcsqa
     exclude_languages:
-      - eng_Latn
+      - eng_Latn # Avoid overlap with CommonsenseQA.
   - name: xcopa
   - name: belebele
   - name: boolq