Separate global_piqa from piqa, and split nno and nob

configs/evals/arc_challenge.yaml

@@ -56,7 +56,7 @@ languages:
     tasks: [arc_challenge_mt_lt]
   - language: lav_Latn
     tasks: [arc_challenge_mt_lv]
-  - language: nor_Latn
+  - language: nob_Latn
     tasks: [arc_challenge_mt_nb]
   - language: nld_Latn
     tasks: [arc_challenge_mt_nl]

configs/evals/belebele.yaml

@@ -50,7 +50,7 @@ languages:
     tasks: [belebele_mlt_Latn]
   - language: nld_Latn
     tasks: [belebele_nld_Latn]
-  - language: nor_Latn
+  - language: nob_Latn
     tasks: [belebele_nob_Latn]
   - language: pol_Latn
     tasks: [belebele_pol_Latn]

configs/evals/flores200.yaml

@@ -119,7 +119,7 @@ languages:
     target_language: nld_Latn
     tasks: ['flores200:eng_Latn-nld_Latn']
   - source_language: eng_Latn
-    target_language: nor_Latn
+    target_language: nob_Latn
     tasks: ['flores200:eng_Latn-nob_Latn']
   - source_language: eng_Latn
     target_language: pol_Latn
@@ -199,7 +199,7 @@ languages:
   - source_language: nld_Latn
     target_language: eng_Latn
     tasks: ['flores200:nld_Latn-eng_Latn']
-  - source_language: nor_Latn
+  - source_language: nob_Latn
     target_language: eng_Latn
     tasks: ['flores200:nob_Latn-eng_Latn']
   - source_language: pol_Latn

configs/evals/global_piqa.yaml

@@ -0,0 +1,81 @@
+name: global_piqa
+category: Commonsense
+match:
+  regex: global_piqa_completions_.+
+metric: acc_norm
+metric_filter: none
+shots: 0
+score:
+  scale: 1
+normalize:
+  min: 0.5
+  max: 1
+  basis: uniform_choice
+  note: The completion tasks compare two solutions. Uniform guessing gives 1/2
+  sources:
+    - https://github.com/OpenEuroLLM/oellm-eval/blob/8a4b2412a8e8f7f0d95e3845e2164c792add6a79/oellm/resources/custom_lm_eval_tasks/global_piqa/completions/_template
+languages:
+  - language: sqi_Latn
+    tasks: [global_piqa_completions_als_latn]
+  - language: bos_Latn
+    tasks: [global_piqa_completions_bos_latn]
+  - language: bul_Cyrl
+    tasks: [global_piqa_completions_bul_cyrl]
+  - language: cat_Latn
+    tasks: [global_piqa_completions_cat_latn]
+  - language: ces_Latn
+    tasks: [global_piqa_completions_ces_latn]
+  - language: deu_Latn
+    tasks: [global_piqa_completions_deu_latn]
+  - language: est_Latn
+    tasks: [global_piqa_completions_ekk_latn]
+  - language: ell_Grek
+    tasks: [global_piqa_completions_ell_grek]
+  - language: eng_Latn
+    tasks: [global_piqa_completions_eng_latn]
+  - language: fin_Latn
+    tasks: [global_piqa_completions_fin_latn]
+  - language: fra_Latn
+    tasks: [global_piqa_completions_fra_latn_fran]
+  - language: glg_Latn
+    tasks: [global_piqa_completions_glg_latn]
+  - language: hrv_Latn
+    tasks: [global_piqa_completions_hrv_latn]
+  - language: hun_Latn
+    tasks: [global_piqa_completions_hun_latn]
+  - language: isl_Latn
+    tasks: [global_piqa_completions_isl_latn]
+  - language: ita_Latn
+    tasks: [global_piqa_completions_ita_latn]
+  - language: kat_Geor
+    tasks: [global_piqa_completions_kat_geor]
+  - language: lit_Latn
+    tasks: [global_piqa_completions_lit_latn]
+  - language: mkd_Cyrl
+    tasks: [global_piqa_completions_mkd_cyrl]
+  - language: nld_Latn
+    tasks: [global_piqa_completions_nld_latn]
+  - language: nno_Latn
+    tasks: [global_piqa_completions_nno_latn]
+  - language: nob_Latn
+    tasks: [global_piqa_completions_nob_latn]
+  - language: pol_Latn
+    tasks: [global_piqa_completions_pol_latn]
+  - language: por_Latn
+    tasks: [global_piqa_completions_por_latn_port]
+  - language: ron_Latn
+    tasks: [global_piqa_completions_ron_latn]
+  - language: slk_Latn
+    tasks: [global_piqa_completions_slk_latn]
+  - language: slv_Latn
+    tasks: [global_piqa_completions_slv_latn]
+  - language: spa_Latn
+    tasks: [global_piqa_completions_spa_latn_spai]
+  - language: srp_Cyrl
+    tasks: [global_piqa_completions_srp_cyrl]
+  - language: swe_Latn
+    tasks: [global_piqa_completions_swe_latn]
+  - language: tur_Latn
+    tasks: [global_piqa_completions_tur_latn]
+  - language: ukr_Cyrl
+    tasks: [global_piqa_completions_ukr_cyrl]

configs/evals/global_piqa_prompted.yaml

@@ -57,8 +57,10 @@ languages:
     tasks: [global_piqa_prompted_mkd_cyrl]
   - language: nld_Latn
     tasks: [global_piqa_prompted_nld_latn]
-  - language: nor_Latn
-    tasks: [global_piqa_prompted_nno_latn, global_piqa_prompted_nob_latn]
+  - language: nno_Latn
+    tasks: [global_piqa_prompted_nno_latn]
+  - language: nob_Latn
+    tasks: [global_piqa_prompted_nob_latn]
   - language: pol_Latn
     tasks: [global_piqa_prompted_pol_latn]
   - language: por_Latn

configs/evals/ifeval.yaml

@@ -1,5 +1,5 @@
 name: ifeval
-category: Instruction following
+category: Instruction
 match:
   name: ifeval
 metric: prompt_level_strict_acc

configs/evals/mgsm.yaml

@@ -2,6 +2,8 @@ name: mgsm
 category: Math
 match:
   regex: (?:mgsm_native_cot_.+|global_mgsm_.+)
+select:
+  regex: global_mgsm_[a-z]+
 metric: exact_match
 metric_filter: flexible-extract
 shots: 5

configs/evals/multiblimp.yaml

@@ -48,7 +48,7 @@ languages:
     tasks: [multiblimp_gle]
   - language: glg_Latn
     tasks: [multiblimp_glg]
-  - language: srp_Latn
+  - language: hbs_Latn
     tasks: [multiblimp_hbs]
     evidence: https://huggingface.co/datasets/jumelet/multiblimp/blob/main/hbs/data.tsv
     note: 'Grouping convention: assign the pooled Croatian/Serbian score to Serbian, the language with more

configs/evals/piqa.yaml

@@ -1,88 +1,21 @@
 name: piqa
 category: Commonsense
 match:
-  regex: (?:piqa|global_piqa_completions_.+)
+  name: piqa
 metric: acc_norm
 metric_filter: none
-shots: 10
+shots: 0
 score:
   scale: 1
 normalize:
   min: 0.5
   max: 1
   basis: uniform_choice
-  note: The completion tasks compare two solutions. Uniform guessing gives 1/2. Prompted Global PIQA is configured
-    separately and excluded from the supplied comparison sets.
+  note: The completion tasks compare two solutions. Uniform guessing gives 1/2.
   sources:
     - https://github.com/EleutherAI/lm-evaluation-harness/blob/d6de81643928d653435c431bae19945d41d32520/lm_eval/tasks/piqa/piqa.yaml
-    - https://github.com/OpenEuroLLM/oellm-eval/blob/8a4b2412a8e8f7f0d95e3845e2164c792add6a79/oellm/resources/custom_lm_eval_tasks/global_piqa/completions/_template
-language_defaults:
-  evidence: https://huggingface.co/datasets/mrlbenchmarks/global-piqa-nonparallel
-  note: Dataset language resolved from the OELLM task registry and benchmark configuration.
 languages:
   - language: eng_Latn
     tasks: [piqa]
     evidence: https://github.com/OpenEuroLLM/oellm-eval/blob/8a4b2412a8e8f7f0d95e3845e2164c792add6a79/oellm/resources/task-groups.yaml
     note: Original English benchmark variant; language is the prompt/question language.
-  - language: sqi_Latn
-    tasks: [global_piqa_completions_als_latn]
-  - language: bos_Latn
-    tasks: [global_piqa_completions_bos_latn]
-  - language: bul_Cyrl
-    tasks: [global_piqa_completions_bul_cyrl]
-  - language: cat_Latn
-    tasks: [global_piqa_completions_cat_latn]
-  - language: ces_Latn
-    tasks: [global_piqa_completions_ces_latn]
-  - language: deu_Latn
-    tasks: [global_piqa_completions_deu_latn]
-  - language: est_Latn
-    tasks: [global_piqa_completions_ekk_latn]
-  - language: ell_Grek
-    tasks: [global_piqa_completions_ell_grek]
-  - language: eng_Latn
-    tasks: [global_piqa_completions_eng_latn]
-  - language: fin_Latn
-    tasks: [global_piqa_completions_fin_latn]
-  - language: fra_Latn
-    tasks: [global_piqa_completions_fra_latn_fran]
-  - language: glg_Latn
-    tasks: [global_piqa_completions_glg_latn]
-  - language: hrv_Latn
-    tasks: [global_piqa_completions_hrv_latn]
-  - language: hun_Latn
-    tasks: [global_piqa_completions_hun_latn]
-  - language: isl_Latn
-    tasks: [global_piqa_completions_isl_latn]
-  - language: ita_Latn
-    tasks: [global_piqa_completions_ita_latn]
-  - language: kat_Geor
-    tasks: [global_piqa_completions_kat_geor]
-  - language: lit_Latn
-    tasks: [global_piqa_completions_lit_latn]
-  - language: mkd_Cyrl
-    tasks: [global_piqa_completions_mkd_cyrl]
-  - language: nld_Latn
-    tasks: [global_piqa_completions_nld_latn]
-  - language: nor_Latn
-    tasks: [global_piqa_completions_nno_latn, global_piqa_completions_nob_latn]
-  - language: pol_Latn
-    tasks: [global_piqa_completions_pol_latn]
-  - language: por_Latn
-    tasks: [global_piqa_completions_por_latn_port]
-  - language: ron_Latn
-    tasks: [global_piqa_completions_ron_latn]
-  - language: slk_Latn
-    tasks: [global_piqa_completions_slk_latn]
-  - language: slv_Latn
-    tasks: [global_piqa_completions_slv_latn]
-  - language: spa_Latn
-    tasks: [global_piqa_completions_spa_latn_spai]
-  - language: srp_Cyrl
-    tasks: [global_piqa_completions_srp_cyrl]
-  - language: swe_Latn
-    tasks: [global_piqa_completions_swe_latn]
-  - language: tur_Latn
-    tasks: [global_piqa_completions_tur_latn]
-  - language: ukr_Cyrl
-    tasks: [global_piqa_completions_ukr_cyrl]

configs/evals/sib200.yaml

@@ -71,7 +71,7 @@ languages:
     tasks: [sib200_mlt_Latn]
   - language: nld_Latn
     tasks: [sib200_nld_Latn]
-  - language: nor_Latn
+  - language: nob_Latn
     tasks: [sib200_nob_Latn]
   - language: pol_Latn
     tasks: [sib200_pol_Latn]

configs/sets/flagship-1.yaml

@@ -35,6 +35,7 @@ evals:
   - name: copa
   - name: hellaswag
   - name: piqa
+  - name: global_piqa
   - name: social_iqa
   - name: wsc273
   - name: winogrande