From b2854576aa03a1089df51b1909647330132dfcf0 Mon Sep 17 00:00:00 2001 From: Sampo Pyysalo Date: Fri, 9 Oct 2026 11:09:27 +0300 Subject: [PATCH 1/3] Separate global_piqa from piqa --- configs/evals/global_piqa.yaml | 79 ++++++++++++++++++++++++++++++++++ configs/evals/piqa.yaml | 71 +----------------------------- configs/sets/flagship-1.yaml | 1 + 3 files changed, 82 insertions(+), 69 deletions(-) create mode 100644 configs/evals/global_piqa.yaml diff --git a/configs/evals/global_piqa.yaml b/configs/evals/global_piqa.yaml new file mode 100644 index 0000000..a9585fe --- /dev/null +++ b/configs/evals/global_piqa.yaml @@ -0,0 +1,79 @@ +name: global_piqa +category: Commonsense +match: + regex: global_piqa_completions_.+ +metric: acc_norm +metric_filter: none +shots: 10 +score: + scale: 1 +normalize: + min: 0.5 + max: 1 + basis: uniform_choice + note: The completion tasks compare two solutions. Uniform guessing gives 1/2 + sources: + - https://github.com/OpenEuroLLM/oellm-eval/blob/8a4b2412a8e8f7f0d95e3845e2164c792add6a79/oellm/resources/custom_lm_eval_tasks/global_piqa/completions/_template +languages: + - language: sqi_Latn + tasks: [global_piqa_completions_als_latn] + - language: bos_Latn + tasks: [global_piqa_completions_bos_latn] + - language: bul_Cyrl + tasks: [global_piqa_completions_bul_cyrl] + - language: cat_Latn + tasks: [global_piqa_completions_cat_latn] + - language: ces_Latn + tasks: [global_piqa_completions_ces_latn] + - language: deu_Latn + tasks: [global_piqa_completions_deu_latn] + - language: est_Latn + tasks: [global_piqa_completions_ekk_latn] + - language: ell_Grek + tasks: [global_piqa_completions_ell_grek] + - language: eng_Latn + tasks: [global_piqa_completions_eng_latn] + - language: fin_Latn + tasks: [global_piqa_completions_fin_latn] + - language: fra_Latn + tasks: [global_piqa_completions_fra_latn_fran] + - language: glg_Latn + tasks: [global_piqa_completions_glg_latn] + - language: hrv_Latn + tasks: [global_piqa_completions_hrv_latn] + - language: hun_Latn + tasks: [global_piqa_completions_hun_latn] + - language: isl_Latn + tasks: [global_piqa_completions_isl_latn] + - language: ita_Latn + tasks: [global_piqa_completions_ita_latn] + - language: kat_Geor + tasks: [global_piqa_completions_kat_geor] + - language: lit_Latn + tasks: [global_piqa_completions_lit_latn] + - language: mkd_Cyrl + tasks: [global_piqa_completions_mkd_cyrl] + - language: nld_Latn + tasks: [global_piqa_completions_nld_latn] + - language: nor_Latn + tasks: [global_piqa_completions_nno_latn, global_piqa_completions_nob_latn] + - language: pol_Latn + tasks: [global_piqa_completions_pol_latn] + - language: por_Latn + tasks: [global_piqa_completions_por_latn_port] + - language: ron_Latn + tasks: [global_piqa_completions_ron_latn] + - language: slk_Latn + tasks: [global_piqa_completions_slk_latn] + - language: slv_Latn + tasks: [global_piqa_completions_slv_latn] + - language: spa_Latn + tasks: [global_piqa_completions_spa_latn_spai] + - language: srp_Cyrl + tasks: [global_piqa_completions_srp_cyrl] + - language: swe_Latn + tasks: [global_piqa_completions_swe_latn] + - language: tur_Latn + tasks: [global_piqa_completions_tur_latn] + - language: ukr_Cyrl + tasks: [global_piqa_completions_ukr_cyrl] diff --git a/configs/evals/piqa.yaml b/configs/evals/piqa.yaml index dc15c62..892f3c5 100644 --- a/configs/evals/piqa.yaml +++ b/configs/evals/piqa.yaml @@ -1,7 +1,7 @@ name: piqa category: Commonsense match: - regex: (?:piqa|global_piqa_completions_.+) + name: piqa metric: acc_norm metric_filter: none shots: 10 @@ -11,78 +11,11 @@ normalize: min: 0.5 max: 1 basis: uniform_choice - note: The completion tasks compare two solutions. Uniform guessing gives 1/2. Prompted Global PIQA is configured - separately and excluded from the supplied comparison sets. + note: The completion tasks compare two solutions. Uniform guessing gives 1/2. sources: - https://github.com/EleutherAI/lm-evaluation-harness/blob/d6de81643928d653435c431bae19945d41d32520/lm_eval/tasks/piqa/piqa.yaml - - https://github.com/OpenEuroLLM/oellm-eval/blob/8a4b2412a8e8f7f0d95e3845e2164c792add6a79/oellm/resources/custom_lm_eval_tasks/global_piqa/completions/_template -language_defaults: - evidence: https://huggingface.co/datasets/mrlbenchmarks/global-piqa-nonparallel - note: Dataset language resolved from the OELLM task registry and benchmark configuration. languages: - language: eng_Latn tasks: [piqa] evidence: https://github.com/OpenEuroLLM/oellm-eval/blob/8a4b2412a8e8f7f0d95e3845e2164c792add6a79/oellm/resources/task-groups.yaml note: Original English benchmark variant; language is the prompt/question language. - - language: sqi_Latn - tasks: [global_piqa_completions_als_latn] - - language: bos_Latn - tasks: [global_piqa_completions_bos_latn] - - language: bul_Cyrl - tasks: [global_piqa_completions_bul_cyrl] - - language: cat_Latn - tasks: [global_piqa_completions_cat_latn] - - language: ces_Latn - tasks: [global_piqa_completions_ces_latn] - - language: deu_Latn - tasks: [global_piqa_completions_deu_latn] - - language: est_Latn - tasks: [global_piqa_completions_ekk_latn] - - language: ell_Grek - tasks: [global_piqa_completions_ell_grek] - - language: eng_Latn - tasks: [global_piqa_completions_eng_latn] - - language: fin_Latn - tasks: [global_piqa_completions_fin_latn] - - language: fra_Latn - tasks: [global_piqa_completions_fra_latn_fran] - - language: glg_Latn - tasks: [global_piqa_completions_glg_latn] - - language: hrv_Latn - tasks: [global_piqa_completions_hrv_latn] - - language: hun_Latn - tasks: [global_piqa_completions_hun_latn] - - language: isl_Latn - tasks: [global_piqa_completions_isl_latn] - - language: ita_Latn - tasks: [global_piqa_completions_ita_latn] - - language: kat_Geor - tasks: [global_piqa_completions_kat_geor] - - language: lit_Latn - tasks: [global_piqa_completions_lit_latn] - - language: mkd_Cyrl - tasks: [global_piqa_completions_mkd_cyrl] - - language: nld_Latn - tasks: [global_piqa_completions_nld_latn] - - language: nor_Latn - tasks: [global_piqa_completions_nno_latn, global_piqa_completions_nob_latn] - - language: pol_Latn - tasks: [global_piqa_completions_pol_latn] - - language: por_Latn - tasks: [global_piqa_completions_por_latn_port] - - language: ron_Latn - tasks: [global_piqa_completions_ron_latn] - - language: slk_Latn - tasks: [global_piqa_completions_slk_latn] - - language: slv_Latn - tasks: [global_piqa_completions_slv_latn] - - language: spa_Latn - tasks: [global_piqa_completions_spa_latn_spai] - - language: srp_Cyrl - tasks: [global_piqa_completions_srp_cyrl] - - language: swe_Latn - tasks: [global_piqa_completions_swe_latn] - - language: tur_Latn - tasks: [global_piqa_completions_tur_latn] - - language: ukr_Cyrl - tasks: [global_piqa_completions_ukr_cyrl] diff --git a/configs/sets/flagship-1.yaml b/configs/sets/flagship-1.yaml index f2d3c05..f8a7d71 100644 --- a/configs/sets/flagship-1.yaml +++ b/configs/sets/flagship-1.yaml @@ -35,6 +35,7 @@ evals: - name: copa - name: hellaswag - name: piqa + - name: global_piqa - name: social_iqa - name: wsc273 - name: winogrande From 9234f84ec9ac95c85f29a85876f0aa8bd8b18db6 Mon Sep 17 00:00:00 2001 From: Sampo Pyysalo Date: Fri, 9 Oct 2026 12:26:04 +0300 Subject: [PATCH 2/3] Split nno and nob in PIQA --- configs/evals/global_piqa.yaml | 6 ++++-- configs/evals/global_piqa_prompted.yaml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/configs/evals/global_piqa.yaml b/configs/evals/global_piqa.yaml index a9585fe..8427342 100644 --- a/configs/evals/global_piqa.yaml +++ b/configs/evals/global_piqa.yaml @@ -55,8 +55,10 @@ languages: tasks: [global_piqa_completions_mkd_cyrl] - language: nld_Latn tasks: [global_piqa_completions_nld_latn] - - language: nor_Latn - tasks: [global_piqa_completions_nno_latn, global_piqa_completions_nob_latn] + - language: nno_Latn + tasks: [global_piqa_completions_nno_latn] + - language: nob_Latn + tasks: [global_piqa_completions_nob_latn] - language: pol_Latn tasks: [global_piqa_completions_pol_latn] - language: por_Latn diff --git a/configs/evals/global_piqa_prompted.yaml b/configs/evals/global_piqa_prompted.yaml index 04bfc63..672ee33 100644 --- a/configs/evals/global_piqa_prompted.yaml +++ b/configs/evals/global_piqa_prompted.yaml @@ -57,8 +57,10 @@ languages: tasks: [global_piqa_prompted_mkd_cyrl] - language: nld_Latn tasks: [global_piqa_prompted_nld_latn] - - language: nor_Latn - tasks: [global_piqa_prompted_nno_latn, global_piqa_prompted_nob_latn] + - language: nno_Latn + tasks: [global_piqa_prompted_nno_latn] + - language: nob_Latn + tasks: [global_piqa_prompted_nob_latn] - language: pol_Latn tasks: [global_piqa_prompted_pol_latn] - language: por_Latn From f6df70cfce4d8e4d5a04240169e865fcf15785ea Mon Sep 17 00:00:00 2001 From: Sampo Pyysalo Date: Fri, 9 Oct 2026 12:59:00 +0300 Subject: [PATCH 3/3] Preserve difference between nno and nob when specified --- configs/evals/arc_challenge.yaml | 2 +- configs/evals/belebele.yaml | 2 +- configs/evals/flores200.yaml | 4 ++-- configs/evals/global_piqa.yaml | 2 +- configs/evals/ifeval.yaml | 2 +- configs/evals/mgsm.yaml | 2 ++ configs/evals/multiblimp.yaml | 2 +- configs/evals/piqa.yaml | 2 +- configs/evals/sib200.yaml | 2 +- 9 files changed, 11 insertions(+), 9 deletions(-) diff --git a/configs/evals/arc_challenge.yaml b/configs/evals/arc_challenge.yaml index 504750e..a4aef61 100644 --- a/configs/evals/arc_challenge.yaml +++ b/configs/evals/arc_challenge.yaml @@ -56,7 +56,7 @@ languages: tasks: [arc_challenge_mt_lt] - language: lav_Latn tasks: [arc_challenge_mt_lv] - - language: nor_Latn + - language: nob_Latn tasks: [arc_challenge_mt_nb] - language: nld_Latn tasks: [arc_challenge_mt_nl] diff --git a/configs/evals/belebele.yaml b/configs/evals/belebele.yaml index a63a698..c6db715 100644 --- a/configs/evals/belebele.yaml +++ b/configs/evals/belebele.yaml @@ -50,7 +50,7 @@ languages: tasks: [belebele_mlt_Latn] - language: nld_Latn tasks: [belebele_nld_Latn] - - language: nor_Latn + - language: nob_Latn tasks: [belebele_nob_Latn] - language: pol_Latn tasks: [belebele_pol_Latn] diff --git a/configs/evals/flores200.yaml b/configs/evals/flores200.yaml index a72eb46..d06e0b9 100644 --- a/configs/evals/flores200.yaml +++ b/configs/evals/flores200.yaml @@ -119,7 +119,7 @@ languages: target_language: nld_Latn tasks: ['flores200:eng_Latn-nld_Latn'] - source_language: eng_Latn - target_language: nor_Latn + target_language: nob_Latn tasks: ['flores200:eng_Latn-nob_Latn'] - source_language: eng_Latn target_language: pol_Latn @@ -199,7 +199,7 @@ languages: - source_language: nld_Latn target_language: eng_Latn tasks: ['flores200:nld_Latn-eng_Latn'] - - source_language: nor_Latn + - source_language: nob_Latn target_language: eng_Latn tasks: ['flores200:nob_Latn-eng_Latn'] - source_language: pol_Latn diff --git a/configs/evals/global_piqa.yaml b/configs/evals/global_piqa.yaml index 8427342..f288608 100644 --- a/configs/evals/global_piqa.yaml +++ b/configs/evals/global_piqa.yaml @@ -4,7 +4,7 @@ match: regex: global_piqa_completions_.+ metric: acc_norm metric_filter: none -shots: 10 +shots: 0 score: scale: 1 normalize: diff --git a/configs/evals/ifeval.yaml b/configs/evals/ifeval.yaml index 9fab7d2..bdc8102 100644 --- a/configs/evals/ifeval.yaml +++ b/configs/evals/ifeval.yaml @@ -1,5 +1,5 @@ name: ifeval -category: Instruction following +category: Instruction match: name: ifeval metric: prompt_level_strict_acc diff --git a/configs/evals/mgsm.yaml b/configs/evals/mgsm.yaml index 857e2d0..1377d9d 100644 --- a/configs/evals/mgsm.yaml +++ b/configs/evals/mgsm.yaml @@ -2,6 +2,8 @@ name: mgsm category: Math match: regex: (?:mgsm_native_cot_.+|global_mgsm_.+) +select: + regex: global_mgsm_[a-z]+ metric: exact_match metric_filter: flexible-extract shots: 5 diff --git a/configs/evals/multiblimp.yaml b/configs/evals/multiblimp.yaml index 3f4b08c..1567d78 100644 --- a/configs/evals/multiblimp.yaml +++ b/configs/evals/multiblimp.yaml @@ -48,7 +48,7 @@ languages: tasks: [multiblimp_gle] - language: glg_Latn tasks: [multiblimp_glg] - - language: srp_Latn + - language: hbs_Latn tasks: [multiblimp_hbs] evidence: https://huggingface.co/datasets/jumelet/multiblimp/blob/main/hbs/data.tsv note: 'Grouping convention: assign the pooled Croatian/Serbian score to Serbian, the language with more diff --git a/configs/evals/piqa.yaml b/configs/evals/piqa.yaml index 892f3c5..f4ba234 100644 --- a/configs/evals/piqa.yaml +++ b/configs/evals/piqa.yaml @@ -4,7 +4,7 @@ match: name: piqa metric: acc_norm metric_filter: none -shots: 10 +shots: 0 score: scale: 1 normalize: diff --git a/configs/evals/sib200.yaml b/configs/evals/sib200.yaml index b1f6ac1..35c10b5 100644 --- a/configs/evals/sib200.yaml +++ b/configs/evals/sib200.yaml @@ -71,7 +71,7 @@ languages: tasks: [sib200_mlt_Latn] - language: nld_Latn tasks: [sib200_nld_Latn] - - language: nor_Latn + - language: nob_Latn tasks: [sib200_nob_Latn] - language: pol_Latn tasks: [sib200_pol_Latn]