From 000736a7033e43ae91557330f75810eaf2d1d2e1 Mon Sep 17 00:00:00 2001 From: Zahra Anvari Date: Sat, 4 Jul 2026 17:15:55 -0700 Subject: [PATCH] Fix evaluator task validation for text2text tasks --- src/evaluate/evaluator/__init__.py | 8 +++++--- tests/test_evaluator.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/evaluate/evaluator/__init__.py b/src/evaluate/evaluator/__init__.py index a2fe4be8a..7510d5ead 100644 --- a/src/evaluate/evaluator/__init__.py +++ b/src/evaluate/evaluator/__init__.py @@ -103,10 +103,12 @@ def check_task(task: str) -> Dict: """ if task in TASK_ALIASES: task = TASK_ALIASES[task] - if not check_pipeline_task(task): - raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.") - if task in SUPPORTED_EVALUATOR_TASKS.keys() and task in SUPPORTED_PIPELINE_TASKS.keys(): + if task in SUPPORTED_EVALUATOR_TASKS: return SUPPORTED_EVALUATOR_TASKS[task] + try: + check_pipeline_task(task) + except KeyError as e: + raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.") from e raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.") diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 259b5c7b9..1e3eeac42 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -921,6 +921,16 @@ def setUp(self): self.pipe = DummyText2TextGenerationPipeline() self.evaluator = evaluator("text2text-generation") + def test_text2text_evaluator_tasks_can_be_created(self): + for task, expected_class_name in [ + ("text2text-generation", "Text2TextGenerationEvaluator"), + ("summarization", "SummarizationEvaluator"), + ("translation", "TranslationEvaluator"), + ]: + e = evaluator(task) + self.assertEqual(type(e).__name__, expected_class_name) + self.assertEqual(e.task, task) + def test_pipe_init(self): results = self.evaluator.compute( model_or_pipeline=self.pipe,