From 03766f59fdc2298bb68dcc5201ced8ae78a53f0c Mon Sep 17 00:00:00 2001 From: SuryaSunil1326 <70116754+SuryaSunil1326@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:23:47 +0530 Subject: [PATCH] Add label_mapping support to TokenClassificationEvaluator --- .../evaluator/token_classification.py | 77 ++++- tests/test_evaluator.py | 268 +++++++++++++++--- 2 files changed, 288 insertions(+), 57 deletions(-) diff --git a/src/evaluate/evaluator/token_classification.py b/src/evaluate/evaluator/token_classification.py index bb711192..cf5b2935 100644 --- a/src/evaluate/evaluator/token_classification.py +++ b/src/evaluate/evaluator/token_classification.py @@ -96,7 +96,13 @@ class TokenClassificationEvaluator(Evaluator): def __init__(self, task="token-classification", default_metric_name=None): super().__init__(task, default_metric_name=default_metric_name) - def predictions_processor(self, predictions: List[List[Dict]], words: List[List[str]], join_by: str): + def predictions_processor( + self, + predictions: List[List[Dict]], + words: List[List[str]], + join_by: str, + label_mapping: Optional[Dict[str, str]] = None, + ): """ Transform the pipeline predictions into a list of predicted labels of the same length as the true labels. @@ -107,6 +113,9 @@ def predictions_processor(self, predictions: List[List[Dict]], words: List[List[ Original input data to the pipeline, used to build predicted labels of the same length. join_by (`str`): String to use to join two words. In English, it will typically be " ". + label_mapping (`Dict[str, str]`, *optional*, defaults to `None`): + Mapping from pipeline output labels to labels in the dataset. For example, + `{"LABEL_0": "O", "LABEL_1": "B-PER"}`. Returns: `dict`: a dictionary holding the predictions @@ -127,9 +136,15 @@ def predictions_processor(self, predictions: List[List[Dict]], words: List[List[ token_index += 1 if prediction[token_index]["start"] > word_offset[0]: # bad indexing - pred_processed.append("O") + entity = "O" elif prediction[token_index]["start"] == word_offset[0]: - pred_processed.append(prediction[token_index]["entity"]) + entity = prediction[token_index]["entity"] + else: + continue + + if label_mapping is not None: + entity = label_mapping.get(entity, entity) + pred_processed.append(entity) preds.append(pred_processed) @@ -158,7 +173,13 @@ def words_to_offsets(self, words: List[str], join_by: str): return offsets - def prepare_data(self, data: Union[str, Dataset], input_column: str, label_column: str, join_by: str): + def prepare_data( + self, + data: Union[str, Dataset], + input_column: str, + label_column: str, + join_by: str, + ): super().prepare_data(data, input_column, label_column) if not isinstance(data.features[input_column], Sequence) or not isinstance( @@ -172,9 +193,14 @@ def prepare_data(self, data: Union[str, Dataset], input_column: str, label_colum # Otherwise, we have to get the list of labels manually. labels_are_int = isinstance(data.features[label_column].feature, ClassLabel) if labels_are_int: - label_list = data.features[label_column].feature.names # list of string labels + label_list = data.features[ + label_column + ].feature.names # list of string labels id_to_label = {i: label for i, label in enumerate(label_list)} - references = [[id_to_label[label_id] for label_id in label_ids] for label_ids in data[label_column]] + references = [ + [id_to_label[label_id] for label_id in label_ids] + for label_ids in data[label_column] + ] elif data.features[label_column].feature.dtype.startswith("int"): raise NotImplementedError( "References provided as integers, but the reference column is not a Sequence of ClassLabels." @@ -192,12 +218,20 @@ def prepare_data(self, data: Union[str, Dataset], input_column: str, label_colum def prepare_pipeline( self, - model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"], # noqa: F821 - tokenizer: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 - feature_extractor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" + ], # noqa: F821 + tokenizer: Union[ + "PreTrainedTokenizerBase", "FeatureExtractionMixin" + ] = None, # noqa: F821 + feature_extractor: Union[ + "PreTrainedTokenizerBase", "FeatureExtractionMixin" + ] = None, # noqa: F821 device: int = None, ): - pipe = super().prepare_pipeline(model_or_pipeline, tokenizer, feature_extractor, device) + pipe = super().prepare_pipeline( + model_or_pipeline, tokenizer, feature_extractor, device + ) # check the pipeline outputs start characters in its predictions dummy_output = pipe(["2003 New York Gregory"], **self.PIPELINE_KWARGS) @@ -214,7 +248,11 @@ def prepare_pipeline( def compute( self, model_or_pipeline: Union[ - str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + str, + "Pipeline", + Callable, + "PreTrainedModel", + "TFPreTrainedModel", # noqa: F821 ] = None, data: Union[str, Dataset] = None, subset: Optional[str] = None, @@ -229,6 +267,7 @@ def compute( input_column: str = "tokens", label_column: str = "ner_tags", join_by: Optional[str] = " ", + label_mapping: Optional[Dict[str, str]] = None, ) -> Tuple[Dict[str, float], Any]: """ input_column (`str`, defaults to `"tokens"`): @@ -238,6 +277,9 @@ def compute( join_by (`str`, *optional*, defaults to `" "`): This evaluator supports dataset whose input column is a list of words. This parameter specifies how to join words to generate a string input. This is especially useful for languages that do not separate words by a space. + label_mapping (`Dict[str, str]`, *optional*, defaults to `None`): + Mapping from pipeline output labels to labels in the dataset. For example, + `{"LABEL_0": "O", "LABEL_1": "B-PER"}`. """ result = {} @@ -246,14 +288,21 @@ def compute( # Prepare inputs data = self.load_data(data=data, subset=subset, split=split) metric_inputs, pipe_inputs = self.prepare_data( - data=data, input_column=input_column, label_column=label_column, join_by=join_by + data=data, + input_column=input_column, + label_column=label_column, + join_by=join_by, + ) + pipe = self.prepare_pipeline( + model_or_pipeline=model_or_pipeline, tokenizer=tokenizer, device=device ) - pipe = self.prepare_pipeline(model_or_pipeline=model_or_pipeline, tokenizer=tokenizer, device=device) metric = self.prepare_metric(metric) # Compute predictions predictions, perf_results = self.call_pipeline(pipe, pipe_inputs) - predictions = self.predictions_processor(predictions, data[input_column], join_by) + predictions = self.predictions_processor( + predictions, data[input_column], join_by, label_mapping + ) metric_inputs.update(predictions) # Compute metrics from references and predictions diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 259b5c7b..9243f527 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -50,13 +50,21 @@ class DummyTextGenerationPipeline: - def __init__(self, prefix="generated", task="text-generation", num_return_sequences=1): + def __init__( + self, prefix="generated", task="text-generation", num_return_sequences=1 + ): self.task = task self.prefix = prefix self.num_return_sequences = num_return_sequences def __call__(self, inputs, **kwargs): - return [[{f"{self.prefix}_text": "Lorem ipsum"} for _ in range(self.num_return_sequences)] for _ in inputs] + return [ + [ + {f"{self.prefix}_text": "Lorem ipsum"} + for _ in range(self.num_return_sequences) + ] + for _ in inputs + ] class DummyText2TextGenerationPipeline: @@ -76,7 +84,10 @@ def __init__(self, sleep_time=None): def __call__(self, inputs, **kwargs): if self.sleep_time is not None: sleep(self.sleep_time) - return [{"label": "NEGATIVE"} if i % 2 == 1 else {"label": "POSITIVE"} for i, _ in enumerate(inputs)] + return [ + {"label": "NEGATIVE"} if i % 2 == 1 else {"label": "POSITIVE"} + for i, _ in enumerate(inputs) + ] class DummyImageClassificationPipeline: @@ -84,7 +95,10 @@ def __init__(self): self.task = "image-classification" def __call__(self, images, **kwargs): - return [[{"score": 0.9, "label": "yurt"}, {"score": 0.1, "label": "umbrella"}] for i, _ in enumerate(images)] + return [ + [{"score": 0.9, "label": "yurt"}, {"score": 0.1, "label": "umbrella"}] + for i, _ in enumerate(images) + ] class DummyQuestionAnsweringPipeline: @@ -95,13 +109,18 @@ def __init__(self, v2: bool): def __call__(self, question, context, **kwargs): if self.v2: return [ - {"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} - if i % 2 == 0 - else {"score": 0.95, "start": 0, "end": 0, "answer": ""} + ( + {"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} + if i % 2 == 0 + else {"score": 0.95, "start": 0, "end": 0, "answer": ""} + ) for i in range(len(question)) ] else: - return [{"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} for _ in question] + return [ + {"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} + for _ in question + ] class DummyTokenClassificationPipeline: @@ -135,18 +154,31 @@ def __init__(self): self.task = "audio-classification" def __call__(self, audio, **kwargs): - return [[{"score": 0.9, "label": "yes"}, {"score": 0.1, "label": "no"}] for i, _ in enumerate(audio)] + return [ + [{"score": 0.9, "label": "yes"}, {"score": 0.1, "label": "no"}] + for i, _ in enumerate(audio) + ] class TestEvaluator(TestCase): def setUp(self): - self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.data = Dataset.from_dict( + {"label": [1, 0], "text": ["great movie", "horrible movie"]} + ) self.default_ckpt = "hf-internal-testing/tiny-random-bert" - self.default_model = AutoModelForSequenceClassification.from_pretrained(self.default_ckpt, num_labels=2) + self.default_model = AutoModelForSequenceClassification.from_pretrained( + self.default_ckpt, num_labels=2 + ) self.default_tokenizer = AutoTokenizer.from_pretrained(self.default_ckpt) - self.pipe = pipeline("text-classification", model=self.default_model, tokenizer=self.default_tokenizer) + self.pipe = pipeline( + "text-classification", + model=self.default_model, + tokenizer=self.default_tokenizer, + ) self.evaluator = evaluator("text-classification") - self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.data = Dataset.from_dict( + {"label": [1, 0], "text": ["great movie", "horrible movie"]} + ) self.label_mapping = {"LABEL_0": 0.0, "LABEL_1": 1.0} def test_wrong_task(self): @@ -200,14 +232,20 @@ def import_pt_tf_mock(name, *args): # pt accelerator found and pipeline instantiated on CPU pt_mock.cuda.is_available.return_value = True self.assertRaises( - ValueError, Evaluator.check_for_mismatch_in_device_setup, Evaluator._infer_device(), self.pipe + ValueError, + Evaluator.check_for_mismatch_in_device_setup, + Evaluator._infer_device(), + self.pipe, ) # tf accelerator found and pipeline instantiated on CPU pt_available = False tf_available = True self.assertRaises( - ValueError, Evaluator.check_for_mismatch_in_device_setup, Evaluator._infer_device(), self.pipe + ValueError, + Evaluator.check_for_mismatch_in_device_setup, + Evaluator._infer_device(), + self.pipe, ) def test_pipe_init(self): @@ -241,7 +279,9 @@ def test_model_str_init(self): class TestTextClassificationEvaluator(TestCase): def setUp(self): - self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.data = Dataset.from_dict( + {"label": [1, 0], "text": ["great movie", "horrible movie"]} + ) self.default_model = "lvwerra/distilbert-imdb" self.input_column = "text" self.label_column = "label" @@ -309,11 +349,21 @@ def test_data_loading(self): # Test passing in dataset by name with split data = self.evaluator.load_data("evaluate/imdb-ci", split="test[:1]") - self.evaluator.prepare_data(data=data, input_column="text", label_column="label", second_input_column=None) + self.evaluator.prepare_data( + data=data, + input_column="text", + label_column="label", + second_input_column=None, + ) # Test passing in dataset by name without split and inferring the optimal split data = self.evaluator.load_data("evaluate/imdb-ci") - self.evaluator.prepare_data(data=data, input_column="text", label_column="label", second_input_column=None) + self.evaluator.prepare_data( + data=data, + input_column="text", + label_column="label", + second_input_column=None, + ) # Test that it chooses the correct one (e.g. imdb only has train and test, but no validation) self.assertEqual(data.split, "test") @@ -347,7 +397,12 @@ def test_overwrite_default_metric(self): self.assertEqual(results["accuracy"], 1.0) def test_bootstrap(self): - data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]}) + data = Dataset.from_dict( + { + "label": [1, 0, 0], + "text": ["great movie", "great movie", "horrible movie"], + } + ) results = self.evaluator.compute( model_or_pipeline=self.pipe, @@ -359,7 +414,9 @@ def test_bootstrap(self): random_state=0, ) self.assertAlmostEqual(results["accuracy"]["score"], 0.666666, 5) - self.assertAlmostEqual(results["accuracy"]["confidence_interval"][0], 0.33557, 5) + self.assertAlmostEqual( + results["accuracy"]["confidence_interval"][0], 0.33557, 5 + ) self.assertAlmostEqual(results["accuracy"]["confidence_interval"][1], 1.0, 5) self.assertAlmostEqual(results["accuracy"]["standard_error"], 0.22498, 5) @@ -376,11 +433,24 @@ def test_perf(self): ) self.assertEqual(results["accuracy"], 1.0) self.assertAlmostEqual(results["total_time_in_seconds"], 0.1, 1) - self.assertAlmostEqual(results["samples_per_second"], len(self.data) / results["total_time_in_seconds"], 5) - self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(self.data), 5) + self.assertAlmostEqual( + results["samples_per_second"], + len(self.data) / results["total_time_in_seconds"], + 5, + ) + self.assertAlmostEqual( + results["latency_in_seconds"], + results["total_time_in_seconds"] / len(self.data), + 5, + ) def test_bootstrap_and_perf(self): - data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]}) + data = Dataset.from_dict( + { + "label": [1, 0, 0], + "text": ["great movie", "great movie", "horrible movie"], + } + ) results = self.evaluator.compute( model_or_pipeline=self.perf_pipe, @@ -394,12 +464,22 @@ def test_bootstrap_and_perf(self): random_state=0, ) self.assertAlmostEqual(results["accuracy"]["score"], 0.666666, 5) - self.assertAlmostEqual(results["accuracy"]["confidence_interval"][0], 0.33557, 5) + self.assertAlmostEqual( + results["accuracy"]["confidence_interval"][0], 0.33557, 5 + ) self.assertAlmostEqual(results["accuracy"]["confidence_interval"][1], 1.0, 5) self.assertAlmostEqual(results["accuracy"]["standard_error"], 0.22498285, 5) self.assertAlmostEqual(results["total_time_in_seconds"], 0.1, 1) - self.assertAlmostEqual(results["samples_per_second"], len(data) / results["total_time_in_seconds"], 5) - self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(data), 5) + self.assertAlmostEqual( + results["samples_per_second"], + len(data) / results["total_time_in_seconds"], + 5, + ) + self.assertAlmostEqual( + results["latency_in_seconds"], + results["total_time_in_seconds"] / len(data), + 5, + ) class TestTextClassificationEvaluatorTwoColumns(TestCase): @@ -464,7 +544,10 @@ def setUp(self): self.data = Dataset.from_dict( { "label": [2, 2], - "image": [Image.new("RGB", (500, 500), (255, 255, 255)), Image.new("RGB", (500, 500), (170, 95, 170))], + "image": [ + Image.new("RGB", (500, 500), (255, 255, 255)), + Image.new("RGB", (500, 500), (170, 95, 170)), + ], } ) self.default_model = "lysandre/tiny-vit-random" @@ -546,16 +629,28 @@ def setUp(self): self.data = Dataset.from_dict( { "id": ["56be4db0acb8001400a502ec", "56be4db0acb8001400a502ed"], - "context": ["My name is Felix and I love cookies!", "Misa name is Felix and misa love cookies!"], - "answers": [{"text": ["Felix"], "answer_start": [11]}, {"text": ["Felix"], "answer_start": [13]}], + "context": [ + "My name is Felix and I love cookies!", + "Misa name is Felix and misa love cookies!", + ], + "answers": [ + {"text": ["Felix"], "answer_start": [11]}, + {"text": ["Felix"], "answer_start": [13]}, + ], "question": ["What is my name?", "What is my name?"], } ) self.data_v2 = Dataset.from_dict( { "id": ["56be4db0acb8001400a502ec", "56be4db0acb8001400a502ed"], - "context": ["My name is Felix and I love cookies!", "Let's explore the city!"], - "answers": [{"text": ["Felix"], "answer_start": [11]}, {"text": [], "answer_start": []}], + "context": [ + "My name is Felix and I love cookies!", + "Let's explore the city!", + ], + "answers": [ + {"text": ["Felix"], "answer_start": [11]}, + {"text": [], "answer_start": []}, + ], "question": ["What is my name?", "What is my name?"], } ) @@ -622,7 +717,8 @@ def test_class_init(self): metric="squad_v2", ) self.assertDictEqual( - {key: results[key] for key in ["HasAns_f1", "NoAns_f1"]}, {"HasAns_f1": 100.0, "NoAns_f1": 100.0} + {key: results[key] for key in ["HasAns_f1", "NoAns_f1"]}, + {"HasAns_f1": 100.0, "NoAns_f1": 100.0}, ) @slow @@ -640,20 +736,29 @@ def test_default_pipe_init(self): metric="squad_v2", ) self.assertDictEqual( - {key: results[key] for key in ["HasAns_f1", "NoAns_f1"]}, {"HasAns_f1": 100.0, "NoAns_f1": 0.0} + {key: results[key] for key in ["HasAns_f1", "NoAns_f1"]}, + {"HasAns_f1": 100.0, "NoAns_f1": 0.0}, ) def test_data_loading(self): # Test passing in dataset by name with data_split data = self.evaluator.load_data("evaluate/squad-ci", split="validation[:1]") self.evaluator.prepare_data( - data=data, question_column="question", context_column="context", id_column="id", label_column="answers" + data=data, + question_column="question", + context_column="context", + id_column="id", + label_column="answers", ) # Test passing in dataset by name without data_split and inferring the optimal split data = self.evaluator.load_data("evaluate/squad-ci") self.evaluator.prepare_data( - data=data, question_column="question", context_column="context", id_column="id", label_column="answers" + data=data, + question_column="question", + context_column="context", + id_column="id", + label_column="answers", ) # Test that it chooses the correct one (e.g. squad only has train and validation, but no test) @@ -818,7 +923,9 @@ def test_predictions_processor(self): ] ] predictions = task_evaluator.predictions_processor(predictions, words, join_by) - self.assertListEqual(predictions["predictions"][0], ["B-LOC", "I-LOC", "O", "O", "B-LOC", "O"]) + self.assertListEqual( + predictions["predictions"][0], ["B-LOC", "I-LOC", "O", "O", "B-LOC", "O"] + ) # non-aligned start and words predictions = [ @@ -832,7 +939,9 @@ def test_predictions_processor(self): ] ] predictions = task_evaluator.predictions_processor(predictions, words, join_by) - self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + self.assertListEqual( + predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"] + ) # non-aligned start and words predictions = [ @@ -846,7 +955,9 @@ def test_predictions_processor(self): ] ] predictions = task_evaluator.predictions_processor(predictions, words, join_by) - self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + self.assertListEqual( + predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"] + ) # non-aligned start and words predictions = [ @@ -859,7 +970,60 @@ def test_predictions_processor(self): ] ] predictions = task_evaluator.predictions_processor(predictions, words, join_by) - self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + self.assertListEqual( + predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"] + ) + + def test_predictions_processor_with_label_mapping(self): + task_evaluator = evaluator("token-classification") + join_by = " " + words = [["New", "York", "a", "nice", "City", "."]] + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 2, "entity": "I-LOC"}, + {"start": 4, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + label_mapping = {"B-LOC": "B-LOCATION", "I-LOC": "I-LOCATION"} + predictions = task_evaluator.predictions_processor( + predictions, words, join_by, label_mapping + ) + # B-LOC and I-LOC are remapped; O is not in label_mapping so it falls back to "O" + self.assertListEqual( + predictions["predictions"][0], + ["B-LOCATION", "I-LOCATION", "O", "O", "B-LOCATION", "O"], + ) + + def test_label_mapping_in_compute(self): + # Dataset uses alternative label names ("B-LOCATION" / "I-LOCATION") while the + # pipeline outputs the shorthand names ("B-LOC" / "I-LOC"). Without label_mapping + # the predictions would never match the references; with it they should match perfectly. + features = Features( + { + "tokens": Sequence(feature=Value(dtype="string")), + "ner_tags": Sequence(feature=Value(dtype="string")), + } + ) + data = Dataset.from_dict( + { + "tokens": [["New", "York", "a", "nice", "City", "."]], + "ner_tags": [["B-LOCATION", "I-LOCATION", "O", "O", "B-LOCATION", "O"]], + }, + features=features, + ) + label_mapping = {"B-LOC": "B-LOCATION", "I-LOC": "I-LOCATION"} + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=data, + metric="seqeval", + label_mapping=label_mapping, + ) + self.assertEqual(results["overall_accuracy"], 1.0) class TestTextGenerationEvaluator(TestCase): @@ -971,7 +1135,9 @@ def test_summarization(self): self.assertEqual(results["rouge1"], 1.0) def test_translation(self): - pipe = DummyText2TextGenerationPipeline(task="translation", prefix="translation") + pipe = DummyText2TextGenerationPipeline( + task="translation", prefix="translation" + ) e = evaluator("translation") results = e.compute( @@ -1045,13 +1211,26 @@ def test_overwrite_default_metric(self): class TestAudioClassificationEvaluator(TestCase): def setUp(self): self.data = Dataset.from_dict( - {"file": ["https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/1.flac"], "label": [11]} + { + "file": [ + "https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/1.flac" + ], + "label": [11], + } ) self.raw_data = Dataset.from_dict( { "audio": [ np.array( - [-0.00048828, -0.00018311, -0.00137329, 0.00079346, 0.00091553, 0.00085449], dtype=np.float32 + [ + -0.00048828, + -0.00018311, + -0.00137329, + 0.00079346, + 0.00091553, + 0.00085449, + ], + dtype=np.float32, ) ], "label": [11], @@ -1072,7 +1251,10 @@ def test_pipe_init(self): def test_raw_pipe_init(self): results = self.evaluator.compute( - model_or_pipeline=self.pipe, data=self.raw_data, label_mapping=self.label_mapping, input_column="audio" + model_or_pipeline=self.pipe, + data=self.raw_data, + label_mapping=self.label_mapping, + input_column="audio", ) self.assertEqual(results["accuracy"], 0)