diff --git a/evalscope/api/dataset/builder.py b/evalscope/api/dataset/builder.py index c633c09a3..2112d5785 100644 --- a/evalscope/api/dataset/builder.py +++ b/evalscope/api/dataset/builder.py @@ -1,11 +1,10 @@ -import copy from pathlib import Path from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Union from .dataset import DatasetDict, FieldSpec, MemoryDataset, Sample, validate_dataset_repeats from .hub import DatasetHub -from .loader import LocalDataLoader, _shuffle_in_place -from .utils import data_to_samples, record_to_sample_fn +from .loader import LocalDataLoader, _build_memory_dataset, _shuffle_in_place +from .utils import record_to_sample_fn if TYPE_CHECKING: from evalscope.api.benchmark import DataAdapter @@ -46,16 +45,18 @@ def build_dataset_from_records( elif isinstance(limit, int) and limit < 0: raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.') - # The limit is applied after filtering, so samples rejected by the filter do not use it up. data_to_sample = record_to_sample_fn(sample_fields) - samples = data_to_samples(data=record_list, data_to_sample=data_to_sample, filter_func=filter_func, limit=limit) - if repeats > 1: - samples = [copy.deepcopy(sample) for sample in samples for _ in range(repeats)] - - dataset = MemoryDataset(samples=samples, name=name, location=location, shuffled=shuffle) - if auto_id: - dataset.reindex(group_size=repeats) - return dataset + return _build_memory_dataset( + data=record_list, + data_to_sample=data_to_sample, + filter_func=filter_func, + limit=limit, + repeats=repeats, + auto_id=auto_id, + name=name, + location=location, + shuffled=shuffle, + ) def build_dataset_dict_from_record_map( diff --git a/evalscope/api/dataset/loader.py b/evalscope/api/dataset/loader.py index 539765cd0..24fb077c7 100644 --- a/evalscope/api/dataset/loader.py +++ b/evalscope/api/dataset/loader.py @@ -10,7 +10,7 @@ from functools import partial from pathlib import Path from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING, Callable, Dict, Iterator, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, Iterable, Iterator, List, Optional, Union from uuid import uuid4 from filelock import FileLock @@ -137,6 +137,31 @@ def repeat_samples(samples: list, repeats: int) -> list: return [copy.deepcopy(sample) for sample in samples for _ in range(repeats)] if repeats > 1 else samples +def _build_memory_dataset( + data: Iterable[dict], + data_to_sample: Callable, + *, + filter_func: Optional[Callable[[Sample], bool]], + limit: Optional[int], + repeats: int, + auto_id: bool, + name: Optional[str] = None, + location: Optional[str] = None, + shuffled: bool = False, + shuffle_choices: Optional[Union[bool, int]] = None, + seed: Optional[int] = None, +) -> MemoryDataset: + """Build a MemoryDataset with the shared conversion, repetition, and indexing pipeline.""" + samples = data_to_samples(data=data, data_to_sample=data_to_sample, filter_func=filter_func, limit=limit) + memory_dataset = MemoryDataset( + samples=repeat_samples(samples, repeats), name=name, location=location, shuffled=shuffled + ) + if auto_id: + memory_dataset.reindex(group_size=repeats) + shuffle_choices_if_requested(memory_dataset, shuffle_choices, seed) + return memory_dataset + + def _shuffle_in_place(data: list, seed: Optional[int]) -> None: """Shuffle a list in place with an optional seed. @@ -330,27 +355,20 @@ def _to_memory_dataset( if self.shuffle: dataset = dataset.shuffle(seed=self.seed) - # The limit is applied after filtering, so samples rejected by the filter do not use it up. resolved_limit = resolve_dataset_limit(self.limit, len(dataset)) - - # return the dataset - samples = data_to_samples( - data=dataset, data_to_sample=data_to_sample, filter_func=self.filter_func, limit=resolved_limit - ) - memory_dataset = MemoryDataset( - samples=repeat_samples(samples, self.repeats), + return _build_memory_dataset( + data=dataset, + data_to_sample=data_to_sample, + filter_func=self.filter_func, + limit=resolved_limit, + repeats=self.repeats, + auto_id=self.auto_id, name=Path(path).stem if Path(path).exists() else path, location=path, + shuffle_choices=self.shuffle_choices, + seed=self.seed, ) - # assign ids and group_ids if requested - if self.auto_id: - memory_dataset.reindex(group_size=self.repeats) - - shuffle_choices_if_requested(memory_dataset, self.shuffle_choices, self.seed) - - return memory_dataset - class LocalDataLoader(DataLoader): """ @@ -422,27 +440,20 @@ def load(self): if self.shuffle: _shuffle_in_place(dataset, self.seed) - # The limit is applied after filtering, so samples rejected by the filter do not use it up. resolved_limit = resolve_dataset_limit(self.limit, len(dataset)) - - # return the dataset - samples = data_to_samples( - data=dataset, data_to_sample=data_to_sample, filter_func=self.filter_func, limit=resolved_limit - ) - memory_dataset = MemoryDataset( - samples=repeat_samples(samples, self.repeats), + return _build_memory_dataset( + data=dataset, + data_to_sample=data_to_sample, + filter_func=self.filter_func, + limit=resolved_limit, + repeats=self.repeats, + auto_id=self.auto_id, name=Path(path).stem if Path(path).exists() else path, location=path, + shuffle_choices=self.shuffle_choices, + seed=self.seed, ) - # assign ids and group_ids if requested - if self.auto_id: - memory_dataset.reindex(group_size=self.repeats) - - shuffle_choices_if_requested(memory_dataset, self.shuffle_choices, self.seed) - - return memory_dataset - class DictDataLoader(DataLoader): """Load dataset from a list of dictionaries.""" @@ -459,19 +470,14 @@ def load(self) -> Dataset: if self.shuffle: _shuffle_in_place(dataset, self.seed) - # The limit is applied after filtering, so samples rejected by the filter do not use it up. resolved_limit = resolve_dataset_limit(self.limit, len(dataset)) - - # return the dataset - samples = data_to_samples( - data=dataset, data_to_sample=data_to_sample, filter_func=self.filter_func, limit=resolved_limit + return _build_memory_dataset( + data=dataset, + data_to_sample=data_to_sample, + filter_func=self.filter_func, + limit=resolved_limit, + repeats=self.repeats, + auto_id=self.auto_id, + shuffle_choices=self.shuffle_choices, + seed=self.seed, ) - memory_dataset = MemoryDataset(samples=repeat_samples(samples, self.repeats)) - - # assign ids and group_ids if requested - if self.auto_id: - memory_dataset.reindex(group_size=self.repeats) - - shuffle_choices_if_requested(memory_dataset, self.shuffle_choices, self.seed) - - return memory_dataset