Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions evalscope/api/dataset/builder.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import copy
from pathlib import Path
from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Union

from .dataset import DatasetDict, FieldSpec, MemoryDataset, Sample, validate_dataset_repeats
from .hub import DatasetHub
from .loader import LocalDataLoader, _shuffle_in_place
from .utils import data_to_samples, record_to_sample_fn
from .loader import LocalDataLoader, _build_memory_dataset, _shuffle_in_place
from .utils import record_to_sample_fn

if TYPE_CHECKING:
from evalscope.api.benchmark import DataAdapter
Expand Down Expand Up @@ -46,16 +45,18 @@ def build_dataset_from_records(
elif isinstance(limit, int) and limit < 0:
raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.')

# The limit is applied after filtering, so samples rejected by the filter do not use it up.
data_to_sample = record_to_sample_fn(sample_fields)
samples = data_to_samples(data=record_list, data_to_sample=data_to_sample, filter_func=filter_func, limit=limit)
if repeats > 1:
samples = [copy.deepcopy(sample) for sample in samples for _ in range(repeats)]

dataset = MemoryDataset(samples=samples, name=name, location=location, shuffled=shuffle)
if auto_id:
dataset.reindex(group_size=repeats)
return dataset
return _build_memory_dataset(
data=record_list,
data_to_sample=data_to_sample,
filter_func=filter_func,
limit=limit,
repeats=repeats,
auto_id=auto_id,
name=name,
location=location,
shuffled=shuffle,
)


def build_dataset_dict_from_record_map(
Expand Down
100 changes: 53 additions & 47 deletions evalscope/api/dataset/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from functools import partial
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING, Callable, Dict, Iterator, List, Optional, Union
from typing import TYPE_CHECKING, Callable, Dict, Iterable, Iterator, List, Optional, Union
from uuid import uuid4

from filelock import FileLock
Expand Down Expand Up @@ -137,6 +137,31 @@ def repeat_samples(samples: list, repeats: int) -> list:
return [copy.deepcopy(sample) for sample in samples for _ in range(repeats)] if repeats > 1 else samples


def _build_memory_dataset(
data: Iterable[dict],
data_to_sample: Callable,
*,
filter_func: Optional[Callable[[Sample], bool]],
limit: Optional[int],
repeats: int,
auto_id: bool,
name: Optional[str] = None,
location: Optional[str] = None,
shuffled: bool = False,
shuffle_choices: Optional[Union[bool, int]] = None,
seed: Optional[int] = None,
) -> MemoryDataset:
"""Build a MemoryDataset with the shared conversion, repetition, and indexing pipeline."""
samples = data_to_samples(data=data, data_to_sample=data_to_sample, filter_func=filter_func, limit=limit)
memory_dataset = MemoryDataset(
samples=repeat_samples(samples, repeats), name=name, location=location, shuffled=shuffled
)
if auto_id:
memory_dataset.reindex(group_size=repeats)
shuffle_choices_if_requested(memory_dataset, shuffle_choices, seed)
return memory_dataset


def _shuffle_in_place(data: list, seed: Optional[int]) -> None:
"""Shuffle a list in place with an optional seed.
Expand Down Expand Up @@ -330,27 +355,20 @@ def _to_memory_dataset(
if self.shuffle:
dataset = dataset.shuffle(seed=self.seed)

# The limit is applied after filtering, so samples rejected by the filter do not use it up.
resolved_limit = resolve_dataset_limit(self.limit, len(dataset))

# return the dataset
samples = data_to_samples(
data=dataset, data_to_sample=data_to_sample, filter_func=self.filter_func, limit=resolved_limit
)
memory_dataset = MemoryDataset(
samples=repeat_samples(samples, self.repeats),
return _build_memory_dataset(
data=dataset,
data_to_sample=data_to_sample,
filter_func=self.filter_func,
limit=resolved_limit,
repeats=self.repeats,
auto_id=self.auto_id,
name=Path(path).stem if Path(path).exists() else path,
location=path,
shuffle_choices=self.shuffle_choices,
seed=self.seed,
)

# assign ids and group_ids if requested
if self.auto_id:
memory_dataset.reindex(group_size=self.repeats)

shuffle_choices_if_requested(memory_dataset, self.shuffle_choices, self.seed)

return memory_dataset


class LocalDataLoader(DataLoader):
"""
Expand Down Expand Up @@ -422,27 +440,20 @@ def load(self):
if self.shuffle:
_shuffle_in_place(dataset, self.seed)

# The limit is applied after filtering, so samples rejected by the filter do not use it up.
resolved_limit = resolve_dataset_limit(self.limit, len(dataset))

# return the dataset
samples = data_to_samples(
data=dataset, data_to_sample=data_to_sample, filter_func=self.filter_func, limit=resolved_limit
)
memory_dataset = MemoryDataset(
samples=repeat_samples(samples, self.repeats),
return _build_memory_dataset(
data=dataset,
data_to_sample=data_to_sample,
filter_func=self.filter_func,
limit=resolved_limit,
repeats=self.repeats,
auto_id=self.auto_id,
name=Path(path).stem if Path(path).exists() else path,
location=path,
shuffle_choices=self.shuffle_choices,
seed=self.seed,
)

# assign ids and group_ids if requested
if self.auto_id:
memory_dataset.reindex(group_size=self.repeats)

shuffle_choices_if_requested(memory_dataset, self.shuffle_choices, self.seed)

return memory_dataset


class DictDataLoader(DataLoader):
"""Load dataset from a list of dictionaries."""
Expand All @@ -459,19 +470,14 @@ def load(self) -> Dataset:
if self.shuffle:
_shuffle_in_place(dataset, self.seed)

# The limit is applied after filtering, so samples rejected by the filter do not use it up.
resolved_limit = resolve_dataset_limit(self.limit, len(dataset))

# return the dataset
samples = data_to_samples(
data=dataset, data_to_sample=data_to_sample, filter_func=self.filter_func, limit=resolved_limit
return _build_memory_dataset(
data=dataset,
data_to_sample=data_to_sample,
filter_func=self.filter_func,
limit=resolved_limit,
repeats=self.repeats,
auto_id=self.auto_id,
shuffle_choices=self.shuffle_choices,
seed=self.seed,
)
memory_dataset = MemoryDataset(samples=repeat_samples(samples, self.repeats))

# assign ids and group_ids if requested
if self.auto_id:
memory_dataset.reindex(group_size=self.repeats)

shuffle_choices_if_requested(memory_dataset, self.shuffle_choices, self.seed)

return memory_dataset
Loading