Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -324,13 +324,14 @@ class OnlineBinPacker(FilePartitioner):
:meth:`has_partition` / :meth:`next_partition` as they become available;
call :meth:`finalize` once all input is added to flush the still-open bins.

By default it packs globally, with one pool of open bins keyed by file, so
it needs every listing row (``requires_global_input=True``). With
``requires_global_input=False`` each listing task packs only its own shard
of files, and ``partition_files`` flushes the open bins when the task ends.
Listing then runs as parallel tasks, at the cost of at most one under-filled
bin per shard. A file's rows never cross shards, because shards split the
path list.
By default it packs globally, so it needs every listing row
(``requires_global_input=True``). Setting ``requires_global_input=False``
lets listing run as parallel tasks: the listing planner
(``plan_list_files_op``) splits the paths passed to ``read_*`` evenly across
up to ``DEFAULT_MAX_NUM_LIST_FILES_TASKS`` (200) tasks, and a folder counts
as one path. It still uses one task when the read shuffles files. The cost
is that bins may not fill as well, since each task packs only its own
files, which may lead to more read tasks.
"""

def __init__(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterable,
Iterator,
Expand Down Expand Up @@ -49,11 +50,14 @@ class _Declared:
Starts as the summary's records and grows with those met inside chunks, so
a channel declared only in an earlier chunk serves the later ones.
``scanned`` holds the offsets of the chunks already read back for records.
``file_summary`` returns the file's whole summary, whose chunk indexes say
where to read back.
"""

channels: Dict[int, "Channel"]
schemas: Dict[int, "Schema"]
scanned: Set[int]
file_summary: Callable[[], "Summary"]


def _scan_declarations(f: Any, chunk_index: "ChunkIndex", declared: _Declared) -> None:
Expand Down Expand Up @@ -90,11 +94,14 @@ def iter_chunks(
path: str,
summary: "Summary",
offsets: Optional[Set[int]],
file_summary: Optional[Callable[[], "Summary"]] = None,
) -> Iterator[_Selected]:
"""Yield the selected messages of the owned chunks of an indexed file.

Chunks are read in file order, or merged by log time with
``log_time_order``.
``log_time_order``. ``summary`` may index only the owned chunks, as a
read task's summary does (``mcap_task_summary``). Then ``file_summary``
returns the whole summary, which reading back a declaration needs.
"""
selected = self._selection.selected_channel_ids(
summary.channels, summary.schemas
Expand All @@ -104,7 +111,12 @@ def iter_chunks(
for c in self.candidate_chunks(summary, selected)
if offsets is None or c.chunk_start_offset in offsets
]
declared = _Declared(dict(summary.channels), dict(summary.schemas), set())
declared = _Declared(
dict(summary.channels),
dict(summary.schemas),
set(),
file_summary or (lambda: summary),
)
if not self._log_time_order:
for chunk_index in chunk_indexes:
yield from self._read_chunk(
Expand Down Expand Up @@ -206,7 +218,7 @@ def _read_chunk(
if isinstance(record, Message):
index += 1
channel, schema = self._channel_and_schema(
f, path, summary, chunk_index, declared, record
f, path, chunk_index, declared, record
)
if not self._channel_is_selected(channel, schema, summary, selected):
continue
Expand All @@ -224,7 +236,6 @@ def _channel_and_schema(
self,
f: Any,
path: str,
summary: "Summary",
chunk_index: "ChunkIndex",
declared: _Declared,
message: "Message",
Expand All @@ -236,7 +247,7 @@ def _channel_and_schema(
channel = declared.channels.get(message.channel_id)
if channel is None:
self._find_in_earlier_chunks(
f, summary, chunk_index, declared, channel_id=message.channel_id
f, chunk_index, declared, channel_id=message.channel_id
)
channel = declared.channels.get(message.channel_id)
if channel is None:
Expand All @@ -250,7 +261,7 @@ def _channel_and_schema(
# (``repeat_schemas=False``). ``message_types`` and the schema
# columns need it.
self._find_in_earlier_chunks(
f, summary, chunk_index, declared, schema_id=channel.schema_id
f, chunk_index, declared, schema_id=channel.schema_id
)
schema = declared.schemas.get(channel.schema_id) if channel.schema_id else None
return channel, schema
Expand Down Expand Up @@ -281,7 +292,6 @@ def _channel_is_selected(
def _find_in_earlier_chunks(
self,
f: Any,
summary: "Summary",
chunk_index: "ChunkIndex",
declared: "_Declared",
*,
Expand All @@ -299,7 +309,7 @@ def _find_in_earlier_chunks(
earlier = sorted(
(
c
for c in summary.chunk_indexes
for c in declared.file_summary().chunk_indexes
if c.chunk_start_offset < chunk_index.chunk_start_offset
and c.chunk_start_offset not in declared.scanned
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -342,13 +342,13 @@ def _summaries(
"""
for path, summary_read in read_summaries(filesystem, paths, lambda path: path):
try:
summary = summary_read.result()
listed = summary_read.result()
except Exception as exc: # noqa: BLE001 - not an MCAP file
if not skip_unreadable:
raise
logger.debug("Skipping %s for the schema: %s", path, exc)
continue
yield path, summary
yield path, None if listed is None else listed.summary


def _paths_past_sample(listed: Iterable[str], sampled: Set[str]) -> List[str]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
chunks into read tasks of about ``RAY_DATA_MCAP_BIN_PACKING_BYTES`` of that
estimate (128 MiB by default). When the read keeps order, ``OrderedFileGrouper``
first joins each file's chunks so that one task reads the file (``mcap_file_cuts``).
``TaskSummaryPartitioner`` adds to each task the summary records it needs.
``MCAPScanner`` holds the optimizer's pushdowns and builds the ``MCAPReader``,
which seeks to each task's chunks. ``infer_schema`` calls ``infer_data_type``
(``mcap_data_column``) to decide whether ``data`` holds JSON values or bytes.
Expand Down Expand Up @@ -51,6 +52,9 @@
TimeRange,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_scanner import MCAPScanner
from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import (
TaskSummaryPartitioner,
)
from ray.data._internal.datasource_v2.interfaces.datasource_v2 import (
DatasourceCategory,
FileDataSourceV2,
Expand Down Expand Up @@ -175,7 +179,7 @@ def get_file_partitioner(self, **kwargs):
#
# Packing is per listing shard, not global, so listing can run as many
# parallel tasks. Each chunk still lands in exactly one read task; the
# cost is at most one under-filled task per shard.
# cost is that bins may not fill as well, which may mean more read tasks.
from ray.data._internal.datasource_v2.common.online_bin_packer import (
OnlineBinPacker,
)
Expand All @@ -185,7 +189,9 @@ def get_file_partitioner(self, **kwargs):
max_shared_open_bins=_BIN_PACKING_MAX_SHARED_OPEN_BINS,
requires_global_input=False,
)
return OrderedFileGrouper(packer, budget=_BIN_PACKING_BYTES)
return TaskSummaryPartitioner(
OrderedFileGrouper(packer, budget=_BIN_PACKING_BYTES)
)

@property
@override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,11 @@ def _read_keeps_order() -> bool:


def _rows_by_file(manifest: FileManifest) -> Iterator[FileManifest]:
"""The manifest's rows grouped by file, in listing order."""
"""The manifest's rows grouped by file, in listing order.

A listing manifest holds one listed file, so a path listed twice arrives as
two manifests and its copies stay apart.
"""
rows_by_path: Dict[str, List[int]] = {}
for i, path in enumerate(manifest.paths):
rows_by_path.setdefault(str(path), []).append(i)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
A file whose summary is missing, has no chunk index or lists no channels is
listed as one whole-file row.

Each row also carries the summary records its read task will need, so read
tasks do not read summaries (``mcap_task_summary``).

A file whose summary cannot be read, such as a recording cut short, does not
fail the listing. It is listed as one row that names no chunk, which the reader
takes as the whole file, and that weighs a whole read task, so the packer puts
Expand Down Expand Up @@ -39,10 +42,15 @@
from ray.data._internal.datasource_v2.formats.mcap.mcap_options import MCAPSelection
from ray.data._internal.datasource_v2.formats.mcap.mcap_summary import (
_DEFAULT_SUMMARY_IO_CONCURRENCY,
ListedSummary,
chunk_run,
chunk_unit_id,
read_summaries,
)
from ray.data._internal.datasource_v2.formats.mcap.mcap_task_summary import (
split_section,
with_listing_records,
)
from ray.data._internal.datasource_v2.interfaces.file_indexer import FileInfo
from ray.data._internal.datasource_v2.interfaces.file_manifest import (
FileChunk,
Expand All @@ -52,7 +60,6 @@
if TYPE_CHECKING:
from concurrent.futures import Future

from mcap.summary import Summary
from pyarrow.fs import FileSystem

from ray.data._internal.datasource_v2.interfaces.file_pruner import FilePruner
Expand Down Expand Up @@ -135,17 +142,17 @@ def list_files(
excluded = excluded_read_unit_ids or frozenset()
for file_info, summary_read in self._read_summaries(file_infos, filesystem):
try:
summary = summary_read.result()
listed = summary_read.result()
except Exception as e: # noqa: BLE001 - the file's own read task fails
manifest = self._unreadable_file_manifest(file_info, e)
else:
manifest = self._manifest_for_file(file_info, summary, excluded)
manifest = self._manifest_for_file(file_info, listed, excluded)
if manifest is not None:
yield manifest

def _read_summaries(
self, file_infos: Iterable[FileInfo], filesystem: "FileSystem"
) -> Iterator[Tuple[FileInfo, "Future[Optional[Summary]]"]]:
) -> Iterator[Tuple[FileInfo, "Future[Optional[ListedSummary]]"]]:
"""Start summary reads concurrently, yielding them in listing order.

Listing order decides how the partitioner groups chunks into tasks, so
Expand All @@ -172,16 +179,18 @@ def _unreadable_file_manifest(
size_bytes=self._read_task_bytes,
fully_matched=False,
)
return FileManifest.construct_manifest(
manifest = FileManifest.construct_manifest(
paths=[file_info.path],
sizes=[file_info.size],
chunk_metadatas=[whole_file.to_metadata()],
)
# No records: the read task reads the summary itself and fails on it.
return with_listing_records(manifest, None, [None])

def _manifest_for_file(
self,
file_info: FileInfo,
summary: Optional["Summary"],
listed: Optional[ListedSummary],
excluded_read_unit_ids: AbstractSet[str],
) -> Optional[FileManifest]:
"""The listing rows of one file, or ``None`` when it cannot match.
Expand All @@ -193,13 +202,16 @@ def _manifest_for_file(
"""
path = file_info.path
assert file_info.size is not None
summary = listed.summary if listed is not None else None

if summary is None or not summary.chunk_indexes or not summary.channels:
# Without a summary, a chunk index or channel records, nothing says
# which chunk holds what.
return FileManifest.construct_manifest(
# which chunk holds what. The read task gets the whole summary.
manifest = FileManifest.construct_manifest(
paths=[path], sizes=[file_info.size], chunk_metadatas=[None]
)
records = b"" if listed is None else bytes(listed.section)
return with_listing_records(manifest, records, [None])

selected = self._selection.selected_channel_ids(
summary.channels, summary.schemas
Expand All @@ -223,8 +235,15 @@ def _manifest_for_file(
]
if not runs:
return None
return FileManifest.construct_manifest(
manifest = FileManifest.construct_manifest(
paths=[path] * len(runs),
sizes=[file_info.size] * len(runs),
chunk_metadatas=[run.to_metadata() for run in runs],
)
assert listed is not None
records = split_section(listed.section)
return with_listing_records(
manifest,
records.declarations,
[records.chunk_indexes[run.unit_ids[0]] for run in runs],
)
Loading
Loading