From dae96ec0707ca0bd192c5ddd23a81f279ec46b64 Mon Sep 17 00:00:00 2001 From: "jiayi.wang" Date: Fri, 9 Oct 2026 15:04:17 +0000 Subject: [PATCH] parquet@py: write a page index and page checksums by default pyarrow's own defaults write neither a ColumnIndex/OffsetIndex nor page CRCs, so every catalog Parquet written by the default writer had no page index, unlike parquet-java and arrow-rs. With RAINCLOUD_PARQUET_PAGE_INDEX and RAINCLOUD_PARQUET_PAGE_CHECKSUMS unset, parquet@py now passes write_page_index (when write.statistics is on) and write_page_checksum, matching parquet-java's defaults. Setting either to 0 restores the old files. Every parquet@py artifact changes bytes and checksums. Co-authored-by: Isaac --- AGENTS.md | 9 +++++---- CHANGELOG.md | 12 ++++++++++++ README.md | 11 ++++++----- raincloud/pipeline/export/exporters.py | 14 +++++++++----- raincloud/pipeline/spec.py | 7 +++++-- sidecars/README.md | 9 +++++---- tests/test_parquet_options.py | 13 +++++++++++++ 7 files changed, 55 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e31447a..dcb7cea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -149,8 +149,9 @@ Every Parquet writer is given one set of options (`spec.parquet_options`): the r `write.compression`, `write.statistics` and row cap, and the install's `RAINCLOUD_PARQUET_*` settings in the table below (compression level, statistics and the page index for all or the first N columns, page size and rows, dictionaries, page -checksums). An unset setting leaves each library's own default, which differ (pyarrow -writes no page index; arrow-rs and parquet-java do). A set one reaches every writer and +checksums). An unset setting leaves each library's own default, which differ, except +that `parquet@py` writes a page index (when statistics are on) and page checksums, as +parquet-java does, where pyarrow's own default writes neither. A set one reaches every writer and becomes part of its toolchain, and a writer whose library cannot do what it asks fails that export as a measurement rather than writing something else; `sidecars/README.md` tabulates which writer honours what, and the `/raincloud-write-settings` skill is the @@ -278,13 +279,13 @@ its own `outputs/` rather than a machine's shared store. | `RAINCLOUD_ROW_GROUP_PROBE_ROWS` | rows the Python Parquet writer samples to size its groups (must be > 0) | 262,144 | | `RAINCLOUD_PARQUET_COMPRESSION_LEVEL` | the recipe's codec at this level in every Parquet writer (zstd 1-22, gzip 0-9, brotli 0-11) | unset (each writer's own) | | `RAINCLOUD_PARQUET_STATISTICS_COLUMNS` | statistics (chunk and page) only for the first N leaf columns (`0`: every column) | unset (each writer's own: every column) | -| `RAINCLOUD_PARQUET_PAGE_INDEX` | `1`: every Parquet writer writes a page index (ColumnIndex + OffsetIndex); `0`: none | unset (each writer's own: pyarrow and Hardwood none, arrow-rs and parquet-java one) | +| `RAINCLOUD_PARQUET_PAGE_INDEX` | `1`: every Parquet writer writes a page index (ColumnIndex + OffsetIndex); `0`: none | unset (each writer's own: pyarrow (raincloud's default for `parquet@py`), arrow-rs and parquet-java one, Hardwood none) | | `RAINCLOUD_PARQUET_PAGE_INDEX_COLUMNS` | page statistics only for the first N leaf columns, chunk statistics for every column (`0`: every column) | unset (each writer's own) | | `RAINCLOUD_PARQUET_PAGE_BYTES` | data page size target in every Parquet writer, each measuring a page its own way (`0`: no limit) | unset (each writer's own) | | `RAINCLOUD_PARQUET_PAGE_ROWS` | data page row limit in every Parquet writer (`0`: no limit) | unset (each writer's own) | | `RAINCLOUD_PARQUET_DICTIONARY` | `0`: no dictionary encoding (PLAIN); `1`: dictionaries | unset (each writer's own: on) | | `RAINCLOUD_PARQUET_DICTIONARY_PAGE_BYTES` | dictionary page size limit, past which a column falls back to PLAIN (`0`: no limit) | unset (each writer's own) | -| `RAINCLOUD_PARQUET_PAGE_CHECKSUMS` | `1`: a CRC in every page header; `0`: none | unset (each writer's own: parquet-java and Hardwood write them, pyarrow and arrow-rs do not) | +| `RAINCLOUD_PARQUET_PAGE_CHECKSUMS` | `1`: a CRC in every page header; `0`: none | unset (each writer's own: pyarrow (raincloud's default for `parquet@py`), parquet-java and Hardwood write them, arrow-rs does not) | | `RAINCLOUD_ORC_COMPRESSION` | ORC codec in every ORC writer: `zstd`, `snappy`, `zlib`, `lz4`, `none` | unset (zstd, as always) | | `RAINCLOUD_ORC_COMPRESSION_STRATEGY` | `speed` or `compression` (ORC C++ only; orc-rust refuses) | unset (each writer's own) | | `RAINCLOUD_ORC_STRIPE_BYTES` | ORC stripe size target, measured encoded and compressed | unset (each writer's own) | diff --git a/CHANGELOG.md b/CHANGELOG.md index a52447a..aba1879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- **`parquet@py` writes a page index and page checksums by default.** With + `RAINCLOUD_PARQUET_PAGE_INDEX` and `RAINCLOUD_PARQUET_PAGE_CHECKSUMS` unset, pyarrow + now writes a ColumnIndex and OffsetIndex for every column chunk (when the recipe's + `write.statistics` is on) and a CRC in every page header, as parquet-java does, where + pyarrow's own default writes neither. Readers can prune pages by their statistics + without a sidecar writer. Set either to `0` for the old files. Every Parquet artifact + `parquet@py` writes changes bytes and checksums. + ## [0.3.1] - 2026-10-08 The compliance ledger, `docs/v2/compliance.json`, is still 0.3.0's measurement: the ORC, diff --git a/README.md b/README.md index 53163d7..5a26f28 100644 --- a/README.md +++ b/README.md @@ -281,17 +281,18 @@ is measured in days. Build what you need. for Parquet a page index (ColumnIndex and OffsetIndex) on every column or only the first N, statistics for only the first N columns, the compression level, page size and rows, dictionaries and page checksums; for ORC and Avro the codec, level and block sizes; for -Vortex compact encodings. Unset, each writer library's default applies, which is how the -catalog's files are written: pyarrow, the default Parquet writer, writes no page index. To -build a dataset's Parquet with one: +Vortex compact encodings. Unset, each writer library's default applies, with one +exception: pyarrow, the default Parquet writer, writes a page index and page checksums, +as parquet-java does, though pyarrow's own default writes neither. To build a dataset's +Parquet without them: ```bash -RAINCLOUD_PARQUET_PAGE_INDEX=1 raincloud build uci-iris --format parquet +RAINCLOUD_PARQUET_PAGE_INDEX=0 RAINCLOUD_PARQUET_PAGE_CHECKSUMS=0 raincloud build uci-iris --format parquet ``` ```python import os, raincloud -os.environ["RAINCLOUD_PARQUET_PAGE_INDEX"] = "1" # before the build runs +os.environ["RAINCLOUD_PARQUET_PAGE_INDEX"] = "0" # before the build runs ds = raincloud.load("uci-iris", format="parquet", build=True) ``` diff --git a/raincloud/pipeline/export/exporters.py b/raincloud/pipeline/export/exporters.py index 91e2e22..b1d0865 100644 --- a/raincloud/pipeline/export/exporters.py +++ b/raincloud/pipeline/export/exporters.py @@ -138,7 +138,9 @@ def _leaf_paths(schema: pa.Schema) -> list[str]: def _writer_options(options: ParquetOptions, schema: pa.Schema) -> dict: """pyarrow's `ParquetWriter` arguments for `options`. An unset setting is - left out, so pyarrow's own default applies. + left out, so pyarrow's own default applies, except two: unset, a page index + (when statistics are on) and page checksums are written, as parquet-java + writes them, where pyarrow writes neither. pyarrow writes a page index for every column with statistics or for none, so `page_index_columns` (page statistics for some columns, chunk statistics @@ -151,12 +153,14 @@ def _writer_options(options: ParquetOptions, schema: pa.Schema) -> dict: statistics: bool | list[str] = options.statistics if options.statistics and options.statistics_columns is not None: statistics = _leaf_paths(schema)[:options.statistics_columns] - kwargs = {"compression": options.compression, "write_statistics": statistics} + page_index = options.statistics if options.page_index is None else options.page_index + page_checksums = True if options.page_checksums is None else options.page_checksums + kwargs = {"compression": options.compression, "write_statistics": statistics, + "write_page_index": page_index, "write_page_checksum": page_checksums} for name, value in (("compression_level", options.compression_level), - ("write_page_index", options.page_index), ("data_page_size", options.page_bytes), + ("data_page_size", options.page_bytes), ("max_rows_per_page", options.page_rows), ("use_dictionary", options.dictionary), - ("dictionary_pagesize_limit", options.dictionary_page_bytes), - ("write_page_checksum", options.page_checksums)): + ("dictionary_pagesize_limit", options.dictionary_page_bytes)): if value is not None: kwargs[name] = value return kwargs diff --git a/raincloud/pipeline/spec.py b/raincloud/pipeline/spec.py index 08254e7..8aabe98 100644 --- a/raincloud/pipeline/spec.py +++ b/raincloud/pipeline/spec.py @@ -236,8 +236,11 @@ def row_group_probe_rows() -> int: # Parquet write options every Parquet writer receives the same way. The install # settings differ from the row-group knobs in one respect: UNSET means each # writer's own default, not a raincloud figure, because the four libraries' -# defaults differ (arrow-rs and parquet-java write a page index, pyarrow and -# Hardwood do not) and no one figure leaves every lane's files as they are. +# defaults differ (arrow-rs and parquet-java write a page index, Hardwood does +# not) and no one figure leaves every lane's files as they are. The exception is +# parquet@py, which writes a page index and page checksums when unset, as +# parquet-java does (`exporters._writer_options`); pyarrow's own default writes +# neither. # Set, a setting reaches every lane, and a writer whose library cannot do what # it asks fails that export rather than writing something else. PARQUET_COMPRESSION = "RAINCLOUD_PARQUET_COMPRESSION" diff --git a/sidecars/README.md b/sidecars/README.md index 3d7acb7..92f9ed4 100644 --- a/sidecars/README.md +++ b/sidecars/README.md @@ -171,7 +171,8 @@ passes to a sidecar in this form: | `RAINCLOUD_PARQUET_PAGE_CHECKSUMS` | `1` or `0`: a CRC in every page header, or none | the install, only when set | An unset variable is the lane's library default (zstd and statistics on, for the first -two). Counts use the count grammar, `0` meaning no limit (every column, for the two +two), except that `parquet@py` writes a page index and page checksums when the two are +unset, though pyarrow's own default writes neither. Counts use the count grammar, `0` meaning no limit (every column, for the two column counts). Switches read `1/true/yes/on` and `0/false/no/off` in any case, empty as unset, in every lane. Settings that contradict each other or the recipe (a page index with statistics off; `PAGE_INDEX=0` with `PAGE_INDEX_COLUMNS`; a level for snappy, lz4 @@ -185,15 +186,15 @@ cannot honour: | compression level | yes | yes | yes | no | | statistics off | yes | yes | yes | no | | statistics, first N columns | yes | yes | yes | no | -| page index on | yes | yes | yes | no | +| page index on | yes (raincloud's default) | yes | yes | no | | page index off | yes | yes | no, with statistics on | yes | | page index, first N columns | no: all or none | yes | no | no | | page bytes | yes | yes | yes | yes | | page rows | yes | yes, checked every 1,024 values | yes | no | | dictionary on / off | yes | yes | yes | yes (off is PLAIN) | | dictionary page bytes | yes | yes | yes | no | -| page checksums on | yes | no | yes (its default) | yes (always) | -| page checksums off | yes (its default) | yes (always) | yes | no | +| page checksums on | yes (raincloud's default) | no | yes (its default) | yes (always) | +| page checksums off | yes | yes (always) | yes | no | Each library measures a page its own way, so the same `RAINCLOUD_PARQUET_PAGE_BYTES` does not give identical pages in every lane. The java column's page index gaps are diff --git a/tests/test_parquet_options.py b/tests/test_parquet_options.py index ae5f921..3cae9fb 100644 --- a/tests/test_parquet_options.py +++ b/tests/test_parquet_options.py @@ -227,6 +227,19 @@ def test_no_page_index_is_written_on_request_or_refused(tmp_path, monkeypatch, c assert _layout(dest)[0] is False +def test_parquet_py_writes_a_page_index_and_page_checksums_when_unset(tmp_path): + roundtrip, note, dest = _write(tmp_path, "parquet@py") + assert roundtrip is True, note + assert _layout(dest)[0] is True + assert all(4 in page for page in _data_pages(dest)) # crc + # A page index is page statistics: with the recipe's statistics off there is none. + (tmp_path / "bare").mkdir() + bare = {**SPEC, "write": {**SPEC["write"], "statistics": False}} + roundtrip, note, dest = _write(tmp_path / "bare", "parquet@py", bare) + assert roundtrip is True, note + assert _layout(dest)[0] is False + + @pytest.mark.parametrize("cell", CELLS) def test_the_page_size_knob_reaches_every_writer(tmp_path, monkeypatch, cell): unset = _write(tmp_path, cell)