diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d381b4..bea73a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,6 +94,25 @@ jobs: - name: Compare starfix.legacy.v0_4 with starfix==0.4.0 from PyPI run: uv run --python 3.12 python scripts/legacy_differential.py + release-differential: + # Every hash (one-shot and through a prepared schema) must stay byte-identical to the + # released starfix-python 0.5.0 (PyPI), under every option combination. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Install uv + uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + + - name: Set up Python 3.12 + run: uv python install 3.12 + + - name: Install dependencies + run: uv sync --dev --python 3.12 + + - name: Compare this checkout with starfix==0.5.0 from PyPI + run: uv run --python 3.12 python scripts/release_differential.py + license-check: runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bf6237..9990966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## 0.6.0 (unreleased) + +No hash changes: every hash is byte-identical to 0.5.0 (hash format 2), for every input and +option combination. + +### Added — prepared schemas (ITL-1177) + +- `ArrowDigester.prepare(schema, *, include_metadata=False, ordered_columns=False, + ordered_struct_fields=False)` analyses a schema once and returns a `PreparedSchema` + (exported from `starfix`), with `schema_digest` (equal to `hash_schema`), + `hash_record_batch(batch)`, `hash_table(table)` and `digester()` (a fresh streaming + `ArrowDigester`). Its hashes equal the one-shot calls; hashing a one-row table of a + six-column schema takes about 35 µs instead of about 150 µs (0.5.0, without + `include_metadata`; about 530 µs with it). See the README section "Hashing many inputs of + one schema" and `scripts/bench_prepared.py`. +- A `PreparedSchema` is immutable and owned by the caller; starfix keeps no cache. A table or + batch is accepted exactly when its schema hashes like the prepared one under the prepared + options (the rule streamed batches already follow); otherwise `ValueError`. Identical + schemas are recognized by comparing their Arrow IPC serialization (about 1.5 µs). +- `scripts/release_differential.py` (and a CI job): this checkout, one-shot and prepared, + against the released `starfix==0.5.0`, under all option combinations. + +### Changed — performance + +- One-shot calls (`hash_table`, `hash_record_batch`, `ArrowDigester(...)`) analyse the + schema once instead of three times (plus once per streamed batch): a one-row `hash_table` + takes about 85 µs instead of about 150 µs (about 175 µs instead of about 530 µs with + `include_metadata`). +- `ArrowDigester.update` checks a batch whose schema is identical to the digester's in about + 1.5 µs instead of re-analysing it. + ## 0.5.0 (2026-11-11) ### Changed — hash format 2 (every hash changes) diff --git a/README.md b/README.md index 0b0e45c..cf0ddd2 100644 --- a/README.md +++ b/README.md @@ -55,11 +55,48 @@ ArrowDigester.hash_record_batch(table.to_batches()[0]) ArrowDigester.hash_array(pa.array([1, 2, 3], pa.int32())) ``` +### Hashing many inputs of one schema + +Every call above first analyses the schema (walks its nodes, builds its canonical text and +digest). To hash many small tables or batches of one schema, analyse it once: + +```python +prepared = ArrowDigester.prepare(schema, include_metadata=False) # options as below + +for row_table in tables_of_this_schema: + key = prepared.hash_table(row_table) # == ArrowDigester.hash_table(row_table) +prepared.hash_record_batch(batch) # == ArrowDigester.hash_record_batch(batch) +prepared.schema_digest # == ArrowDigester.hash_schema(schema) +digester = prepared.digester() # a fresh streaming ArrowDigester +``` + +- Every hash equals the one-shot call with the same options, byte for byte. +- A `PreparedSchema` is immutable and belongs to the caller: starfix keeps no cache or other + state between calls. It can be shared between threads and pickled (it is prepared again when + loaded). +- **Same-schema rule.** A table or batch is accepted exactly when its schema hashes like the + prepared one under the prepared options (`hash_schema(its_schema, **options) == + prepared.schema_digest`): the same column names, logical types and nullability at every + depth, the same column order with `ordered_columns`, the same struct field order with + `ordered_struct_fields`, and with `include_metadata` the same field metadata at every node, + schema metadata and extension types. It is the rule `update` applies to streamed batches. + Anything else raises `ValueError` ("Table schema does not match the prepared schema" / + "Record batch schema does not match the prepared schema"); nothing is ever hashed with + another schema's analysis. +- The check costs about 1.5 µs when the input's schema is identical to the prepared one (equal + Arrow IPC serialization, compared as bytes). Any other accepted schema (for example `string` + data for a prepared `large_string` column, or metadata that differs while `include_metadata` + is off) is checked by computing its canonical schema text, as expensive as a one-shot call's + analysis. +- `prepare` raises what `hash_schema` raises. A schema whose data cannot be hashed (see + [Rejected inputs](#rejected-inputs)) still prepares, for its `schema_digest`; hashing data + with it raises `ValueError`. + ## Options Keyword-only, all `False` by default, the same names as the Rust `HasherConfig` fields, accepted by -`ArrowDigester(...)`, `hash_schema`, `hash_record_batch`, `hash_table` and `hash_array` -(`hash_array` has no `include_metadata`). +`ArrowDigester(...)`, `ArrowDigester.prepare`, `hash_schema`, `hash_record_batch`, `hash_table` and +`hash_array` (`hash_array` has no `include_metadata`). | Option | Effect | |---|---| @@ -132,7 +169,8 @@ rebuilds the type from them, so starfix-python hashes what pyarrow can observe: - a non-nullable field holding a null that no enclosing struct null explains; - with `include_metadata`: a duplicate metadata key on a field or the schema, or explicit `ARROW:extension:*` keys that conflict with the field's registered extension type; -- streaming: a batch whose schema differs from the digester's. Physically different but +- streaming, and hashing through a prepared schema: a batch or table whose schema differs from + the digester's or the prepared one (see the same-schema rule above). Physically different but equivalent types (`Utf8`/`LargeUtf8`, `Binary`/`LargeBinary`, `List`/`LargeList`, dictionary and plain values) are accepted; reordered columns or struct fields are accepted unless the matching order option is on. @@ -158,8 +196,10 @@ are materialized (decoded with `take`) before they are traversed. A fixed-size l without null slots is hashed from one zero-copy slice of its child; with null slots, its items are selected in chunks of about one million items. Fixed-width, Boolean and list columns hash at tens of millions of rows per second. String and binary columns hash at about 3 to 5 million rows per second, because each value is -visited in Python. Each `update` call also costs a few microseconds per column, to check the -batch's schema. starfix-python is a pure-Python implementation; the Rust crate is faster on +visited in Python. Each call also analyses the schema, which dominates for small inputs (tens to +hundreds of microseconds, more with `include_metadata`); `ArrowDigester.prepare` does it once +for many inputs. A streamed batch whose schema is identical to the digester's is checked in +about 1.5 µs; `scripts/bench_prepared.py` measures both paths. starfix-python is a pure-Python implementation; the Rust crate is faster on string-heavy data. ## Hash format diff --git a/docs/metamorphic/plans/2026-11-16-prepared-schema.md b/docs/metamorphic/plans/2026-11-16-prepared-schema.md new file mode 100644 index 0000000..ca09fca --- /dev/null +++ b/docs/metamorphic/plans/2026-11-16-prepared-schema.md @@ -0,0 +1,107 @@ +# Prepared Schema Implementation Plan (ITL-1177) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let a caller analyse an Arrow schema once (`ArrowDigester.prepare`) and hash many tables or record +batches of that schema without redoing the analysis, with every digest byte-identical to the one-shot calls. + +**Architecture:** `_format2.analyse_schema` runs the schema analysis once and returns an immutable +`SchemaAnalysis` (node list, canonical schema key, schema digest, entry specs, the data-support verdict, and +the schema's Arrow IPC bytes for the same-schema check). `Digester` is built from an analysis; the one-shot +entry points build one analysis per call and never re-check their own input. The public +`starfix.PreparedSchema` (in `arrow_digester.py`) wraps one analysis and its schema, owned by the caller. +No module-level, global or cached state is added. + +**Tech Stack:** Python >= 3.10, pyarrow >= 19, pytest, uv. + +--- + +## Interface + +```python +ArrowDigester.prepare(schema, *, include_metadata=False, ordered_columns=False, + ordered_struct_fields=False) -> PreparedSchema + +class PreparedSchema: # built only by prepare; immutable; pickles by re-preparing + schema: pa.Schema # read-only properties + include_metadata: bool + ordered_columns: bool + ordered_struct_fields: bool + schema_digest: bytes # == ArrowDigester.hash_schema(schema, same options) + def hash_record_batch(self, record_batch) -> bytes + def hash_table(self, table) -> bytes + def digester(self) -> ArrowDigester +``` + +* `prepare` raises exactly what `hash_schema` raises. Types whose schema hashes but whose data does not + (Null, Map, Interval, view types) do not make `prepare` fail; `hash_record_batch`, `hash_table` and + `digester()` raise the existing `ValueError` for them. +* Same-schema rule: accepted iff the input schema's canonical schema key under the prepared options equals + the prepared one (Phase 1 text; plus Phase 2 text with `include_metadata`), i.e. iff + `hash_schema(input.schema, options) == prepared.schema_digest`. This is today's rule of + `ArrowDigester.update`. +* Check: (1) `input.schema.serialize()` bytes equal the prepared schema's IPC bytes -> identical schema -> + accept; (2) otherwise compute the input's key (today's analysis) and compare; mismatch -> `ValueError` + ("Record batch schema does not match the prepared schema" / "Table schema does not match the prepared + schema"; `digester().update` keeps "Record batch schema does not match ArrowDigester schema"). + `pa.Schema.equals(check_metadata=True)` is not used: it ignores nested metadata of dictionary value types + and the parameters of Python-defined extension types. + +## Files + +* Modify `src/starfix/_format2.py`: `SchemaAnalysis`, `analyse_schema`, `phase2_json(schema, nodes)`, + `_Entry(kind, nullable)`, `Digester.from_analysis`, unchecked ingestion for one-shot calls. +* Modify `src/starfix/arrow_digester.py`: `ArrowDigester.prepare`, `PreparedSchema`. +* Modify `src/starfix/__init__.py`: export `PreparedSchema`. +* Create `tests/random_arrow.py`: seeded random schemas and data (nested, dictionary, fixed-size list, + extension types, metadata, empty and sliced inputs). +* Create `tests/test_prepared_schema.py`: API, immutability, no module state, golden vectors x all option + combinations, the type matrix, accept/refuse rule, seeded differential, analysis-count tests. +* Create `scripts/release_differential.py`: this checkout (one-shot and prepared) against the released + `starfix==0.5.0` from PyPI, byte-identical. Add a CI job for it beside `legacy-differential`. +* Create `scripts/bench_prepared.py`: time per one-row hash, one-shot and prepared. +* Modify `README.md` (usage + same-schema rule), `CHANGELOG.md` (Unreleased, proposed 0.6.0). + +## Tasks + +### Task 1: failing tests for the public interface +- [ ] Write `tests/test_prepared_schema.py` tests: `prepare` returns a `PreparedSchema` whose properties + echo the options; `schema_digest == hash_schema` for all 8 option combinations; `PreparedSchema()` raises + `TypeError`; setting/deleting attributes raises `AttributeError`; pickle round trip hashes the same; + `hash_record_batch` / `hash_table` / `digester()` equal the one-shot results. +- [ ] Run `uv run pytest tests/test_prepared_schema.py -q`; expect failures (no `prepare`). + +### Task 2: analysis once (refactor, no behaviour change) +- [ ] `analyse_schema` computes nodes once, Phase 1 once, Phase 2 once (from the same nodes), the digest + from that key, sorted entry specs, the data-support verdict and the IPC bytes. +- [ ] `Digester.__init__` = `analyse_schema` + `from_analysis`; `update` checks via the analysis; + `hash_record_batch` / `hash_table` analyse once and ingest without re-checking (a table's batches carry + the table's schema). +- [ ] Test: counting wrappers on `schema_nodes`, `phase1_json`, `phase2_json` show one call each per + one-shot `hash_table` / `hash_record_batch`, and none for a prepared hash of an identical schema. +- [ ] Run the full suite: `uv run pytest tests/ -q` (golden vectors pin the bytes). + +### Task 3: `PreparedSchema` and `ArrowDigester.prepare` +- [ ] Implement in `arrow_digester.py`; export from `starfix`. Run Task 1 tests: pass. + +### Task 4: same-schema rule +- [ ] Refusal cases (each with every relevant option combination): column name, type, nullability (top + and nested), column order with `ordered_columns`, struct field order with `ordered_struct_fields`, field + metadata at top level, in a struct child, a list / fixed-size-list element, the struct inside a dictionary + value, schema metadata, a Python extension type's parameter, extension vs storage type, all with + `include_metadata`. Acceptance cases: string vs large_string, dictionary vs plain, reordered columns or + struct fields when unordered, metadata differences without `include_metadata`; accepted results equal the + one-shot hash of the input. +- [ ] Seeded pairs: accepted iff `hash_schema` digests are equal. + +### Task 5: differential proofs +- [ ] Golden vectors (`tests/golden/format2_v0.5.json`) x 8 option combinations: prepared == one-shot; + for the vector's own options == expected. +- [ ] Type matrix (`tests/test_format2_type_matrix.py::_hashes`) also through a prepared schema. +- [ ] Seeded random inputs (`tests/random_arrow.py`), all option combinations, batch / table / stream. +- [ ] `scripts/release_differential.py` against `starfix==0.5.0`; CI job. + +### Task 6: benchmark, docs, changelog, PR +- [ ] `scripts/bench_prepared.py`; numbers before (origin/main) and after. +- [ ] README section "Hashing many inputs of one schema"; CHANGELOG Unreleased. +- [ ] `uv run pytest tests/ -q`; commit (conventional commits); PR against `main`. diff --git a/scripts/bench_prepared.py b/scripts/bench_prepared.py new file mode 100644 index 0000000..989de2c --- /dev/null +++ b/scripts/bench_prepared.py @@ -0,0 +1,94 @@ +"""Time per hash of small tables: one-shot calls against a prepared schema (ITL-1177). + +Usage (from the repository root): + + uv run python scripts/bench_prepared.py [--rows 1] [--repeat 7] + +Hashes tables of a few-column schema (ints, strings, a list, a struct, field and schema +metadata), each built anew as a caller hashing row by row would (so every table carries a new +``pa.Schema`` object). Prints the best-of-``--repeat`` mean time per call in microseconds. +Against a checkout without ``ArrowDigester.prepare`` (0.5.0 and earlier) only the one-shot rows +are printed, so the same script measures before and after: for example + + git worktree add /tmp/starfix-main origin/main + PYTHONPATH=/tmp/starfix-main/src uv run python scripts/bench_prepared.py +""" + +from __future__ import annotations + +import argparse +import platform +import timeit + +import pyarrow as pa + +import starfix +from starfix import ArrowDigester + +SCHEMA = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("name", pa.large_string()), + pa.field("score", pa.float64(), metadata={"unit": "points"}), + pa.field("tags", pa.list_(pa.large_string())), + pa.field("pos", pa.struct([pa.field("x", pa.float32()), pa.field("y", pa.float32())])), + pa.field("path", pa.large_string(), metadata={"semantic": "path"}), + ], + metadata={"producer": "bench"}, +) + + +def _rows(n: int) -> list[dict]: + return [ + {"id": i, "name": f"n{i}", "score": i / 3, "tags": ["a", "b"][: i % 3], "pos": {"x": 1.0, "y": 2.0}, + "path": f"/data/{i}.bin"} + for i in range(n) + ] + + +def _bench_options(bench, tables, batches, options: dict[str, bool], has_prepare: bool) -> None: + bench("one-shot ArrowDigester.hash_table", lambda i: ArrowDigester.hash_table(tables[i], **options)) + bench("one-shot ArrowDigester.hash_record_batch", lambda i: ArrowDigester.hash_record_batch(batches[i], **options)) + if not has_prepare: + return + prepared = ArrowDigester.prepare(SCHEMA, **options) + bench("prepared.hash_table", lambda i: prepared.hash_table(tables[i])) + bench("prepared.hash_record_batch", lambda i: prepared.hash_record_batch(batches[i])) + bench(" of which: ArrowDigester.prepare (once per schema)", lambda i: ArrowDigester.prepare(SCHEMA, **options)) + analysis = prepared._analysis + bench(" of which: same-schema check (identical schema)", lambda i: analysis.matches(tables[i].schema)) + string_schema = pa.schema([f.with_type(pa.string()) if f.type == pa.large_string() else f for f in SCHEMA], + metadata=SCHEMA.metadata) + bench(" same-schema check, hash-equivalent (string) schema", lambda i: analysis.matches(string_schema)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--rows", type=int, default=1) + parser.add_argument("--repeat", type=int, default=7) + parser.add_argument("--number", type=int, default=2000) + args = parser.parse_args() + rows = _rows(args.rows) + tables = [pa.Table.from_pylist(rows, schema=SCHEMA) for _ in range(64)] # distinct Schema objects + batches = [t.to_batches()[0] for t in tables] + has_prepare = hasattr(ArrowDigester, "prepare") + print(f"starfix from {starfix.__file__}; pyarrow {pa.__version__}; Python {platform.python_version()}; " + f"{platform.machine()}; {args.rows} row(s), {len(SCHEMA)} columns") + + def bench(label: str, call) -> None: + it = iter(range(1 << 62)) + + def step() -> None: + call(next(it) % len(tables)) + + best = min(timeit.repeat(step, number=args.number, repeat=args.repeat)) / args.number + print(f" {label:<52} {best * 1e6:8.1f} us") + + for include_metadata in (False, True): + print(f"include_metadata={include_metadata}") + _bench_options(bench, tables, batches, {"include_metadata": include_metadata}, has_prepare) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_differential.py b/scripts/release_differential.py new file mode 100644 index 0000000..ecd89c8 --- /dev/null +++ b/scripts/release_differential.py @@ -0,0 +1,162 @@ +"""Differential test: this checkout (one-shot and prepared) against the released starfix 0.5.0. + +Usage (from the repository root): + + uv run python scripts/release_differential.py [--seeds N] + +The driver hashes seeded random tables (``tests/random_arrow.py``: nested types, dictionaries, +fixed-size lists, extension types, metadata at every depth, empty and multi-batch tables) under +all 8 option combinations, through the one-shot calls and through ``ArrowDigester.prepare`` +(``hash_table``, ``hash_record_batch``, ``digester()`` and ``schema_digest``), and requires the +prepared results to equal the one-shot ones. It then runs this same script with +``--impl release`` in an isolated environment containing the released ``starfix==0.5.0`` from +PyPI and the same pyarrow version, and requires every one-shot result to be byte-identical +(exceptions must match by type). Both processes build the inputs themselves from the seeds. +Exit status 0 means identical. +""" + +from __future__ import annotations + +import argparse +import itertools +import json +import random +import subprocess +import sys +import tempfile +from importlib import metadata +from pathlib import Path + +import pyarrow as pa + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +import random_arrow + +RELEASE = "0.5.0" +OPTION_SETS = [ + {"include_metadata": m, "ordered_columns": c, "ordered_struct_fields": s} + for m, c, s in itertools.product([False, True], repeat=3) +] + + +def _outcome(call, *args, **kwargs) -> str: + try: + return call(*args, **kwargs).hex() + except Exception as e: # noqa: BLE001 - exceptions must match by type + return f"raises {type(e).__name__}" + + +def _stream(digester, batches) -> bytes: + for batch in batches: + digester.update(batch) + return digester.finalize() + + +def _one_shot_stream(D, schema, batches, **options) -> bytes: + return _stream(D(schema, **options), batches) + + +def _prepared_schema(D, schema, **options) -> bytes: + return D.prepare(schema, **options).schema_digest + + +def _prepared_table(D, table, **options) -> bytes: + return D.prepare(table.schema, **options).hash_table(table) + + +def _prepared_batch(D, schema, batch, **options) -> bytes: + return D.prepare(schema, **options).hash_record_batch(batch) + + +def _prepared_stream(D, schema, batches, **options) -> bytes: + return _stream(D.prepare(schema, **options).digester(), batches) + + +def _special_tables(): + """Inputs that raise: schemas whose data does not hash, rejected schemas, nulls in + non-nullable fields.""" + yield "map", pa.table({"m": pa.array([[("k", 1)]], pa.map_(pa.string(), pa.int8()))}) + yield "null", pa.table({"n": pa.nulls(2)}) + yield "union", pa.table({"u": pa.UnionArray.from_sparse(pa.array([0], pa.int8()), [pa.array([1])])}) + yield "duplicate", pa.Table.from_arrays([pa.array([1]), pa.array([2])], names=["a", "a"]) + nulls = pa.StructArray.from_arrays([pa.array([1, None])], fields=[pa.field("x", pa.int64(), nullable=False)]) + yield "non_nullable_null", pa.Table.from_arrays([nulls], names=["s"]) + meta = pa.schema([pa.field("a", pa.int8(), metadata={"ARROW:extension:name": "x"})]) + yield "extension_name_without_type", pa.table({"a": pa.array([1], pa.int8())}, schema=meta) + + +def _tables(seeds: int): + yield from _special_tables() + for seed in range(seeds): + yield seed, random_arrow.random_table(random.Random(seed)) + + +def _run(impl: str, seeds: int) -> tuple[dict[str, str], list[str]]: + from starfix import ArrowDigester as D + + if impl == "release" and metadata.version("starfix") != RELEASE: + raise SystemExit(f"not the starfix {RELEASE} release: {metadata.version('starfix')}") + results: dict[str, str] = {} + prepared_differ: list[str] = [] + for seed, table in _tables(seeds): + schema, batches = table.schema, table.to_batches() + for options in OPTION_SETS: + tag = f"{seed}:m{int(options['include_metadata'])}c{int(options['ordered_columns'])}" \ + f"s{int(options['ordered_struct_fields'])}" + one_shot = { + "schema": _outcome(D.hash_schema, schema, **options), + "table": _outcome(D.hash_table, table, **options), + "stream": _outcome(_one_shot_stream, D, schema, batches, **options), + } + if batches: + one_shot["batch"] = _outcome(D.hash_record_batch, batches[0], **options) + for name, value in one_shot.items(): + results[f"{tag}:{name}"] = value + if impl != "current": + continue + prepared = { + "schema": _outcome(_prepared_schema, D, schema, **options), + "table": _outcome(_prepared_table, D, table, **options), + "stream": _outcome(_prepared_stream, D, schema, batches, **options), + } + if batches: + prepared["batch"] = _outcome(_prepared_batch, D, schema, batches[0], **options) + prepared_differ += [f"{tag}:{k}" for k in one_shot if prepared[k] != one_shot[k]] + return results, prepared_differ + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--impl", choices=["driver", "current", "release"], default="driver") + parser.add_argument("--seeds", type=int, default=1000) + parser.add_argument("--out") + args = parser.parse_args() + if args.impl != "driver": + results, _ = _run(args.impl, args.seeds) + Path(args.out).write_text(json.dumps(results)) + return 0 + + current, prepared_differ = _run("current", args.seeds) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "release.json" + python = f"{sys.version_info.major}.{sys.version_info.minor}" + subprocess.run( + ["uv", "run", "--no-project", "--python", python, "--with", f"starfix=={RELEASE}", + "--with", f"pyarrow=={pa.__version__}", "python", str(Path(__file__).resolve()), + "--impl", "release", "--seeds", str(args.seeds), "--out", str(out)], + check=True, cwd=tmp, + ) + release = json.loads(out.read_text()) + differ = sorted(k for k in current if current[k] != release.get(k)) + sorted(set(release) - set(current)) + raised = sum(1 for v in current.values() if v.startswith("raises")) + print(f"{len(current)} one-shot results compared with starfix {RELEASE} ({raised} raise the same exception " + f"in both); {len(differ)} differ; {len(prepared_differ)} prepared results differ from one-shot") + for key in differ[:20]: + print(f" DIFFER {key}: current={current.get(key)} release={release.get(key)}") + for key in prepared_differ[:20]: + print(f" PREPARED DIFFERS {key}") + return 1 if differ or prepared_differ else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/starfix/__init__.py b/src/starfix/__init__.py index ef830ad..4e53c74 100644 --- a/src/starfix/__init__.py +++ b/src/starfix/__init__.py @@ -1,3 +1,3 @@ -from starfix.arrow_digester import ArrowDigester +from starfix.arrow_digester import ArrowDigester, PreparedSchema -__all__ = ["ArrowDigester"] +__all__ = ["ArrowDigester", "PreparedSchema"] diff --git a/src/starfix/_format2.py b/src/starfix/_format2.py index cc5d82d..1701a11 100644 --- a/src/starfix/_format2.py +++ b/src/starfix/_format2.py @@ -334,10 +334,12 @@ def _field_metadata(field: pa.Field) -> dict[str, str]: return meta -def phase2_json(schema: pa.Schema) -> str | None: - """Section 2.3: metadata keyed by node path; None when there is none.""" +def phase2_json(schema: pa.Schema, nodes: list[tuple[str, pa.Field]] | None = None) -> str | None: + """Section 2.3: metadata keyed by node path; None when there is none. + + `nodes` are the schema's nodes when already walked (``schema_nodes(schema)``).""" fields = {} - for path, field in schema_nodes(schema): + for path, field in (schema_nodes(schema) if nodes is None else nodes): try: meta = _field_metadata(field) except _AmbiguousMetadata as exc: @@ -353,14 +355,19 @@ def phase2_json(schema: pa.Schema) -> str | None: return _json(doc) if doc else None -def schema_key(schema: pa.Schema, include_metadata: bool, flags: int) -> tuple[str, str | None]: - return phase1_json(schema, flags), (phase2_json(schema) if include_metadata else None) +def schema_key( + schema: pa.Schema, include_metadata: bool, flags: int, nodes: list[tuple[str, pa.Field]] | None = None +) -> tuple[str, str | None]: + """The canonical schema text (Phase 1, and Phase 2 with include_metadata): everything of the + schema that enters a hash. Walks the nodes (Section 7 rejections) unless `nodes` is given.""" + if nodes is None: + nodes = schema_nodes(schema) + return phase1_json(schema, flags), (phase2_json(schema, nodes) if include_metadata else None) -def schema_digest(schema: pa.Schema, include_metadata: bool, flags: int) -> bytes: +def _digest_of_key(key: tuple[str, str | None], flags: int) -> bytes: """Section 2.5.""" - schema_nodes(schema) # Section 7 rejections - phase1, phase2 = schema_key(schema, include_metadata, flags) + phase1, phase2 = key h = hashlib.sha256(header(KIND_SCHEMA, flags)) h.update(phase1.encode()) if phase2 is not None: @@ -368,6 +375,11 @@ def schema_digest(schema: pa.Schema, include_metadata: bool, flags: int) -> byte return h.digest() +def schema_digest(schema: pa.Schema, include_metadata: bool, flags: int) -> bytes: + """Section 2.5.""" + return _digest_of_key(schema_key(schema, include_metadata, flags), flags) + + # --------------------------------------------------------------------------- # Entries (Section 3.2) # --------------------------------------------------------------------------- @@ -418,22 +430,25 @@ def restore(self, state: tuple[int, int, int | None]) -> None: self._len = bit_len +def _entry_kind(field_type: pa.DataType) -> str: + t = _logical(field_type) + if pa.types.is_struct(t): + return "struct" + if _is_list(t): + return "list" + if pa.types.is_fixed_size_list(t): + return "fixed_size_list" # Section 3.2: validity only (n is in the schema) + return "leaf" + + class _Entry: __slots__ = ("kind", "lengths", "validity", "values") - def __init__(self, field: pa.Field) -> None: - t = _logical(field.type) - if pa.types.is_struct(t): - self.kind = "struct" - elif _is_list(t): - self.kind = "list" - elif pa.types.is_fixed_size_list(t): - self.kind = "fixed_size_list" # Section 3.2: validity only (n is in the schema) - else: - self.kind = "leaf" - self.validity = _Bits() if field.nullable else None - self.lengths = hashlib.sha256() if self.kind == "list" else None - self.values = hashlib.sha256() if self.kind == "leaf" else None + def __init__(self, kind: str, nullable: bool) -> None: + self.kind = kind + self.validity = _Bits() if nullable else None + self.lengths = hashlib.sha256() if kind == "list" else None + self.values = hashlib.sha256() if kind == "leaf" else None def snapshot(self) -> tuple: return ( @@ -784,16 +799,24 @@ def _is_unsupported_data(t: pa.DataType) -> bool: ) -def _check_data_supported(nodes: list[tuple[str, pa.Field]]) -> None: - """Section 7 item 3: runs before any data is hashed, whatever the number of rows.""" +def _data_support_error(nodes: list[tuple[str, pa.Field]]) -> str | None: + """Section 7 item 3: the error for a schema whose data cannot be hashed (None: it can).""" for path, field in nodes: t = _logical(field.type) if _is_unsupported_data(t): - raise ValueError(f"Hashing data of type {t} (node {path!r}) is not supported") + return f"Hashing data of type {t} (node {path!r}) is not supported" + return None + + +def _check_data_supported(nodes: list[tuple[str, pa.Field]]) -> None: + """Section 7 item 3: runs before any data is hashed, whatever the number of rows.""" + error = _data_support_error(nodes) + if error is not None: + raise ValueError(error) def _new_entries(nodes: list[tuple[str, pa.Field]]) -> dict[str, _Entry]: - return {path: _Entry(field) for path, field in nodes} + return {path: _Entry(_entry_kind(field.type), field.nullable) for path, field in nodes} def _feed_entries(entries: dict[str, _Entry], digest) -> None: @@ -806,6 +829,72 @@ def _feed_entries(entries: dict[str, _Entry], digest) -> None: # Public entry points (Sections 2.5, 3.3, 4, 5) # --------------------------------------------------------------------------- +def _ipc_bytes(schema: pa.Schema) -> bytes | None: + """The schema as an Arrow IPC schema message (None if pyarrow cannot serialize it).""" + try: + return schema.serialize().to_pybytes() + except Exception: # noqa: BLE001 - only disables the fast path of SchemaAnalysis.matches + return None + + +class SchemaAnalysis: + """Everything the schema analysis produces, computed once (ITL-1177); immutable. + + Holds the hashing options, the canonical schema key and digest (Section 2.5), the + entries to create for the schema's nodes in node-path order (Section 3.2), the + data-support verdict (Section 7 item 3) and the schema's Arrow IPC bytes, which make + `matches` cheap for an identical schema. + """ + + __slots__ = ("_data_error", "_ipc", "_specs", "digest", "flags", "include_metadata", "key") + + def __init__(self, schema: pa.Schema, include_metadata: bool, flags: int) -> None: + nodes = schema_nodes(schema) # Section 7 rejections, once + self.include_metadata = include_metadata + self.flags = flags + self.key = schema_key(schema, include_metadata, flags, nodes) + self.digest = _digest_of_key(self.key, flags) + # Entries in byte-wise node-path order (Section 3.3), so finalize needs no sort. + self._specs = tuple( + (path, _entry_kind(field.type), field.nullable) for path, field in sorted(nodes, key=lambda n: n[0]) + ) + self._data_error = _data_support_error(nodes) + self._ipc = _ipc_bytes(schema) + + def __setattr__(self, name: str, value: Any) -> None: + if hasattr(self, "_ipc"): # the last slot set by __init__ + raise AttributeError("SchemaAnalysis is immutable") + object.__setattr__(self, name, value) + + def require_data_supported(self) -> None: + if self._data_error is not None: + raise ValueError(self._data_error) + + def new_entries(self) -> dict[str, _Entry]: + return {path: _Entry(kind, nullable) for path, kind, nullable in self._specs} + + def matches(self, schema: pa.Schema) -> bool: + """Whether `schema` has this schema key: everything that enters a hash under these + options is the same (Section 5's rule for streamed batches). + + Fast path: identical Arrow IPC bytes mean an identical schema (names, types, + nullability, order, all field and schema metadata at every depth, extension names, + serializations and storage types). Otherwise the key is computed and compared, + which accepts hash-equivalent schemas (e.g. string vs large_string) and raises + Section 7 errors exactly as the full analysis does. + """ + if self._ipc is not None and _ipc_bytes(schema) == self._ipc: + return True + return schema_key(schema, self.include_metadata, self.flags) == self.key + + +def analyse(schema: pa.Schema, entry_point: str, *, include_metadata: bool = False, ordered_columns: bool = False, + ordered_struct_fields: bool = False) -> SchemaAnalysis: + _require_little_endian() + _require(schema, pa.Schema, entry_point, "a pyarrow.Schema") + return SchemaAnalysis(schema, include_metadata, flags_of(ordered_columns, ordered_struct_fields)) + + class Digester: """Streaming record-batch hasher (Section 3.3).""" @@ -817,41 +906,51 @@ def __init__( ordered_columns: bool = False, ordered_struct_fields: bool = False, ) -> None: - _require_little_endian() - _require(schema, pa.Schema, "ArrowDigester", "a pyarrow.Schema") - self._flags = flags_of(ordered_columns, ordered_struct_fields) - self._include_metadata = include_metadata - self._schema_digest = schema_digest(schema, include_metadata, self._flags) - self._key = schema_key(schema, include_metadata, self._flags) - nodes = schema_nodes(schema) - _check_data_supported(nodes) - self._entries = _new_entries(nodes) + self._start(analyse(schema, "ArrowDigester", include_metadata=include_metadata, + ordered_columns=ordered_columns, ordered_struct_fields=ordered_struct_fields)) + + @classmethod + def from_analysis(cls, analysis: SchemaAnalysis) -> Digester: + """A digester for a schema analysed earlier (no analysis is repeated).""" + digester = cls.__new__(cls) + digester._start(analysis) + return digester + + def _start(self, analysis: SchemaAnalysis) -> None: + analysis.require_data_supported() + self._analysis = analysis + self._entries = analysis.new_entries() self._rows = 0 def update(self, batch: pa.RecordBatch) -> None: _require(batch, pa.RecordBatch, "update", "a pyarrow.RecordBatch (use hash_table for a Table)") schema = batch.schema - schema_nodes(schema) # Section 7 - if schema_key(schema, self._include_metadata, self._flags) != self._key: + if not self._analysis.matches(schema): raise ValueError(_SCHEMA_MISMATCH) # A batch rejected while it is traversed (Section 7 item 4) leaves the digester # exactly as it was. saved = {path: entry.snapshot() for path, entry in self._entries.items()} try: - for i in range(batch.num_columns): - field = schema.field(i) - _visit(field, batch.column(i), field.name, None, self._entries) + self.ingest(batch, schema) except BaseException: for path, state in saved.items(): self._entries[path].restore(state) raise + + def ingest(self, batch: pa.RecordBatch, schema: pa.Schema) -> None: + """Add a batch whose schema `schema` is known to match (unchecked; on error the + digester is left partly updated and must be discarded).""" + for i in range(batch.num_columns): + field = schema.field(i) + _visit(field, batch.column(i), field.name, None, self._entries) self._rows += batch.num_rows def finalize(self) -> bytes: - final = hashlib.sha256(header(KIND_RECORDS, self._flags)) - final.update(self._schema_digest) + final = hashlib.sha256(header(KIND_RECORDS, self._analysis.flags)) + final.update(self._analysis.digest) final.update(_U64.pack(self._rows)) - _feed_entries(self._entries, final) + for entry in self._entries.values(): # created in node-path order + entry.feed(final) return VERSION_BYTES + final.digest() @@ -867,21 +966,50 @@ def hash_schema(schema: pa.Schema, *, include_metadata: bool = False, ordered_co return VERSION_BYTES + schema_digest(schema, include_metadata, flags_of(ordered_columns, ordered_struct_fields)) +_BATCH_TYPE = "a pyarrow.RecordBatch (use hash_table for a Table)" +_TABLE_TYPE = "a pyarrow.Table (use hash_record_batch for a RecordBatch)" + + def hash_record_batch(batch: pa.RecordBatch, **options: bool) -> bytes: - _require(batch, pa.RecordBatch, "hash_record_batch", "a pyarrow.RecordBatch (use hash_table for a Table)") - d = Digester(batch.schema, **options) - d.update(batch) + _require(batch, pa.RecordBatch, "hash_record_batch", _BATCH_TYPE) + schema = batch.schema + d = Digester.from_analysis(analyse(schema, "hash_record_batch", **options)) + d.ingest(batch, schema) # the batch's own schema: nothing to check return d.finalize() def hash_table(table: pa.Table, **options: bool) -> bytes: - _require(table, pa.Table, "hash_table", "a pyarrow.Table (use hash_record_batch for a RecordBatch)") - d = Digester(table.schema, **options) - for batch in table.to_batches(): - d.update(batch) + _require(table, pa.Table, "hash_table", _TABLE_TYPE) + schema = table.schema + return _hash_table_batches(Digester.from_analysis(analyse(schema, "hash_table", **options)), table, schema) + + +def _hash_table_batches(d: Digester, table: pa.Table, schema: pa.Schema) -> bytes: + for batch in table.to_batches(): # a table's batches carry the table's schema + d.ingest(batch, schema) return d.finalize() +def hash_prepared_record_batch(analysis: SchemaAnalysis, batch: pa.RecordBatch) -> bytes: + _require(batch, pa.RecordBatch, "hash_record_batch", _BATCH_TYPE) + analysis.require_data_supported() + schema = batch.schema + if not analysis.matches(schema): + raise ValueError("Record batch schema does not match the prepared schema") + d = Digester.from_analysis(analysis) + d.ingest(batch, schema) + return d.finalize() + + +def hash_prepared_table(analysis: SchemaAnalysis, table: pa.Table) -> bytes: + _require(table, pa.Table, "hash_table", _TABLE_TYPE) + analysis.require_data_supported() + schema = table.schema + if not analysis.matches(schema): + raise ValueError("Table schema does not match the prepared schema") + return _hash_table_batches(Digester.from_analysis(analysis), table, schema) + + def hash_array(array: pa.Array, *, ordered_columns: bool = False, ordered_struct_fields: bool = False) -> bytes: """Section 4: the array is the node at path "", always nullable. diff --git a/src/starfix/arrow_digester.py b/src/starfix/arrow_digester.py index cdb641c..c5a4cd3 100644 --- a/src/starfix/arrow_digester.py +++ b/src/starfix/arrow_digester.py @@ -37,6 +37,9 @@ class ArrowDigester: A digester is not thread-safe: feed one digester from one thread at a time. A batch rejected by ``update`` leaves the digester unchanged. + + To hash many tables or batches of one schema, analyse the schema once with + ``ArrowDigester.prepare`` and hash through the returned ``PreparedSchema``. """ def __init__( @@ -65,6 +68,43 @@ def __init__( ordered_struct_fields=ordered_struct_fields, ) + @classmethod + def _from_analysis(cls, analysis: _format2.SchemaAnalysis) -> ArrowDigester: + digester = cls.__new__(cls) + digester._digester = _format2.Digester.from_analysis(analysis) + return digester + + @staticmethod + def prepare( + schema: pa.Schema, + *, + include_metadata: bool = False, + ordered_columns: bool = False, + ordered_struct_fields: bool = False, + ) -> PreparedSchema: + """Analyse a schema once, for hashing many tables or record batches of it. + + Returns an immutable ``PreparedSchema`` owned by the caller; its hashes equal the + one-shot calls with the same options. Raises what ``hash_schema`` raises. A schema + whose data cannot be hashed (Null, Map, Interval, view types) still prepares, for its + ``schema_digest``; hashing data with it raises ``ValueError``. + + Args: + schema: The Arrow schema of the data to hash. + include_metadata: Hash schema- and field-level metadata. + ordered_columns: Make the order of the top-level columns part of the hash. + ordered_struct_fields: Make the order of struct fields, at every depth, + part of the hash. + """ + analysis = _format2.analyse( + schema, + "prepare", + include_metadata=include_metadata, + ordered_columns=ordered_columns, + ordered_struct_fields=ordered_struct_fields, + ) + return PreparedSchema._create(schema, analysis) + def update(self, record_batch: pa.RecordBatch) -> None: """Add a record batch. Splitting rows into batches does not change the hash.""" self._digester.update(record_batch) @@ -136,3 +176,102 @@ def hash_array( return _format2.hash_array( array, ordered_columns=ordered_columns, ordered_struct_fields=ordered_struct_fields ) + + +class PreparedSchema: + """A schema analysed once, with its hashing options (create with ``ArrowDigester.prepare``). + + Hashes many tables or record batches of the schema without analysing it again; every + hash is byte-identical to the one-shot ``ArrowDigester`` call with the same options. + Immutable and owned by the caller: starfix keeps no cache or other state of its own. Safe + to share between threads (each hash or ``digester()`` has its own state). + + **Same-schema rule.** A table or record batch is accepted exactly when its schema has the + prepared schema's canonical schema key under the prepared options, i.e. when + ``ArrowDigester.hash_schema(its_schema, **options) == prepared.schema_digest``: the same + column names, logical types and nullability at every depth (column order when + ``ordered_columns``, struct field order when ``ordered_struct_fields``) and, with + ``include_metadata``, the same field metadata at every node, schema metadata and extension + type names and serializations. This is the rule ``ArrowDigester.update`` applies to + streamed batches. Anything else raises ``ValueError``; nothing is hashed with another + schema's analysis. An input whose schema is identical to the prepared one (equal Arrow + IPC serialization) is accepted by a byte comparison; other inputs, such as a ``string`` + column for a prepared ``large_string`` one, are checked by computing their schema key. + """ + + __slots__ = ("_analysis", "_schema") + + def __init__(self, *args: object, **kwargs: object) -> None: + raise TypeError("PreparedSchema is created by ArrowDigester.prepare(schema, ...)") + + @classmethod + def _create(cls, schema: pa.Schema, analysis: _format2.SchemaAnalysis) -> PreparedSchema: + prepared = object.__new__(cls) + object.__setattr__(prepared, "_schema", schema) + object.__setattr__(prepared, "_analysis", analysis) + return prepared + + def __setattr__(self, name: str, value: object) -> None: + raise AttributeError(f"PreparedSchema is immutable (cannot set {name!r})") + + def __delattr__(self, name: str) -> None: + raise AttributeError(f"PreparedSchema is immutable (cannot delete {name!r})") + + def __copy__(self) -> PreparedSchema: + return self + + def __deepcopy__(self, memo: dict) -> PreparedSchema: + return self + + def __reduce__(self) -> tuple: + # Pickled as (schema, options) and prepared again when loaded. + return (_prepare, (self._schema, self.include_metadata, self.ordered_columns, self.ordered_struct_fields)) + + def __repr__(self) -> str: + return ( + f"PreparedSchema(include_metadata={self.include_metadata}, ordered_columns={self.ordered_columns}, " + f"ordered_struct_fields={self.ordered_struct_fields}, schema_digest={self.schema_digest.hex()})" + ) + + @property + def schema(self) -> pa.Schema: + """The prepared Arrow schema.""" + return self._schema + + @property + def include_metadata(self) -> bool: + return self._analysis.include_metadata + + @property + def ordered_columns(self) -> bool: + return bool(self._analysis.flags & _format2.ORDERED_COLUMNS) + + @property + def ordered_struct_fields(self) -> bool: + return bool(self._analysis.flags & _format2.ORDERED_STRUCT_FIELDS) + + @property + def schema_digest(self) -> bytes: + """The 35-byte schema hash, equal to ``ArrowDigester.hash_schema`` with the same options.""" + return VERSION_BYTES + self._analysis.digest + + def hash_record_batch(self, record_batch: pa.RecordBatch) -> bytes: + """Hash one record batch of this schema (see the same-schema rule).""" + return _format2.hash_prepared_record_batch(self._analysis, record_batch) + + def hash_table(self, table: pa.Table) -> bytes: + """Hash a table of this schema (see the same-schema rule); checked once per table.""" + return _format2.hash_prepared_table(self._analysis, table) + + def digester(self) -> ArrowDigester: + """A fresh streaming ``ArrowDigester`` for this schema and options, without analysis.""" + return ArrowDigester._from_analysis(self._analysis) + + +def _prepare(schema: pa.Schema, include_metadata: bool, ordered_columns: bool, ordered_struct_fields: bool) -> PreparedSchema: + return ArrowDigester.prepare( + schema, + include_metadata=include_metadata, + ordered_columns=ordered_columns, + ordered_struct_fields=ordered_struct_fields, + ) diff --git a/tests/random_arrow.py b/tests/random_arrow.py new file mode 100644 index 0000000..3eef5aa --- /dev/null +++ b/tests/random_arrow.py @@ -0,0 +1,168 @@ +"""Seeded random Arrow schemas and data for differential tests (ITL-1177). + +Used by ``tests/test_prepared_schema.py`` and ``scripts/release_differential.py`` (which runs it +against the released starfix in an isolated environment), so it depends on pyarrow only. +Everything is built from ``random.Random(seed)``: the same seed gives the same inputs in any +process. Covers every type family whose data hashes: fixed-width and variable-length leaves, +booleans, dictionaries (of leaves and of structs), list / large_list, fixed-size lists, structs, +extension types (canonical and Python-defined), field metadata at every depth, schema metadata, +nulls, sliced arrays, empty batches and multi-batch tables. +""" + +from __future__ import annotations + +import random +from decimal import Decimal + +import pyarrow as pa +import pyarrow.compute as pc + + +class UnitType(pa.ExtensionType): + """A Python-defined extension type with a parameter carried only in its serialization.""" + + def __init__(self, unit: str = "m") -> None: + self.unit = unit + super().__init__(pa.int64(), "starfix.test.unit") + + def __arrow_ext_serialize__(self) -> bytes: + return self.unit.encode() + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type: pa.DataType, serialized: bytes) -> UnitType: + return cls(serialized.decode()) + + +LEAVES: list[pa.DataType] = [ + pa.int8(), pa.int16(), pa.int32(), pa.int64(), pa.uint8(), pa.uint32(), pa.uint64(), + pa.float16(), pa.float32(), pa.float64(), pa.decimal128(10, 2), pa.decimal256(40, 5), + pa.date32(), pa.date64(), pa.time32("ms"), pa.time64("ns"), + pa.timestamp("us"), pa.timestamp("ms", tz="UTC"), pa.duration("s"), + pa.binary(4), pa.binary(0), pa.string(), pa.large_string(), pa.binary(), pa.large_binary(), pa.bool_(), +] + + +def _extension_types() -> list[pa.DataType]: + types: list[pa.DataType] = [UnitType("m"), UnitType("s"), pa.fixed_shape_tensor(pa.float32(), [2, 2])] + for name in ("uuid", "json_", "bool8"): + factory = getattr(pa, name, None) + if factory is not None: + types.append(factory()) + return types + + +EXTENSIONS = _extension_types() +_NAMES = ["a", "b", "c", "d", "é"] + + +def random_type(rng: random.Random, depth: int = 0) -> pa.DataType: + kind = rng.random() + if depth >= 3 or kind < 0.35: + return rng.choice(LEAVES) + if kind < 0.45: + value = rng.choice([pa.string(), pa.int64(), pa.float32()]) + return pa.dictionary(rng.choice([pa.int8(), pa.int32()]), value) + if kind < 0.5: + return pa.dictionary(pa.int32(), pa.struct([random_field(rng, n, depth + 1) for n in ("x", "y")])) + if kind < 0.6: + return rng.choice(EXTENSIONS) + if kind < 0.75: + element = random_field(rng, "item", depth + 1) + return pa.large_list(element) if rng.random() < 0.3 else pa.list_(element) + if kind < 0.82: + return pa.list_(random_field(rng, "item", depth + 1), rng.choice([0, 1, 2, 3])) + names = rng.sample(_NAMES, rng.randint(0, 3)) + return pa.struct([random_field(rng, n, depth + 1) for n in names]) + + +def random_metadata(rng: random.Random) -> dict[str, str] | None: + if rng.random() < 0.6: + return None + return {rng.choice(["unit", "z", "é", "a"]): rng.choice(["kg", "", "v1", "ü"]) for _ in range(rng.randint(1, 2))} + + +def random_field(rng: random.Random, name: str, depth: int = 0) -> pa.Field: + return pa.field(name, random_type(rng, depth), nullable=rng.random() < 0.6, metadata=random_metadata(rng)) + + +def random_schema(rng: random.Random) -> pa.Schema: + names = rng.sample(_NAMES, rng.randint(0, 4)) + return pa.schema([random_field(rng, n) for n in names], metadata=random_metadata(rng)) + + +def _leaf_values(t: pa.DataType, n: int, nullable: bool, rng: random.Random) -> pa.Array: + def value(): + if pa.types.is_boolean(t): + return rng.random() < 0.5 + if pa.types.is_floating(t): + return rng.uniform(-1e3, 1e3) + if pa.types.is_decimal(t): + return Decimal(rng.randint(-99999, 99999)) / 100 + if pa.types.is_fixed_size_binary(t): + return bytes(rng.randrange(256) for _ in range(t.byte_width)) + if pa.types.is_binary(t) or pa.types.is_large_binary(t): + return bytes(rng.randrange(256) for _ in range(rng.randint(0, 5))) + if pa.types.is_string(t) or pa.types.is_large_string(t): + return "".join(rng.choice("xyzé") for _ in range(rng.randint(0, 5))) + if pa.types.is_temporal(t): + return rng.randint(0, 20000) + if pa.types.is_unsigned_integer(t): + return rng.randint(0, 2 ** min(t.bit_width, 16) - 1) + return rng.randint(-(2 ** (t.bit_width - 1)), 2 ** (t.bit_width - 1) - 1) + + values = [None if nullable and rng.random() < 0.25 else value() for _ in range(n)] + if pa.types.is_float16(t): # half-float bit patterns (no numpy needed) + return pa.array([None if v is None else rng.randrange(1 << 16) for v in values], pa.uint16()).view(t) + if pa.types.is_temporal(t): + storage = pa.int64() if t.bit_width == 64 else pa.int32() + return pa.array([None if v is None else v * (86_400_000 if pa.types.is_date64(t) else 1) for v in values], + storage).cast(t) + return pa.array(values, t) + + +def random_array(t: pa.DataType, n: int, nullable: bool, rng: random.Random) -> pa.Array: + """An array of `n` elements of type `t`, sliced out of a longer one (non-zero offsets).""" + pad = rng.randint(0, 3) + total = n + pad + mask = pa.array([rng.random() < 0.25 for _ in range(total)], pa.bool_()) if nullable else None + if isinstance(t, pa.BaseExtensionType): + arr = pa.ExtensionArray.from_storage(t, random_array(t.storage_type, total, nullable, rng)) + elif pa.types.is_dictionary(t): + values = random_array(t.value_type, rng.randint(1, 4), nullable and rng.random() < 0.3, rng) + index_values = [None if nullable and rng.random() < 0.2 else rng.randrange(len(values)) for _ in range(total)] + arr = pa.DictionaryArray.from_arrays(pa.array(index_values, t.index_type), values) + elif pa.types.is_list(t) or pa.types.is_large_list(t): + lengths = [rng.choice([0, 0, 1, 2, 3]) for _ in range(total)] + offsets = [0] + for length in lengths: + offsets.append(offsets[-1] + length) + child = random_array(t.value_type, offsets[-1], t.value_field.nullable, rng) + cls = pa.LargeListArray if pa.types.is_large_list(t) else pa.ListArray + index_type = pa.int64() if pa.types.is_large_list(t) else pa.int32() + arr = cls.from_arrays(pa.array(offsets, index_type), child, type=t, mask=mask) + elif pa.types.is_fixed_size_list(t): + child = random_array(t.value_type, total * t.list_size, t.value_field.nullable, rng) + validity = None if mask is None else pc.invert(mask).buffers()[1] + arr = pa.Array.from_buffers(t, total, [validity], null_count=-1, children=[child]) # also n = 0 + elif pa.types.is_struct(t) and t.num_fields == 0: + validity = None if mask is None else pc.invert(mask).buffers()[1] + arr = pa.Array.from_buffers(t, total, [validity], null_count=-1) + elif pa.types.is_struct(t): + children = [random_array(f.type, total, f.nullable, rng) for f in t] + arr = pa.StructArray.from_arrays(children, fields=list(t), mask=mask) + else: + arr = _leaf_values(t, total, nullable, rng) + return arr.slice(rng.randint(0, pad), n) + + +def random_table(rng: random.Random, schema: pa.Schema | None = None) -> pa.Table: + """A table of a random (or the given) schema, split into 1-3 batches (some may be empty).""" + schema = schema if schema is not None else random_schema(rng) + rows = rng.choice([0, 1, 1, 2, 5, 17]) + columns = [random_array(f.type, rows, f.nullable, rng) for f in schema] + batch = pa.RecordBatch.from_arrays(columns, schema=schema) if columns else pa.RecordBatch.from_pylist( + [{}] * rows, schema=schema + ) + cuts = sorted(rng.choices(range(rows + 1), k=rng.randint(0, 2))) + pieces = [batch.slice(a, b - a) for a, b in zip([0, *cuts], [*cuts, rows], strict=True)] + return pa.Table.from_batches(pieces, schema=schema) diff --git a/tests/test_format2_type_matrix.py b/tests/test_format2_type_matrix.py index e0dd080..7bf656f 100644 --- a/tests/test_format2_type_matrix.py +++ b/tests/test_format2_type_matrix.py @@ -291,23 +291,33 @@ def wrap(child: pa.Array) -> pa.Array: def _hashes(arr: pa.Array | pa.ChunkedArray) -> dict[str, str]: """hash_array, the record hash (one batch, or a table of the chunks) and the record hash - streamed through the digester in two batches per chunk.""" + streamed through the digester in two batches per chunk; the record hashes also through a + prepared schema (ITL-1177).""" schema = pa.schema([pa.field("c", arr.type, nullable=True)]) + prepared = ArrowDigester.prepare(schema) if isinstance(arr, pa.ChunkedArray): chunks = arr.chunks - records = ArrowDigester.hash_table(pa.Table.from_arrays([arr], schema=schema)) + table = pa.Table.from_arrays([arr], schema=schema) + records = ArrowDigester.hash_table(table) + prepared_records = prepared.hash_table(table) else: chunks = [arr] - records = ArrowDigester.hash_record_batch(pa.record_batch([arr], schema=schema)) + batch = pa.record_batch([arr], schema=schema) + records = ArrowDigester.hash_record_batch(batch) + prepared_records = prepared.hash_record_batch(batch) digester = ArrowDigester(schema) + prepared_digester = prepared.digester() for chunk in chunks: k = len(chunk) // 2 # the second batch starts at a non-zero offset for part in (chunk.slice(0, k), chunk.slice(k)): digester.update(pa.record_batch([part], schema=schema)) + prepared_digester.update(pa.record_batch([part], schema=schema)) return { "hash_array": ArrowDigester.hash_array(arr).hex(), "records": records.hex(), "stream": digester.finalize().hex(), + "prepared_records": prepared_records.hex(), + "prepared_stream": prepared_digester.finalize().hex(), } @@ -333,6 +343,7 @@ def test_type_matrix(leaf_type: pa.DataType, base: pa.Array, golden: str | None, reference = reference_wrap(base) expected = _hashes(reference) assert expected["stream"] == expected["records"], "streamed batches differ from one batch" + assert expected["prepared_records"] == expected["prepared_stream"] == expected["records"], "prepared differs" if golden is not None: assert expected["hash_array"] == golden, "plain array differs from the Rust golden vector" diff --git a/tests/test_prepared_schema.py b/tests/test_prepared_schema.py new file mode 100644 index 0000000..31fb1dd --- /dev/null +++ b/tests/test_prepared_schema.py @@ -0,0 +1,547 @@ +"""A prepared schema: analyse once, hash many inputs of that schema (ITL-1177). + +Every digest of the prepared path must be byte-identical to the one-shot path for the same input +and options; a batch or table whose schema differs from the prepared one under the prepared +options is refused; starfix holds no state of its own between calls. +""" + +from __future__ import annotations + +import base64 +import copy +import io +import itertools +import json +import pickle +import random +from functools import cache +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest +import random_arrow as ra # tests/ is on sys.path (pytest rootdir-relative import) + +import starfix +from starfix import ArrowDigester, PreparedSchema, _format2, arrow_digester + +OPTION_SETS = [ + {"include_metadata": m, "ordered_columns": c, "ordered_struct_fields": s} + for m, c, s in itertools.product([False, True], repeat=3) +] +_OPTION_IDS = [f"m{int(o['include_metadata'])}c{int(o['ordered_columns'])}s{int(o['ordered_struct_fields'])}" + for o in OPTION_SETS] +_BATCH_MISMATCH = "^Record batch schema does not match the prepared schema$" +_TABLE_MISMATCH = "^Table schema does not match the prepared schema$" +_STREAM_MISMATCH = "^Record batch schema does not match ArrowDigester schema$" + + +def _one_shot_stream(schema: pa.Schema, batches: list[pa.RecordBatch], **options: bool) -> bytes: + d = ArrowDigester(schema, **options) + for batch in batches: + d.update(batch) + return d.finalize() + + +def _prepared_stream(prepared: PreparedSchema, batches: list[pa.RecordBatch]) -> bytes: + d = prepared.digester() + for batch in batches: + d.update(batch) + return d.finalize() + + +def _assert_same_as_one_shot(table: pa.Table, options: dict[str, bool], prepared: PreparedSchema | None = None) -> None: + """Every prepared entry point equals its one-shot counterpart on `table` (and its batches).""" + prepared = prepared if prepared is not None else ArrowDigester.prepare(table.schema, **options) + batches = table.to_batches() + expected = ArrowDigester.hash_table(table, **options) + assert prepared.hash_table(table) == expected + assert _prepared_stream(prepared, batches) == expected + assert _one_shot_stream(table.schema, batches, **options) == expected + for batch in batches: + assert prepared.hash_record_batch(batch) == ArrowDigester.hash_record_batch(batch, **options) + + +def _simple_table() -> pa.Table: + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False, metadata={"role": "key"}), + pa.field("name", pa.string()), + pa.field("tags", pa.list_(pa.field("item", pa.string(), metadata={"k": "v"}))), + pa.field("s", pa.struct([pa.field("b", pa.float64()), pa.field("a", pa.int8())])), + ], + metadata={"origin": "test"}, + ) + return pa.table( + {"id": [1, 2, 3], "name": ["x", None, "z"], "tags": [["p"], [], None], "s": [{"b": 1.0, "a": 1}, None, {"b": None, "a": 3}]}, + schema=schema, + ) + + +# --------------------------------------------------------------------------- +# The interface +# --------------------------------------------------------------------------- + +def test_prepared_schema_is_exported() -> None: + assert starfix.PreparedSchema is PreparedSchema + assert "PreparedSchema" in starfix.__all__ + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +def test_prepare_echoes_schema_and_options(options: dict[str, bool]) -> None: + table = _simple_table() + prepared = ArrowDigester.prepare(table.schema, **options) + assert isinstance(prepared, PreparedSchema) + assert prepared.schema.equals(table.schema, check_metadata=True) + for name, value in options.items(): + assert getattr(prepared, name) is value + assert prepared.schema_digest == ArrowDigester.hash_schema(table.schema, **options) + assert len(prepared.schema_digest) == 35 + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +def test_prepared_hashes_equal_one_shot(options: dict[str, bool]) -> None: + table = _simple_table() + _assert_same_as_one_shot(table, options) + _assert_same_as_one_shot(table.slice(1, 1), options) + _assert_same_as_one_shot(table.slice(0, 0), options) + two_batches = pa.Table.from_batches([table.slice(0, 1).to_batches()[0], table.slice(1).to_batches()[0]]) + _assert_same_as_one_shot(two_batches, options) + + +def test_prepare_options_are_keyword_only() -> None: + with pytest.raises(TypeError): + ArrowDigester.prepare(_simple_table().schema, True) # type: ignore[misc] + + +def test_prepared_schema_cannot_be_constructed_directly() -> None: + with pytest.raises(TypeError, match=r"ArrowDigester\.prepare"): + PreparedSchema() # type: ignore[call-arg] + + +def test_prepared_schema_is_immutable() -> None: + prepared = ArrowDigester.prepare(_simple_table().schema) + for name in ("schema", "schema_digest", "include_metadata", "_analysis", "anything"): + with pytest.raises(AttributeError): + setattr(prepared, name, None) + with pytest.raises(AttributeError): + del prepared.schema + assert not hasattr(prepared, "__dict__") + + +def test_prepared_schema_copies_and_pickles() -> None: + table = _simple_table() + options = {"include_metadata": True, "ordered_columns": True, "ordered_struct_fields": False} + prepared = ArrowDigester.prepare(table.schema, **options) + assert copy.copy(prepared) is prepared + assert copy.deepcopy(prepared) is prepared + restored = pickle.loads(pickle.dumps(prepared)) # re-prepared from (schema, options) + assert restored is not prepared + assert restored.schema_digest == prepared.schema_digest + assert restored.hash_table(table) == prepared.hash_table(table) + for name, value in options.items(): + assert getattr(restored, name) is value + + +def test_digester_returns_a_fresh_arrow_digester() -> None: + table = _simple_table() + prepared = ArrowDigester.prepare(table.schema) + first, second = prepared.digester(), prepared.digester() + assert isinstance(first, ArrowDigester) and first is not second + first.update(table.to_batches()[0]) + assert second.finalize() == ArrowDigester.hash_table(table.slice(0, 0)) + assert first.finalize() == ArrowDigester.hash_table(table) + + +def test_wrong_container_types() -> None: + table = _simple_table() + prepared = ArrowDigester.prepare(table.schema) + with pytest.raises(TypeError, match="use hash_table for a Table"): + prepared.hash_record_batch(table) # type: ignore[arg-type] + with pytest.raises(TypeError, match="use hash_record_batch for a RecordBatch"): + prepared.hash_table(table.to_batches()[0]) # type: ignore[arg-type] + with pytest.raises(TypeError, match="a pyarrow.Schema"): + ArrowDigester.prepare(table) # type: ignore[arg-type] + + +def test_prepare_raises_what_hash_schema_raises() -> None: + duplicate = pa.schema([pa.field("a", pa.int8()), pa.field("a", pa.int8())]) + with pytest.raises(ValueError, match="Ambiguous node path"): + ArrowDigester.hash_schema(duplicate) + with pytest.raises(ValueError, match="Ambiguous node path"): + ArrowDigester.prepare(duplicate) + rejected = pa.schema([pa.field("u", pa.dense_union([pa.field("x", pa.int8())]))]) + with pytest.raises(ValueError, match="Unsupported Arrow type"): + ArrowDigester.prepare(rejected) + + +@pytest.mark.parametrize( + "data_type", + [pa.null(), pa.map_(pa.string(), pa.int8()), pa.month_day_nano_interval(), pa.string_view(), pa.list_(pa.null())], + ids=str, +) +def test_schema_only_types_prepare_but_do_not_hash_data(data_type: pa.DataType) -> None: + schema = pa.schema([pa.field("c", data_type)]) + prepared = ArrowDigester.prepare(schema) + assert prepared.schema_digest == ArrowDigester.hash_schema(schema) + table = pa.table({"c": pa.array([], data_type)}) + match = r"^Hashing data of type .* \(node 'c/?'\) is not supported$" + for call in (lambda: prepared.digester(), lambda: prepared.hash_table(table), + lambda: prepared.hash_record_batch(pa.RecordBatch.from_pylist([], schema=table.schema))): + with pytest.raises(ValueError, match=match): + call() + with pytest.raises(ValueError, match=match): + ArrowDigester(schema) # unchanged: the streaming constructor still raises + + +# --------------------------------------------------------------------------- +# No state outside the prepared object +# --------------------------------------------------------------------------- + +def _module_state() -> dict[str, dict[str, int]]: + return {m.__name__: {k: id(v) for k, v in vars(m).items()} for m in (_format2, arrow_digester, starfix)} + + +def test_no_module_level_state_is_created_or_changed() -> None: + before = _module_state() + table = _simple_table() + for options in OPTION_SETS: + prepared = ArrowDigester.prepare(table.schema, **options) + prepared.hash_table(table) + ArrowDigester.hash_table(table, **options) + prepared.digester().update(table.to_batches()[0]) + assert _module_state() == before + + +def test_preparing_a_second_schema_does_not_affect_the_first() -> None: + table = _simple_table() + first = ArrowDigester.prepare(table.schema) + expected = first.hash_table(table) + other = pa.table({"q": [1.5]}) + ArrowDigester.prepare(other.schema).hash_table(other) + assert first.hash_table(table) == expected + + +# --------------------------------------------------------------------------- +# The analysis runs once +# --------------------------------------------------------------------------- + +@pytest.fixture +def analysis_calls(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: + counts = {"schema_nodes": 0, "phase1_json": 0, "phase2_json": 0} + for name in counts: + original = getattr(_format2, name) + + def counting(*args: Any, _name: str = name, _original: Any = original, **kwargs: Any) -> Any: + counts[_name] += 1 + return _original(*args, **kwargs) + + monkeypatch.setattr(_format2, name, counting) + return counts + + +def test_one_shot_calls_analyse_once(analysis_calls: dict[str, int]) -> None: + table = _simple_table() + batch = table.to_batches()[0] + for call in (lambda: ArrowDigester.hash_table(table, include_metadata=True), + lambda: ArrowDigester.hash_record_batch(batch, include_metadata=True), + lambda: ArrowDigester.hash_schema(table.schema, include_metadata=True)): + for name in analysis_calls: + analysis_calls[name] = 0 + call() + assert analysis_calls == {"schema_nodes": 1, "phase1_json": 1, "phase2_json": 1} + + +_ANALYSED_ONCE = {"schema_nodes": 1, "phase1_json": 1, "phase2_json": 1} +_NOT_ANALYSED = {"schema_nodes": 0, "phase1_json": 0, "phase2_json": 0} + + +def test_prepared_hashes_of_an_identical_schema_do_not_analyse(analysis_calls: dict[str, int]) -> None: + table = _simple_table() + rebuilt = pa.table(table.to_pydict(), schema=_simple_table().schema) # equal schema, other objects + batch = rebuilt.to_batches()[0] + prepared = ArrowDigester.prepare(table.schema, include_metadata=True) + assert analysis_calls == _ANALYSED_ONCE # the counters see the analysis (else the zeros below are vacuous) + for name in analysis_calls: + analysis_calls[name] = 0 + # From here on the counts accumulate and are never reset: every prepared operation is checked. + digesters: list[ArrowDigester] = [] + for operation, call in ( + ("digester", lambda: digesters.append(prepared.digester())), + ("digester().update", lambda: digesters[0].update(batch)), + ("digester().finalize", lambda: digesters[0].finalize()), + ("hash_table", lambda: prepared.hash_table(table)), + ("hash_table of an equal schema", lambda: prepared.hash_table(rebuilt)), + ("hash_record_batch", lambda: prepared.hash_record_batch(batch)), + ("schema_digest", lambda: prepared.schema_digest), + ): + call() + assert analysis_calls == _NOT_ANALYSED, f"prepared {operation} analysed the schema" + + +def test_streaming_update_of_an_identical_schema_does_not_analyse(analysis_calls: dict[str, int]) -> None: + table = _simple_table() + batch = pa.table(table.to_pydict(), schema=_simple_table().schema).to_batches()[0] + streaming = ArrowDigester(table.schema, include_metadata=True) + assert analysis_calls == _ANALYSED_ONCE # the constructor analyses, once + for name in analysis_calls: + analysis_calls[name] = 0 + streaming.update(batch) + streaming.update(batch) + assert analysis_calls == _NOT_ANALYSED + + +# --------------------------------------------------------------------------- +# The same-schema rule +# --------------------------------------------------------------------------- + +def _field(name: str, t: pa.DataType, nullable: bool = True, metadata: dict[str, str] | None = None) -> pa.Field: + return pa.field(name, t, nullable=nullable, metadata=metadata) + + +def _tensor(dim_names: list[str] | None) -> pa.DataType: + return pa.fixed_shape_tensor(pa.int8(), [1, 2], dim_names=dim_names) + + +# (id, prepared schema, input schema, options under which the input differs in the hash) +# "all" = every option combination; "meta" = include_metadata=True only; "cols" = ordered_columns +# only; "structs" = ordered_struct_fields only. +_MD = {"k": "1"} +_MD2 = {"k": "2"} +_DIFFERENCES: list[tuple[str, pa.Schema, pa.Schema, str]] = [ + ("column-name", pa.schema([_field("a", pa.int8())]), pa.schema([_field("b", pa.int8())]), "all"), + ("column-type", pa.schema([_field("a", pa.int8())]), pa.schema([_field("a", pa.int16())]), "all"), + ("column-nullability", pa.schema([_field("a", pa.int8())]), pa.schema([_field("a", pa.int8(), False)]), "all"), + ("extra-column", pa.schema([_field("a", pa.int8())]), + pa.schema([_field("a", pa.int8()), _field("b", pa.int8())]), "all"), + ("struct-child-nullability", pa.schema([_field("s", pa.struct([_field("x", pa.int8())]))]), + pa.schema([_field("s", pa.struct([_field("x", pa.int8(), False)]))]), "all"), + ("list-element-nullability", pa.schema([_field("l", pa.list_(_field("item", pa.int8())))]), + pa.schema([_field("l", pa.list_(_field("item", pa.int8(), False)))]), "all"), + ("list-vs-fixed-size-list", pa.schema([_field("l", pa.list_(pa.int8()))]), + pa.schema([_field("l", pa.list_(pa.int8(), 2))]), "all"), + ("fixed-size-list-size", pa.schema([_field("l", pa.list_(pa.int8(), 2))]), + pa.schema([_field("l", pa.list_(pa.int8(), 3))]), "all"), + ("timestamp-zone", pa.schema([_field("t", pa.timestamp("us"))]), + pa.schema([_field("t", pa.timestamp("us", "UTC"))]), "all"), + ("decimal-scale", pa.schema([_field("d", pa.decimal128(10, 2))]), + pa.schema([_field("d", pa.decimal128(10, 3))]), "all"), + ("column-order", pa.schema([_field("a", pa.int8()), _field("b", pa.int8())]), + pa.schema([_field("b", pa.int8()), _field("a", pa.int8())]), "cols"), + ("struct-field-order", pa.schema([_field("s", pa.struct([_field("x", pa.int8()), _field("y", pa.int8())]))]), + pa.schema([_field("s", pa.struct([_field("y", pa.int8()), _field("x", pa.int8())]))]), "structs"), + ("column-metadata", pa.schema([_field("a", pa.int8(), metadata=_MD)]), + pa.schema([_field("a", pa.int8(), metadata=_MD2)]), "meta"), + ("column-metadata-removed", pa.schema([_field("a", pa.int8(), metadata=_MD)]), + pa.schema([_field("a", pa.int8())]), "meta"), + ("schema-metadata", pa.schema([_field("a", pa.int8())], metadata=_MD), + pa.schema([_field("a", pa.int8())], metadata=_MD2), "meta"), + ("struct-child-metadata", pa.schema([_field("s", pa.struct([_field("x", pa.int8(), metadata=_MD)]))]), + pa.schema([_field("s", pa.struct([_field("x", pa.int8(), metadata=_MD2)]))]), "meta"), + ("list-element-metadata", pa.schema([_field("l", pa.list_(_field("item", pa.int8(), metadata=_MD)))]), + pa.schema([_field("l", pa.list_(_field("item", pa.int8(), metadata=_MD2)))]), "meta"), + ("fixed-size-list-element-metadata", pa.schema([_field("l", pa.list_(_field("item", pa.int8(), metadata=_MD), 2))]), + pa.schema([_field("l", pa.list_(_field("item", pa.int8(), metadata=_MD2), 2))]), "meta"), + # pyarrow's Schema.equals(check_metadata=True) ignores this difference (pyarrow 26). + ("dictionary-value-struct-child-metadata", + pa.schema([_field("d", pa.dictionary(pa.int8(), pa.struct([_field("x", pa.int8(), metadata=_MD)])))]), + pa.schema([_field("d", pa.dictionary(pa.int8(), pa.struct([_field("x", pa.int8(), metadata=_MD2)])))]), "meta"), + # pyarrow's default ExtensionType equality ignores the serialized parameter. + ("python-extension-parameter", pa.schema([_field("u", ra.UnitType("m"))]), + pa.schema([_field("u", ra.UnitType("s"))]), "meta"), + ("canonical-extension-parameter", pa.schema([_field("t", _tensor(None))]), + pa.schema([_field("t", _tensor(["r", "c"]))]), "meta"), + ("extension-vs-storage", pa.schema([_field("u", ra.UnitType("m"))]), + pa.schema([_field("u", pa.int64())]), "meta"), + ("extension-storage-child-metadata", + pa.schema([_field("t", _tensor(None))]), + pa.schema([_field("t", pa.list_(_field("item", pa.int8(), metadata=_MD), 2))]), "meta"), +] + + +def _differs_under(kind: str, options: dict[str, bool]) -> bool: + return { + "all": True, + "meta": options["include_metadata"], + "cols": options["ordered_columns"], + "structs": options["ordered_struct_fields"], + }[kind] + + +def _empty(schema: pa.Schema) -> pa.Table: + rng = random.Random(0) + return pa.Table.from_arrays([ra.random_array(f.type, 0, f.nullable, rng) for f in schema], schema=schema) + + +def _one_row(schema: pa.Schema, seed: int = 0) -> pa.Table: + rng = random.Random(seed) + columns = [ra.random_array(f.type, 1, f.nullable, rng) for f in schema] + return pa.Table.from_arrays(columns, schema=schema) + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +@pytest.mark.parametrize(("case", "prepared_schema", "input_schema", "kind"), _DIFFERENCES, + ids=[d[0] for d in _DIFFERENCES]) +def test_same_schema_rule(case: str, prepared_schema: pa.Schema, input_schema: pa.Schema, kind: str, + options: dict[str, bool]) -> None: + """Refused exactly when the input's schema hashes differently under the prepared options; + otherwise accepted, hashing byte-identically to the one-shot path.""" + prepared = ArrowDigester.prepare(prepared_schema, **options) + differs = ArrowDigester.hash_schema(input_schema, **options) != prepared.schema_digest + assert differs == _differs_under(kind, options), case + for table in (_empty(input_schema), _one_row(input_schema)): + batch = pa.RecordBatch.from_arrays([c.combine_chunks() for c in table.columns], schema=input_schema) + if differs: + with pytest.raises(ValueError, match=_TABLE_MISMATCH): + prepared.hash_table(table) + with pytest.raises(ValueError, match=_BATCH_MISMATCH): + prepared.hash_record_batch(batch) + d = prepared.digester() + with pytest.raises(ValueError, match=_STREAM_MISMATCH): + d.update(batch) + assert d.finalize() == prepared.hash_table(_empty(prepared_schema)) # left unchanged + else: + _assert_same_as_one_shot(table, options, prepared) + + +# Hash-equivalent but not identical schemas: accepted (today's streaming rule) under every option set +# where they hash alike, through the analysis fallback. +_EQUIVALENT: list[tuple[str, pa.Schema, pa.Schema]] = [ + ("string-vs-large-string", pa.schema([_field("a", pa.string())]), pa.schema([_field("a", pa.large_string())])), + ("binary-vs-large-binary", pa.schema([_field("a", pa.binary())]), pa.schema([_field("a", pa.large_binary())])), + ("list-vs-large-list", pa.schema([_field("l", pa.list_(pa.int8()))]), pa.schema([_field("l", pa.large_list(pa.int8()))])), + ("dictionary-vs-plain", pa.schema([_field("a", pa.string())]), + pa.schema([_field("a", pa.dictionary(pa.int32(), pa.string()))])), + ("dictionary-index-type", pa.schema([_field("a", pa.dictionary(pa.int8(), pa.string()))]), + pa.schema([_field("a", pa.dictionary(pa.int32(), pa.string()))])), + ("list-element-name", pa.schema([_field("l", pa.list_(_field("item", pa.int8())))]), + pa.schema([_field("l", pa.list_(_field("element", pa.int8())))])), + ("metadata-key-order", pa.schema([_field("a", pa.int8(), metadata={"x": "1", "y": "2"})], metadata={"p": "1", "q": "2"}), + pa.schema([_field("a", pa.int8(), metadata={"y": "2", "x": "1"})], metadata={"q": "2", "p": "1"})), +] + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +@pytest.mark.parametrize(("case", "prepared_schema", "input_schema"), _EQUIVALENT, ids=[e[0] for e in _EQUIVALENT]) +def test_hash_equivalent_schemas_are_accepted(case: str, prepared_schema: pa.Schema, input_schema: pa.Schema, + options: dict[str, bool]) -> None: + assert ArrowDigester.hash_schema(input_schema, **options) == ArrowDigester.hash_schema(prepared_schema, **options) + prepared = ArrowDigester.prepare(prepared_schema, **options) + for table in (_empty(input_schema), _one_row(input_schema)): + _assert_same_as_one_shot(table, options, prepared) + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +def test_seeded_schema_pairs_accepted_iff_schema_digests_equal(options: dict[str, bool]) -> None: + """Random pairs, many of them hash-equivalent after dropping metadata or reordering.""" + rng = random.Random(1177) + accepted = refused = 0 + for _ in range(150): + a = ra.random_schema(rng) + b = rng.choice([ + ra.random_schema(rng), + a.remove_metadata(), + pa.schema(list(reversed(list(a))), metadata=a.metadata), + pa.schema([f.remove_metadata() if rng.random() < 0.5 else f for f in a], metadata=a.metadata), + a, + ]) + prepared = ArrowDigester.prepare(a, **options) + table = ra.random_table(rng, b) + if ArrowDigester.hash_schema(b, **options) == prepared.schema_digest: + accepted += 1 + assert prepared.hash_table(table) == ArrowDigester.hash_table(table, **options) + else: + refused += 1 + with pytest.raises(ValueError, match=_TABLE_MISMATCH): + prepared.hash_table(table) + assert accepted > 20 and refused > 20 + + +def test_ipc_fast_path_is_deterministic_for_equal_schemas() -> None: + """The fast path compares Arrow IPC schema bytes; equal schemas built in different ways + (pandas, a JSON-like round trip, IPC) serialize identically or fall back to the full check.""" + table = _simple_table() + schema = table.schema + assert schema.serialize().to_pybytes() == schema.serialize().to_pybytes() + via_ipc = pa.ipc.read_schema(pa.py_buffer(schema.serialize().to_pybytes())) + rebuilt = pa.schema([pa.field(f.name, f.type, f.nullable, f.metadata) for f in schema], metadata=schema.metadata) + for options in OPTION_SETS: + prepared = ArrowDigester.prepare(schema, **options) + for other in (via_ipc, rebuilt): + assert other.serialize().to_pybytes() == schema.serialize().to_pybytes() + _assert_same_as_one_shot(pa.Table.from_batches(table.to_batches(), schema=other), options, prepared) + + +# --------------------------------------------------------------------------- +# Differential: golden vectors, seeded random inputs +# --------------------------------------------------------------------------- + +_GOLDEN = Path(__file__).parent / "golden" / "format2_v0.5.json" + + +@cache +def _golden_record_vectors() -> list[dict[str, Any]]: + vectors = json.loads(_GOLDEN.read_text(encoding="utf-8"))["vectors"] + return [v for v in vectors if v["method"] in ("hash_schema", "hash_record_batch", "hash_stream")] + + +def _decode(vector: dict[str, Any]) -> tuple[pa.Schema, list[pa.RecordBatch]]: + reader = pa.ipc.open_stream(io.BytesIO(base64.b64decode(vector["ipc_b64"]))) + return reader.schema, list(reader) + + +def _outcome(call: Any, *args: Any, **kwargs: Any) -> str: + try: + return call(*args, **kwargs).hex() + except (ValueError, TypeError, NotImplementedError) as e: + return f"raises {type(e).__name__}: {e}" + + +def _prepared_schema_digest(schema: pa.Schema, **options: bool) -> bytes: + return ArrowDigester.prepare(schema, **options).schema_digest + + +def _prepared_hash_table(table: pa.Table, **options: bool) -> bytes: + return ArrowDigester.prepare(table.schema, **options).hash_table(table) + + +def test_golden_vectors_prepared_equal_one_shot_for_every_option_set() -> None: + """Every record/schema vector of the Rust golden file, under all 8 option combinations: the + prepared path equals the one-shot path (and the Rust hash for the vector's own options).""" + vectors = _golden_record_vectors() + assert len(vectors) > 300 + for vector in vectors: + schema, batches = _decode(vector) + own = {"include_metadata": vector["include_metadata"], "ordered_columns": vector["ordered_columns"], + "ordered_struct_fields": vector["ordered_struct_fields"]} + for options in OPTION_SETS: + prepared_or_error = _outcome(_prepared_schema_digest, schema, **options) + assert prepared_or_error == _outcome(ArrowDigester.hash_schema, schema, **options) + if prepared_or_error.startswith("raises"): + continue + prepared = ArrowDigester.prepare(schema, **options) + if vector["method"] == "hash_schema": + got = prepared.schema_digest.hex() + else: + table = pa.Table.from_batches(batches, schema=schema) + got = _outcome(prepared.hash_table, table) + assert got == _outcome(ArrowDigester.hash_table, table, **options), vector["id"] + assert got == _outcome(_prepared_stream, prepared, batches), vector["id"] + assert got == _outcome(_one_shot_stream, schema, batches, **options), vector["id"] + if len(batches) == 1: + assert got == _outcome(prepared.hash_record_batch, batches[0]), vector["id"] + if options == own: + assert got == vector["expected_hash"], vector["id"] + + +@pytest.mark.parametrize("chunk", range(8)) +def test_seeded_random_inputs_prepared_equal_one_shot(chunk: int) -> None: + for seed in range(chunk * 40, chunk * 40 + 40): + rng = random.Random(seed) + table = ra.random_table(rng) + for options in OPTION_SETS: + one_shot = _outcome(ArrowDigester.hash_table, table, **options) + if one_shot.startswith("raises"): + assert _outcome(_prepared_hash_table, table, **options) == one_shot + continue + _assert_same_as_one_shot(table, options)