From 58046057ed20638391ff2bf31acfacdc152330eb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 17:27:39 -0800 Subject: [PATCH 1/4] feat: ArrowDigester.prepare / PreparedSchema: analyse a schema once, hash many inputs (ITL-1177) PreparedSchema holds the schema analysis (nodes, canonical key, schema digest, entry specs, data-support verdict) and hashes tables, record batches and streams of that schema without analysing it again. Its hashes equal the one-shot calls byte for byte. An input is accepted exactly when its schema has the prepared canonical schema key under the prepared options (the streaming rule); identical schemas are recognised by their Arrow IPC bytes, others by computing their key. No cache or module state. One-shot calls now analyse the schema once instead of three times, and ArrowDigester.update no longer re-analyses a batch of an identical schema. Co-Authored-By: Claude Opus 5.5 --- src/starfix/__init__.py | 4 +- src/starfix/_format2.py | 226 ++++++++++--- src/starfix/arrow_digester.py | 139 ++++++++ tests/random_arrow.py | 168 ++++++++++ tests/test_format2_type_matrix.py | 17 +- tests/test_prepared_schema.py | 527 ++++++++++++++++++++++++++++++ 6 files changed, 1027 insertions(+), 54 deletions(-) create mode 100644 tests/random_arrow.py create mode 100644 tests/test_prepared_schema.py diff --git a/src/starfix/__init__.py b/src/starfix/__init__.py index ef830ad..4e53c74 100644 --- a/src/starfix/__init__.py +++ b/src/starfix/__init__.py @@ -1,3 +1,3 @@ -from starfix.arrow_digester import ArrowDigester +from starfix.arrow_digester import ArrowDigester, PreparedSchema -__all__ = ["ArrowDigester"] +__all__ = ["ArrowDigester", "PreparedSchema"] diff --git a/src/starfix/_format2.py b/src/starfix/_format2.py index cc5d82d..1701a11 100644 --- a/src/starfix/_format2.py +++ b/src/starfix/_format2.py @@ -334,10 +334,12 @@ def _field_metadata(field: pa.Field) -> dict[str, str]: return meta -def phase2_json(schema: pa.Schema) -> str | None: - """Section 2.3: metadata keyed by node path; None when there is none.""" +def phase2_json(schema: pa.Schema, nodes: list[tuple[str, pa.Field]] | None = None) -> str | None: + """Section 2.3: metadata keyed by node path; None when there is none. + + `nodes` are the schema's nodes when already walked (``schema_nodes(schema)``).""" fields = {} - for path, field in schema_nodes(schema): + for path, field in (schema_nodes(schema) if nodes is None else nodes): try: meta = _field_metadata(field) except _AmbiguousMetadata as exc: @@ -353,14 +355,19 @@ def phase2_json(schema: pa.Schema) -> str | None: return _json(doc) if doc else None -def schema_key(schema: pa.Schema, include_metadata: bool, flags: int) -> tuple[str, str | None]: - return phase1_json(schema, flags), (phase2_json(schema) if include_metadata else None) +def schema_key( + schema: pa.Schema, include_metadata: bool, flags: int, nodes: list[tuple[str, pa.Field]] | None = None +) -> tuple[str, str | None]: + """The canonical schema text (Phase 1, and Phase 2 with include_metadata): everything of the + schema that enters a hash. Walks the nodes (Section 7 rejections) unless `nodes` is given.""" + if nodes is None: + nodes = schema_nodes(schema) + return phase1_json(schema, flags), (phase2_json(schema, nodes) if include_metadata else None) -def schema_digest(schema: pa.Schema, include_metadata: bool, flags: int) -> bytes: +def _digest_of_key(key: tuple[str, str | None], flags: int) -> bytes: """Section 2.5.""" - schema_nodes(schema) # Section 7 rejections - phase1, phase2 = schema_key(schema, include_metadata, flags) + phase1, phase2 = key h = hashlib.sha256(header(KIND_SCHEMA, flags)) h.update(phase1.encode()) if phase2 is not None: @@ -368,6 +375,11 @@ def schema_digest(schema: pa.Schema, include_metadata: bool, flags: int) -> byte return h.digest() +def schema_digest(schema: pa.Schema, include_metadata: bool, flags: int) -> bytes: + """Section 2.5.""" + return _digest_of_key(schema_key(schema, include_metadata, flags), flags) + + # --------------------------------------------------------------------------- # Entries (Section 3.2) # --------------------------------------------------------------------------- @@ -418,22 +430,25 @@ def restore(self, state: tuple[int, int, int | None]) -> None: self._len = bit_len +def _entry_kind(field_type: pa.DataType) -> str: + t = _logical(field_type) + if pa.types.is_struct(t): + return "struct" + if _is_list(t): + return "list" + if pa.types.is_fixed_size_list(t): + return "fixed_size_list" # Section 3.2: validity only (n is in the schema) + return "leaf" + + class _Entry: __slots__ = ("kind", "lengths", "validity", "values") - def __init__(self, field: pa.Field) -> None: - t = _logical(field.type) - if pa.types.is_struct(t): - self.kind = "struct" - elif _is_list(t): - self.kind = "list" - elif pa.types.is_fixed_size_list(t): - self.kind = "fixed_size_list" # Section 3.2: validity only (n is in the schema) - else: - self.kind = "leaf" - self.validity = _Bits() if field.nullable else None - self.lengths = hashlib.sha256() if self.kind == "list" else None - self.values = hashlib.sha256() if self.kind == "leaf" else None + def __init__(self, kind: str, nullable: bool) -> None: + self.kind = kind + self.validity = _Bits() if nullable else None + self.lengths = hashlib.sha256() if kind == "list" else None + self.values = hashlib.sha256() if kind == "leaf" else None def snapshot(self) -> tuple: return ( @@ -784,16 +799,24 @@ def _is_unsupported_data(t: pa.DataType) -> bool: ) -def _check_data_supported(nodes: list[tuple[str, pa.Field]]) -> None: - """Section 7 item 3: runs before any data is hashed, whatever the number of rows.""" +def _data_support_error(nodes: list[tuple[str, pa.Field]]) -> str | None: + """Section 7 item 3: the error for a schema whose data cannot be hashed (None: it can).""" for path, field in nodes: t = _logical(field.type) if _is_unsupported_data(t): - raise ValueError(f"Hashing data of type {t} (node {path!r}) is not supported") + return f"Hashing data of type {t} (node {path!r}) is not supported" + return None + + +def _check_data_supported(nodes: list[tuple[str, pa.Field]]) -> None: + """Section 7 item 3: runs before any data is hashed, whatever the number of rows.""" + error = _data_support_error(nodes) + if error is not None: + raise ValueError(error) def _new_entries(nodes: list[tuple[str, pa.Field]]) -> dict[str, _Entry]: - return {path: _Entry(field) for path, field in nodes} + return {path: _Entry(_entry_kind(field.type), field.nullable) for path, field in nodes} def _feed_entries(entries: dict[str, _Entry], digest) -> None: @@ -806,6 +829,72 @@ def _feed_entries(entries: dict[str, _Entry], digest) -> None: # Public entry points (Sections 2.5, 3.3, 4, 5) # --------------------------------------------------------------------------- +def _ipc_bytes(schema: pa.Schema) -> bytes | None: + """The schema as an Arrow IPC schema message (None if pyarrow cannot serialize it).""" + try: + return schema.serialize().to_pybytes() + except Exception: # noqa: BLE001 - only disables the fast path of SchemaAnalysis.matches + return None + + +class SchemaAnalysis: + """Everything the schema analysis produces, computed once (ITL-1177); immutable. + + Holds the hashing options, the canonical schema key and digest (Section 2.5), the + entries to create for the schema's nodes in node-path order (Section 3.2), the + data-support verdict (Section 7 item 3) and the schema's Arrow IPC bytes, which make + `matches` cheap for an identical schema. + """ + + __slots__ = ("_data_error", "_ipc", "_specs", "digest", "flags", "include_metadata", "key") + + def __init__(self, schema: pa.Schema, include_metadata: bool, flags: int) -> None: + nodes = schema_nodes(schema) # Section 7 rejections, once + self.include_metadata = include_metadata + self.flags = flags + self.key = schema_key(schema, include_metadata, flags, nodes) + self.digest = _digest_of_key(self.key, flags) + # Entries in byte-wise node-path order (Section 3.3), so finalize needs no sort. + self._specs = tuple( + (path, _entry_kind(field.type), field.nullable) for path, field in sorted(nodes, key=lambda n: n[0]) + ) + self._data_error = _data_support_error(nodes) + self._ipc = _ipc_bytes(schema) + + def __setattr__(self, name: str, value: Any) -> None: + if hasattr(self, "_ipc"): # the last slot set by __init__ + raise AttributeError("SchemaAnalysis is immutable") + object.__setattr__(self, name, value) + + def require_data_supported(self) -> None: + if self._data_error is not None: + raise ValueError(self._data_error) + + def new_entries(self) -> dict[str, _Entry]: + return {path: _Entry(kind, nullable) for path, kind, nullable in self._specs} + + def matches(self, schema: pa.Schema) -> bool: + """Whether `schema` has this schema key: everything that enters a hash under these + options is the same (Section 5's rule for streamed batches). + + Fast path: identical Arrow IPC bytes mean an identical schema (names, types, + nullability, order, all field and schema metadata at every depth, extension names, + serializations and storage types). Otherwise the key is computed and compared, + which accepts hash-equivalent schemas (e.g. string vs large_string) and raises + Section 7 errors exactly as the full analysis does. + """ + if self._ipc is not None and _ipc_bytes(schema) == self._ipc: + return True + return schema_key(schema, self.include_metadata, self.flags) == self.key + + +def analyse(schema: pa.Schema, entry_point: str, *, include_metadata: bool = False, ordered_columns: bool = False, + ordered_struct_fields: bool = False) -> SchemaAnalysis: + _require_little_endian() + _require(schema, pa.Schema, entry_point, "a pyarrow.Schema") + return SchemaAnalysis(schema, include_metadata, flags_of(ordered_columns, ordered_struct_fields)) + + class Digester: """Streaming record-batch hasher (Section 3.3).""" @@ -817,41 +906,51 @@ def __init__( ordered_columns: bool = False, ordered_struct_fields: bool = False, ) -> None: - _require_little_endian() - _require(schema, pa.Schema, "ArrowDigester", "a pyarrow.Schema") - self._flags = flags_of(ordered_columns, ordered_struct_fields) - self._include_metadata = include_metadata - self._schema_digest = schema_digest(schema, include_metadata, self._flags) - self._key = schema_key(schema, include_metadata, self._flags) - nodes = schema_nodes(schema) - _check_data_supported(nodes) - self._entries = _new_entries(nodes) + self._start(analyse(schema, "ArrowDigester", include_metadata=include_metadata, + ordered_columns=ordered_columns, ordered_struct_fields=ordered_struct_fields)) + + @classmethod + def from_analysis(cls, analysis: SchemaAnalysis) -> Digester: + """A digester for a schema analysed earlier (no analysis is repeated).""" + digester = cls.__new__(cls) + digester._start(analysis) + return digester + + def _start(self, analysis: SchemaAnalysis) -> None: + analysis.require_data_supported() + self._analysis = analysis + self._entries = analysis.new_entries() self._rows = 0 def update(self, batch: pa.RecordBatch) -> None: _require(batch, pa.RecordBatch, "update", "a pyarrow.RecordBatch (use hash_table for a Table)") schema = batch.schema - schema_nodes(schema) # Section 7 - if schema_key(schema, self._include_metadata, self._flags) != self._key: + if not self._analysis.matches(schema): raise ValueError(_SCHEMA_MISMATCH) # A batch rejected while it is traversed (Section 7 item 4) leaves the digester # exactly as it was. saved = {path: entry.snapshot() for path, entry in self._entries.items()} try: - for i in range(batch.num_columns): - field = schema.field(i) - _visit(field, batch.column(i), field.name, None, self._entries) + self.ingest(batch, schema) except BaseException: for path, state in saved.items(): self._entries[path].restore(state) raise + + def ingest(self, batch: pa.RecordBatch, schema: pa.Schema) -> None: + """Add a batch whose schema `schema` is known to match (unchecked; on error the + digester is left partly updated and must be discarded).""" + for i in range(batch.num_columns): + field = schema.field(i) + _visit(field, batch.column(i), field.name, None, self._entries) self._rows += batch.num_rows def finalize(self) -> bytes: - final = hashlib.sha256(header(KIND_RECORDS, self._flags)) - final.update(self._schema_digest) + final = hashlib.sha256(header(KIND_RECORDS, self._analysis.flags)) + final.update(self._analysis.digest) final.update(_U64.pack(self._rows)) - _feed_entries(self._entries, final) + for entry in self._entries.values(): # created in node-path order + entry.feed(final) return VERSION_BYTES + final.digest() @@ -867,21 +966,50 @@ def hash_schema(schema: pa.Schema, *, include_metadata: bool = False, ordered_co return VERSION_BYTES + schema_digest(schema, include_metadata, flags_of(ordered_columns, ordered_struct_fields)) +_BATCH_TYPE = "a pyarrow.RecordBatch (use hash_table for a Table)" +_TABLE_TYPE = "a pyarrow.Table (use hash_record_batch for a RecordBatch)" + + def hash_record_batch(batch: pa.RecordBatch, **options: bool) -> bytes: - _require(batch, pa.RecordBatch, "hash_record_batch", "a pyarrow.RecordBatch (use hash_table for a Table)") - d = Digester(batch.schema, **options) - d.update(batch) + _require(batch, pa.RecordBatch, "hash_record_batch", _BATCH_TYPE) + schema = batch.schema + d = Digester.from_analysis(analyse(schema, "hash_record_batch", **options)) + d.ingest(batch, schema) # the batch's own schema: nothing to check return d.finalize() def hash_table(table: pa.Table, **options: bool) -> bytes: - _require(table, pa.Table, "hash_table", "a pyarrow.Table (use hash_record_batch for a RecordBatch)") - d = Digester(table.schema, **options) - for batch in table.to_batches(): - d.update(batch) + _require(table, pa.Table, "hash_table", _TABLE_TYPE) + schema = table.schema + return _hash_table_batches(Digester.from_analysis(analyse(schema, "hash_table", **options)), table, schema) + + +def _hash_table_batches(d: Digester, table: pa.Table, schema: pa.Schema) -> bytes: + for batch in table.to_batches(): # a table's batches carry the table's schema + d.ingest(batch, schema) return d.finalize() +def hash_prepared_record_batch(analysis: SchemaAnalysis, batch: pa.RecordBatch) -> bytes: + _require(batch, pa.RecordBatch, "hash_record_batch", _BATCH_TYPE) + analysis.require_data_supported() + schema = batch.schema + if not analysis.matches(schema): + raise ValueError("Record batch schema does not match the prepared schema") + d = Digester.from_analysis(analysis) + d.ingest(batch, schema) + return d.finalize() + + +def hash_prepared_table(analysis: SchemaAnalysis, table: pa.Table) -> bytes: + _require(table, pa.Table, "hash_table", _TABLE_TYPE) + analysis.require_data_supported() + schema = table.schema + if not analysis.matches(schema): + raise ValueError("Table schema does not match the prepared schema") + return _hash_table_batches(Digester.from_analysis(analysis), table, schema) + + def hash_array(array: pa.Array, *, ordered_columns: bool = False, ordered_struct_fields: bool = False) -> bytes: """Section 4: the array is the node at path "", always nullable. diff --git a/src/starfix/arrow_digester.py b/src/starfix/arrow_digester.py index cdb641c..c5a4cd3 100644 --- a/src/starfix/arrow_digester.py +++ b/src/starfix/arrow_digester.py @@ -37,6 +37,9 @@ class ArrowDigester: A digester is not thread-safe: feed one digester from one thread at a time. A batch rejected by ``update`` leaves the digester unchanged. + + To hash many tables or batches of one schema, analyse the schema once with + ``ArrowDigester.prepare`` and hash through the returned ``PreparedSchema``. """ def __init__( @@ -65,6 +68,43 @@ def __init__( ordered_struct_fields=ordered_struct_fields, ) + @classmethod + def _from_analysis(cls, analysis: _format2.SchemaAnalysis) -> ArrowDigester: + digester = cls.__new__(cls) + digester._digester = _format2.Digester.from_analysis(analysis) + return digester + + @staticmethod + def prepare( + schema: pa.Schema, + *, + include_metadata: bool = False, + ordered_columns: bool = False, + ordered_struct_fields: bool = False, + ) -> PreparedSchema: + """Analyse a schema once, for hashing many tables or record batches of it. + + Returns an immutable ``PreparedSchema`` owned by the caller; its hashes equal the + one-shot calls with the same options. Raises what ``hash_schema`` raises. A schema + whose data cannot be hashed (Null, Map, Interval, view types) still prepares, for its + ``schema_digest``; hashing data with it raises ``ValueError``. + + Args: + schema: The Arrow schema of the data to hash. + include_metadata: Hash schema- and field-level metadata. + ordered_columns: Make the order of the top-level columns part of the hash. + ordered_struct_fields: Make the order of struct fields, at every depth, + part of the hash. + """ + analysis = _format2.analyse( + schema, + "prepare", + include_metadata=include_metadata, + ordered_columns=ordered_columns, + ordered_struct_fields=ordered_struct_fields, + ) + return PreparedSchema._create(schema, analysis) + def update(self, record_batch: pa.RecordBatch) -> None: """Add a record batch. Splitting rows into batches does not change the hash.""" self._digester.update(record_batch) @@ -136,3 +176,102 @@ def hash_array( return _format2.hash_array( array, ordered_columns=ordered_columns, ordered_struct_fields=ordered_struct_fields ) + + +class PreparedSchema: + """A schema analysed once, with its hashing options (create with ``ArrowDigester.prepare``). + + Hashes many tables or record batches of the schema without analysing it again; every + hash is byte-identical to the one-shot ``ArrowDigester`` call with the same options. + Immutable and owned by the caller: starfix keeps no cache or other state of its own. Safe + to share between threads (each hash or ``digester()`` has its own state). + + **Same-schema rule.** A table or record batch is accepted exactly when its schema has the + prepared schema's canonical schema key under the prepared options, i.e. when + ``ArrowDigester.hash_schema(its_schema, **options) == prepared.schema_digest``: the same + column names, logical types and nullability at every depth (column order when + ``ordered_columns``, struct field order when ``ordered_struct_fields``) and, with + ``include_metadata``, the same field metadata at every node, schema metadata and extension + type names and serializations. This is the rule ``ArrowDigester.update`` applies to + streamed batches. Anything else raises ``ValueError``; nothing is hashed with another + schema's analysis. An input whose schema is identical to the prepared one (equal Arrow + IPC serialization) is accepted by a byte comparison; other inputs, such as a ``string`` + column for a prepared ``large_string`` one, are checked by computing their schema key. + """ + + __slots__ = ("_analysis", "_schema") + + def __init__(self, *args: object, **kwargs: object) -> None: + raise TypeError("PreparedSchema is created by ArrowDigester.prepare(schema, ...)") + + @classmethod + def _create(cls, schema: pa.Schema, analysis: _format2.SchemaAnalysis) -> PreparedSchema: + prepared = object.__new__(cls) + object.__setattr__(prepared, "_schema", schema) + object.__setattr__(prepared, "_analysis", analysis) + return prepared + + def __setattr__(self, name: str, value: object) -> None: + raise AttributeError(f"PreparedSchema is immutable (cannot set {name!r})") + + def __delattr__(self, name: str) -> None: + raise AttributeError(f"PreparedSchema is immutable (cannot delete {name!r})") + + def __copy__(self) -> PreparedSchema: + return self + + def __deepcopy__(self, memo: dict) -> PreparedSchema: + return self + + def __reduce__(self) -> tuple: + # Pickled as (schema, options) and prepared again when loaded. + return (_prepare, (self._schema, self.include_metadata, self.ordered_columns, self.ordered_struct_fields)) + + def __repr__(self) -> str: + return ( + f"PreparedSchema(include_metadata={self.include_metadata}, ordered_columns={self.ordered_columns}, " + f"ordered_struct_fields={self.ordered_struct_fields}, schema_digest={self.schema_digest.hex()})" + ) + + @property + def schema(self) -> pa.Schema: + """The prepared Arrow schema.""" + return self._schema + + @property + def include_metadata(self) -> bool: + return self._analysis.include_metadata + + @property + def ordered_columns(self) -> bool: + return bool(self._analysis.flags & _format2.ORDERED_COLUMNS) + + @property + def ordered_struct_fields(self) -> bool: + return bool(self._analysis.flags & _format2.ORDERED_STRUCT_FIELDS) + + @property + def schema_digest(self) -> bytes: + """The 35-byte schema hash, equal to ``ArrowDigester.hash_schema`` with the same options.""" + return VERSION_BYTES + self._analysis.digest + + def hash_record_batch(self, record_batch: pa.RecordBatch) -> bytes: + """Hash one record batch of this schema (see the same-schema rule).""" + return _format2.hash_prepared_record_batch(self._analysis, record_batch) + + def hash_table(self, table: pa.Table) -> bytes: + """Hash a table of this schema (see the same-schema rule); checked once per table.""" + return _format2.hash_prepared_table(self._analysis, table) + + def digester(self) -> ArrowDigester: + """A fresh streaming ``ArrowDigester`` for this schema and options, without analysis.""" + return ArrowDigester._from_analysis(self._analysis) + + +def _prepare(schema: pa.Schema, include_metadata: bool, ordered_columns: bool, ordered_struct_fields: bool) -> PreparedSchema: + return ArrowDigester.prepare( + schema, + include_metadata=include_metadata, + ordered_columns=ordered_columns, + ordered_struct_fields=ordered_struct_fields, + ) diff --git a/tests/random_arrow.py b/tests/random_arrow.py new file mode 100644 index 0000000..3eef5aa --- /dev/null +++ b/tests/random_arrow.py @@ -0,0 +1,168 @@ +"""Seeded random Arrow schemas and data for differential tests (ITL-1177). + +Used by ``tests/test_prepared_schema.py`` and ``scripts/release_differential.py`` (which runs it +against the released starfix in an isolated environment), so it depends on pyarrow only. +Everything is built from ``random.Random(seed)``: the same seed gives the same inputs in any +process. Covers every type family whose data hashes: fixed-width and variable-length leaves, +booleans, dictionaries (of leaves and of structs), list / large_list, fixed-size lists, structs, +extension types (canonical and Python-defined), field metadata at every depth, schema metadata, +nulls, sliced arrays, empty batches and multi-batch tables. +""" + +from __future__ import annotations + +import random +from decimal import Decimal + +import pyarrow as pa +import pyarrow.compute as pc + + +class UnitType(pa.ExtensionType): + """A Python-defined extension type with a parameter carried only in its serialization.""" + + def __init__(self, unit: str = "m") -> None: + self.unit = unit + super().__init__(pa.int64(), "starfix.test.unit") + + def __arrow_ext_serialize__(self) -> bytes: + return self.unit.encode() + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type: pa.DataType, serialized: bytes) -> UnitType: + return cls(serialized.decode()) + + +LEAVES: list[pa.DataType] = [ + pa.int8(), pa.int16(), pa.int32(), pa.int64(), pa.uint8(), pa.uint32(), pa.uint64(), + pa.float16(), pa.float32(), pa.float64(), pa.decimal128(10, 2), pa.decimal256(40, 5), + pa.date32(), pa.date64(), pa.time32("ms"), pa.time64("ns"), + pa.timestamp("us"), pa.timestamp("ms", tz="UTC"), pa.duration("s"), + pa.binary(4), pa.binary(0), pa.string(), pa.large_string(), pa.binary(), pa.large_binary(), pa.bool_(), +] + + +def _extension_types() -> list[pa.DataType]: + types: list[pa.DataType] = [UnitType("m"), UnitType("s"), pa.fixed_shape_tensor(pa.float32(), [2, 2])] + for name in ("uuid", "json_", "bool8"): + factory = getattr(pa, name, None) + if factory is not None: + types.append(factory()) + return types + + +EXTENSIONS = _extension_types() +_NAMES = ["a", "b", "c", "d", "é"] + + +def random_type(rng: random.Random, depth: int = 0) -> pa.DataType: + kind = rng.random() + if depth >= 3 or kind < 0.35: + return rng.choice(LEAVES) + if kind < 0.45: + value = rng.choice([pa.string(), pa.int64(), pa.float32()]) + return pa.dictionary(rng.choice([pa.int8(), pa.int32()]), value) + if kind < 0.5: + return pa.dictionary(pa.int32(), pa.struct([random_field(rng, n, depth + 1) for n in ("x", "y")])) + if kind < 0.6: + return rng.choice(EXTENSIONS) + if kind < 0.75: + element = random_field(rng, "item", depth + 1) + return pa.large_list(element) if rng.random() < 0.3 else pa.list_(element) + if kind < 0.82: + return pa.list_(random_field(rng, "item", depth + 1), rng.choice([0, 1, 2, 3])) + names = rng.sample(_NAMES, rng.randint(0, 3)) + return pa.struct([random_field(rng, n, depth + 1) for n in names]) + + +def random_metadata(rng: random.Random) -> dict[str, str] | None: + if rng.random() < 0.6: + return None + return {rng.choice(["unit", "z", "é", "a"]): rng.choice(["kg", "", "v1", "ü"]) for _ in range(rng.randint(1, 2))} + + +def random_field(rng: random.Random, name: str, depth: int = 0) -> pa.Field: + return pa.field(name, random_type(rng, depth), nullable=rng.random() < 0.6, metadata=random_metadata(rng)) + + +def random_schema(rng: random.Random) -> pa.Schema: + names = rng.sample(_NAMES, rng.randint(0, 4)) + return pa.schema([random_field(rng, n) for n in names], metadata=random_metadata(rng)) + + +def _leaf_values(t: pa.DataType, n: int, nullable: bool, rng: random.Random) -> pa.Array: + def value(): + if pa.types.is_boolean(t): + return rng.random() < 0.5 + if pa.types.is_floating(t): + return rng.uniform(-1e3, 1e3) + if pa.types.is_decimal(t): + return Decimal(rng.randint(-99999, 99999)) / 100 + if pa.types.is_fixed_size_binary(t): + return bytes(rng.randrange(256) for _ in range(t.byte_width)) + if pa.types.is_binary(t) or pa.types.is_large_binary(t): + return bytes(rng.randrange(256) for _ in range(rng.randint(0, 5))) + if pa.types.is_string(t) or pa.types.is_large_string(t): + return "".join(rng.choice("xyzé") for _ in range(rng.randint(0, 5))) + if pa.types.is_temporal(t): + return rng.randint(0, 20000) + if pa.types.is_unsigned_integer(t): + return rng.randint(0, 2 ** min(t.bit_width, 16) - 1) + return rng.randint(-(2 ** (t.bit_width - 1)), 2 ** (t.bit_width - 1) - 1) + + values = [None if nullable and rng.random() < 0.25 else value() for _ in range(n)] + if pa.types.is_float16(t): # half-float bit patterns (no numpy needed) + return pa.array([None if v is None else rng.randrange(1 << 16) for v in values], pa.uint16()).view(t) + if pa.types.is_temporal(t): + storage = pa.int64() if t.bit_width == 64 else pa.int32() + return pa.array([None if v is None else v * (86_400_000 if pa.types.is_date64(t) else 1) for v in values], + storage).cast(t) + return pa.array(values, t) + + +def random_array(t: pa.DataType, n: int, nullable: bool, rng: random.Random) -> pa.Array: + """An array of `n` elements of type `t`, sliced out of a longer one (non-zero offsets).""" + pad = rng.randint(0, 3) + total = n + pad + mask = pa.array([rng.random() < 0.25 for _ in range(total)], pa.bool_()) if nullable else None + if isinstance(t, pa.BaseExtensionType): + arr = pa.ExtensionArray.from_storage(t, random_array(t.storage_type, total, nullable, rng)) + elif pa.types.is_dictionary(t): + values = random_array(t.value_type, rng.randint(1, 4), nullable and rng.random() < 0.3, rng) + index_values = [None if nullable and rng.random() < 0.2 else rng.randrange(len(values)) for _ in range(total)] + arr = pa.DictionaryArray.from_arrays(pa.array(index_values, t.index_type), values) + elif pa.types.is_list(t) or pa.types.is_large_list(t): + lengths = [rng.choice([0, 0, 1, 2, 3]) for _ in range(total)] + offsets = [0] + for length in lengths: + offsets.append(offsets[-1] + length) + child = random_array(t.value_type, offsets[-1], t.value_field.nullable, rng) + cls = pa.LargeListArray if pa.types.is_large_list(t) else pa.ListArray + index_type = pa.int64() if pa.types.is_large_list(t) else pa.int32() + arr = cls.from_arrays(pa.array(offsets, index_type), child, type=t, mask=mask) + elif pa.types.is_fixed_size_list(t): + child = random_array(t.value_type, total * t.list_size, t.value_field.nullable, rng) + validity = None if mask is None else pc.invert(mask).buffers()[1] + arr = pa.Array.from_buffers(t, total, [validity], null_count=-1, children=[child]) # also n = 0 + elif pa.types.is_struct(t) and t.num_fields == 0: + validity = None if mask is None else pc.invert(mask).buffers()[1] + arr = pa.Array.from_buffers(t, total, [validity], null_count=-1) + elif pa.types.is_struct(t): + children = [random_array(f.type, total, f.nullable, rng) for f in t] + arr = pa.StructArray.from_arrays(children, fields=list(t), mask=mask) + else: + arr = _leaf_values(t, total, nullable, rng) + return arr.slice(rng.randint(0, pad), n) + + +def random_table(rng: random.Random, schema: pa.Schema | None = None) -> pa.Table: + """A table of a random (or the given) schema, split into 1-3 batches (some may be empty).""" + schema = schema if schema is not None else random_schema(rng) + rows = rng.choice([0, 1, 1, 2, 5, 17]) + columns = [random_array(f.type, rows, f.nullable, rng) for f in schema] + batch = pa.RecordBatch.from_arrays(columns, schema=schema) if columns else pa.RecordBatch.from_pylist( + [{}] * rows, schema=schema + ) + cuts = sorted(rng.choices(range(rows + 1), k=rng.randint(0, 2))) + pieces = [batch.slice(a, b - a) for a, b in zip([0, *cuts], [*cuts, rows], strict=True)] + return pa.Table.from_batches(pieces, schema=schema) diff --git a/tests/test_format2_type_matrix.py b/tests/test_format2_type_matrix.py index e0dd080..7bf656f 100644 --- a/tests/test_format2_type_matrix.py +++ b/tests/test_format2_type_matrix.py @@ -291,23 +291,33 @@ def wrap(child: pa.Array) -> pa.Array: def _hashes(arr: pa.Array | pa.ChunkedArray) -> dict[str, str]: """hash_array, the record hash (one batch, or a table of the chunks) and the record hash - streamed through the digester in two batches per chunk.""" + streamed through the digester in two batches per chunk; the record hashes also through a + prepared schema (ITL-1177).""" schema = pa.schema([pa.field("c", arr.type, nullable=True)]) + prepared = ArrowDigester.prepare(schema) if isinstance(arr, pa.ChunkedArray): chunks = arr.chunks - records = ArrowDigester.hash_table(pa.Table.from_arrays([arr], schema=schema)) + table = pa.Table.from_arrays([arr], schema=schema) + records = ArrowDigester.hash_table(table) + prepared_records = prepared.hash_table(table) else: chunks = [arr] - records = ArrowDigester.hash_record_batch(pa.record_batch([arr], schema=schema)) + batch = pa.record_batch([arr], schema=schema) + records = ArrowDigester.hash_record_batch(batch) + prepared_records = prepared.hash_record_batch(batch) digester = ArrowDigester(schema) + prepared_digester = prepared.digester() for chunk in chunks: k = len(chunk) // 2 # the second batch starts at a non-zero offset for part in (chunk.slice(0, k), chunk.slice(k)): digester.update(pa.record_batch([part], schema=schema)) + prepared_digester.update(pa.record_batch([part], schema=schema)) return { "hash_array": ArrowDigester.hash_array(arr).hex(), "records": records.hex(), "stream": digester.finalize().hex(), + "prepared_records": prepared_records.hex(), + "prepared_stream": prepared_digester.finalize().hex(), } @@ -333,6 +343,7 @@ def test_type_matrix(leaf_type: pa.DataType, base: pa.Array, golden: str | None, reference = reference_wrap(base) expected = _hashes(reference) assert expected["stream"] == expected["records"], "streamed batches differ from one batch" + assert expected["prepared_records"] == expected["prepared_stream"] == expected["records"], "prepared differs" if golden is not None: assert expected["hash_array"] == golden, "plain array differs from the Rust golden vector" diff --git a/tests/test_prepared_schema.py b/tests/test_prepared_schema.py new file mode 100644 index 0000000..520c8f2 --- /dev/null +++ b/tests/test_prepared_schema.py @@ -0,0 +1,527 @@ +"""A prepared schema: analyse once, hash many inputs of that schema (ITL-1177). + +Every digest of the prepared path must be byte-identical to the one-shot path for the same input +and options; a batch or table whose schema differs from the prepared one under the prepared +options is refused; starfix holds no state of its own between calls. +""" + +from __future__ import annotations + +import base64 +import copy +import io +import itertools +import json +import pickle +import random +from functools import cache +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest +import random_arrow as ra # tests/ is on sys.path (pytest rootdir-relative import) + +import starfix +from starfix import ArrowDigester, PreparedSchema, _format2, arrow_digester + +OPTION_SETS = [ + {"include_metadata": m, "ordered_columns": c, "ordered_struct_fields": s} + for m, c, s in itertools.product([False, True], repeat=3) +] +_OPTION_IDS = [f"m{int(o['include_metadata'])}c{int(o['ordered_columns'])}s{int(o['ordered_struct_fields'])}" + for o in OPTION_SETS] +_BATCH_MISMATCH = "^Record batch schema does not match the prepared schema$" +_TABLE_MISMATCH = "^Table schema does not match the prepared schema$" +_STREAM_MISMATCH = "^Record batch schema does not match ArrowDigester schema$" + + +def _one_shot_stream(schema: pa.Schema, batches: list[pa.RecordBatch], **options: bool) -> bytes: + d = ArrowDigester(schema, **options) + for batch in batches: + d.update(batch) + return d.finalize() + + +def _prepared_stream(prepared: PreparedSchema, batches: list[pa.RecordBatch]) -> bytes: + d = prepared.digester() + for batch in batches: + d.update(batch) + return d.finalize() + + +def _assert_same_as_one_shot(table: pa.Table, options: dict[str, bool], prepared: PreparedSchema | None = None) -> None: + """Every prepared entry point equals its one-shot counterpart on `table` (and its batches).""" + prepared = prepared if prepared is not None else ArrowDigester.prepare(table.schema, **options) + batches = table.to_batches() + expected = ArrowDigester.hash_table(table, **options) + assert prepared.hash_table(table) == expected + assert _prepared_stream(prepared, batches) == expected + assert _one_shot_stream(table.schema, batches, **options) == expected + for batch in batches: + assert prepared.hash_record_batch(batch) == ArrowDigester.hash_record_batch(batch, **options) + + +def _simple_table() -> pa.Table: + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False, metadata={"role": "key"}), + pa.field("name", pa.string()), + pa.field("tags", pa.list_(pa.field("item", pa.string(), metadata={"k": "v"}))), + pa.field("s", pa.struct([pa.field("b", pa.float64()), pa.field("a", pa.int8())])), + ], + metadata={"origin": "test"}, + ) + return pa.table( + {"id": [1, 2, 3], "name": ["x", None, "z"], "tags": [["p"], [], None], "s": [{"b": 1.0, "a": 1}, None, {"b": None, "a": 3}]}, + schema=schema, + ) + + +# --------------------------------------------------------------------------- +# The interface +# --------------------------------------------------------------------------- + +def test_prepared_schema_is_exported() -> None: + assert starfix.PreparedSchema is PreparedSchema + assert "PreparedSchema" in starfix.__all__ + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +def test_prepare_echoes_schema_and_options(options: dict[str, bool]) -> None: + table = _simple_table() + prepared = ArrowDigester.prepare(table.schema, **options) + assert isinstance(prepared, PreparedSchema) + assert prepared.schema.equals(table.schema, check_metadata=True) + for name, value in options.items(): + assert getattr(prepared, name) is value + assert prepared.schema_digest == ArrowDigester.hash_schema(table.schema, **options) + assert len(prepared.schema_digest) == 35 + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +def test_prepared_hashes_equal_one_shot(options: dict[str, bool]) -> None: + table = _simple_table() + _assert_same_as_one_shot(table, options) + _assert_same_as_one_shot(table.slice(1, 1), options) + _assert_same_as_one_shot(table.slice(0, 0), options) + two_batches = pa.Table.from_batches([table.slice(0, 1).to_batches()[0], table.slice(1).to_batches()[0]]) + _assert_same_as_one_shot(two_batches, options) + + +def test_prepare_options_are_keyword_only() -> None: + with pytest.raises(TypeError): + ArrowDigester.prepare(_simple_table().schema, True) # type: ignore[misc] + + +def test_prepared_schema_cannot_be_constructed_directly() -> None: + with pytest.raises(TypeError, match=r"ArrowDigester\.prepare"): + PreparedSchema() # type: ignore[call-arg] + + +def test_prepared_schema_is_immutable() -> None: + prepared = ArrowDigester.prepare(_simple_table().schema) + for name in ("schema", "schema_digest", "include_metadata", "_analysis", "anything"): + with pytest.raises(AttributeError): + setattr(prepared, name, None) + with pytest.raises(AttributeError): + del prepared.schema + assert not hasattr(prepared, "__dict__") + + +def test_prepared_schema_copies_and_pickles() -> None: + table = _simple_table() + options = {"include_metadata": True, "ordered_columns": True, "ordered_struct_fields": False} + prepared = ArrowDigester.prepare(table.schema, **options) + assert copy.copy(prepared) is prepared + assert copy.deepcopy(prepared) is prepared + restored = pickle.loads(pickle.dumps(prepared)) # re-prepared from (schema, options) + assert restored is not prepared + assert restored.schema_digest == prepared.schema_digest + assert restored.hash_table(table) == prepared.hash_table(table) + for name, value in options.items(): + assert getattr(restored, name) is value + + +def test_digester_returns_a_fresh_arrow_digester() -> None: + table = _simple_table() + prepared = ArrowDigester.prepare(table.schema) + first, second = prepared.digester(), prepared.digester() + assert isinstance(first, ArrowDigester) and first is not second + first.update(table.to_batches()[0]) + assert second.finalize() == ArrowDigester.hash_table(table.slice(0, 0)) + assert first.finalize() == ArrowDigester.hash_table(table) + + +def test_wrong_container_types() -> None: + table = _simple_table() + prepared = ArrowDigester.prepare(table.schema) + with pytest.raises(TypeError, match="use hash_table for a Table"): + prepared.hash_record_batch(table) # type: ignore[arg-type] + with pytest.raises(TypeError, match="use hash_record_batch for a RecordBatch"): + prepared.hash_table(table.to_batches()[0]) # type: ignore[arg-type] + with pytest.raises(TypeError, match="a pyarrow.Schema"): + ArrowDigester.prepare(table) # type: ignore[arg-type] + + +def test_prepare_raises_what_hash_schema_raises() -> None: + duplicate = pa.schema([pa.field("a", pa.int8()), pa.field("a", pa.int8())]) + with pytest.raises(ValueError, match="Ambiguous node path"): + ArrowDigester.hash_schema(duplicate) + with pytest.raises(ValueError, match="Ambiguous node path"): + ArrowDigester.prepare(duplicate) + rejected = pa.schema([pa.field("u", pa.dense_union([pa.field("x", pa.int8())]))]) + with pytest.raises(ValueError, match="Unsupported Arrow type"): + ArrowDigester.prepare(rejected) + + +@pytest.mark.parametrize( + "data_type", + [pa.null(), pa.map_(pa.string(), pa.int8()), pa.month_day_nano_interval(), pa.string_view(), pa.list_(pa.null())], + ids=str, +) +def test_schema_only_types_prepare_but_do_not_hash_data(data_type: pa.DataType) -> None: + schema = pa.schema([pa.field("c", data_type)]) + prepared = ArrowDigester.prepare(schema) + assert prepared.schema_digest == ArrowDigester.hash_schema(schema) + table = pa.table({"c": pa.array([], data_type)}) + match = r"^Hashing data of type .* \(node 'c/?'\) is not supported$" + for call in (lambda: prepared.digester(), lambda: prepared.hash_table(table), + lambda: prepared.hash_record_batch(pa.RecordBatch.from_pylist([], schema=table.schema))): + with pytest.raises(ValueError, match=match): + call() + with pytest.raises(ValueError, match=match): + ArrowDigester(schema) # unchanged: the streaming constructor still raises + + +# --------------------------------------------------------------------------- +# No state outside the prepared object +# --------------------------------------------------------------------------- + +def _module_state() -> dict[str, dict[str, int]]: + return {m.__name__: {k: id(v) for k, v in vars(m).items()} for m in (_format2, arrow_digester, starfix)} + + +def test_no_module_level_state_is_created_or_changed() -> None: + before = _module_state() + table = _simple_table() + for options in OPTION_SETS: + prepared = ArrowDigester.prepare(table.schema, **options) + prepared.hash_table(table) + ArrowDigester.hash_table(table, **options) + prepared.digester().update(table.to_batches()[0]) + assert _module_state() == before + + +def test_preparing_a_second_schema_does_not_affect_the_first() -> None: + table = _simple_table() + first = ArrowDigester.prepare(table.schema) + expected = first.hash_table(table) + other = pa.table({"q": [1.5]}) + ArrowDigester.prepare(other.schema).hash_table(other) + assert first.hash_table(table) == expected + + +# --------------------------------------------------------------------------- +# The analysis runs once +# --------------------------------------------------------------------------- + +@pytest.fixture +def analysis_calls(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]: + counts = {"schema_nodes": 0, "phase1_json": 0, "phase2_json": 0} + for name in counts: + original = getattr(_format2, name) + + def counting(*args: Any, _name: str = name, _original: Any = original, **kwargs: Any) -> Any: + counts[_name] += 1 + return _original(*args, **kwargs) + + monkeypatch.setattr(_format2, name, counting) + return counts + + +def test_one_shot_calls_analyse_once(analysis_calls: dict[str, int]) -> None: + table = _simple_table() + batch = table.to_batches()[0] + for call in (lambda: ArrowDigester.hash_table(table, include_metadata=True), + lambda: ArrowDigester.hash_record_batch(batch, include_metadata=True), + lambda: ArrowDigester.hash_schema(table.schema, include_metadata=True)): + for name in analysis_calls: + analysis_calls[name] = 0 + call() + assert analysis_calls == {"schema_nodes": 1, "phase1_json": 1, "phase2_json": 1} + + +def test_prepared_hashes_of_an_identical_schema_do_not_analyse(analysis_calls: dict[str, int]) -> None: + table = _simple_table() + prepared = ArrowDigester.prepare(table.schema, include_metadata=True) + rebuilt = pa.table(table.to_pydict(), schema=_simple_table().schema) # equal schema, other objects + for name in analysis_calls: + analysis_calls[name] = 0 + prepared.hash_table(table) + prepared.hash_table(rebuilt) + prepared.hash_record_batch(rebuilt.to_batches()[0]) + d = prepared.digester() + d.update(rebuilt.to_batches()[0]) + d.finalize() + streaming = ArrowDigester(table.schema, include_metadata=True) + for name in analysis_calls: + analysis_calls[name] = 0 + streaming.update(rebuilt.to_batches()[0]) + assert analysis_calls == {"schema_nodes": 0, "phase1_json": 0, "phase2_json": 0} + + +# --------------------------------------------------------------------------- +# The same-schema rule +# --------------------------------------------------------------------------- + +def _field(name: str, t: pa.DataType, nullable: bool = True, metadata: dict[str, str] | None = None) -> pa.Field: + return pa.field(name, t, nullable=nullable, metadata=metadata) + + +def _tensor(dim_names: list[str] | None) -> pa.DataType: + return pa.fixed_shape_tensor(pa.int8(), [1, 2], dim_names=dim_names) + + +# (id, prepared schema, input schema, options under which the input differs in the hash) +# "all" = every option combination; "meta" = include_metadata=True only; "cols" = ordered_columns +# only; "structs" = ordered_struct_fields only. +_MD = {"k": "1"} +_MD2 = {"k": "2"} +_DIFFERENCES: list[tuple[str, pa.Schema, pa.Schema, str]] = [ + ("column-name", pa.schema([_field("a", pa.int8())]), pa.schema([_field("b", pa.int8())]), "all"), + ("column-type", pa.schema([_field("a", pa.int8())]), pa.schema([_field("a", pa.int16())]), "all"), + ("column-nullability", pa.schema([_field("a", pa.int8())]), pa.schema([_field("a", pa.int8(), False)]), "all"), + ("extra-column", pa.schema([_field("a", pa.int8())]), + pa.schema([_field("a", pa.int8()), _field("b", pa.int8())]), "all"), + ("struct-child-nullability", pa.schema([_field("s", pa.struct([_field("x", pa.int8())]))]), + pa.schema([_field("s", pa.struct([_field("x", pa.int8(), False)]))]), "all"), + ("list-element-nullability", pa.schema([_field("l", pa.list_(_field("item", pa.int8())))]), + pa.schema([_field("l", pa.list_(_field("item", pa.int8(), False)))]), "all"), + ("list-vs-fixed-size-list", pa.schema([_field("l", pa.list_(pa.int8()))]), + pa.schema([_field("l", pa.list_(pa.int8(), 2))]), "all"), + ("fixed-size-list-size", pa.schema([_field("l", pa.list_(pa.int8(), 2))]), + pa.schema([_field("l", pa.list_(pa.int8(), 3))]), "all"), + ("timestamp-zone", pa.schema([_field("t", pa.timestamp("us"))]), + pa.schema([_field("t", pa.timestamp("us", "UTC"))]), "all"), + ("decimal-scale", pa.schema([_field("d", pa.decimal128(10, 2))]), + pa.schema([_field("d", pa.decimal128(10, 3))]), "all"), + ("column-order", pa.schema([_field("a", pa.int8()), _field("b", pa.int8())]), + pa.schema([_field("b", pa.int8()), _field("a", pa.int8())]), "cols"), + ("struct-field-order", pa.schema([_field("s", pa.struct([_field("x", pa.int8()), _field("y", pa.int8())]))]), + pa.schema([_field("s", pa.struct([_field("y", pa.int8()), _field("x", pa.int8())]))]), "structs"), + ("column-metadata", pa.schema([_field("a", pa.int8(), metadata=_MD)]), + pa.schema([_field("a", pa.int8(), metadata=_MD2)]), "meta"), + ("column-metadata-removed", pa.schema([_field("a", pa.int8(), metadata=_MD)]), + pa.schema([_field("a", pa.int8())]), "meta"), + ("schema-metadata", pa.schema([_field("a", pa.int8())], metadata=_MD), + pa.schema([_field("a", pa.int8())], metadata=_MD2), "meta"), + ("struct-child-metadata", pa.schema([_field("s", pa.struct([_field("x", pa.int8(), metadata=_MD)]))]), + pa.schema([_field("s", pa.struct([_field("x", pa.int8(), metadata=_MD2)]))]), "meta"), + ("list-element-metadata", pa.schema([_field("l", pa.list_(_field("item", pa.int8(), metadata=_MD)))]), + pa.schema([_field("l", pa.list_(_field("item", pa.int8(), metadata=_MD2)))]), "meta"), + ("fixed-size-list-element-metadata", pa.schema([_field("l", pa.list_(_field("item", pa.int8(), metadata=_MD), 2))]), + pa.schema([_field("l", pa.list_(_field("item", pa.int8(), metadata=_MD2), 2))]), "meta"), + # pyarrow's Schema.equals(check_metadata=True) ignores this difference (pyarrow 26). + ("dictionary-value-struct-child-metadata", + pa.schema([_field("d", pa.dictionary(pa.int8(), pa.struct([_field("x", pa.int8(), metadata=_MD)])))]), + pa.schema([_field("d", pa.dictionary(pa.int8(), pa.struct([_field("x", pa.int8(), metadata=_MD2)])))]), "meta"), + # pyarrow's default ExtensionType equality ignores the serialized parameter. + ("python-extension-parameter", pa.schema([_field("u", ra.UnitType("m"))]), + pa.schema([_field("u", ra.UnitType("s"))]), "meta"), + ("canonical-extension-parameter", pa.schema([_field("t", _tensor(None))]), + pa.schema([_field("t", _tensor(["r", "c"]))]), "meta"), + ("extension-vs-storage", pa.schema([_field("u", ra.UnitType("m"))]), + pa.schema([_field("u", pa.int64())]), "meta"), + ("extension-storage-child-metadata", + pa.schema([_field("t", _tensor(None))]), + pa.schema([_field("t", pa.list_(_field("item", pa.int8(), metadata=_MD), 2))]), "meta"), +] + + +def _differs_under(kind: str, options: dict[str, bool]) -> bool: + return { + "all": True, + "meta": options["include_metadata"], + "cols": options["ordered_columns"], + "structs": options["ordered_struct_fields"], + }[kind] + + +def _empty(schema: pa.Schema) -> pa.Table: + rng = random.Random(0) + return pa.Table.from_arrays([ra.random_array(f.type, 0, f.nullable, rng) for f in schema], schema=schema) + + +def _one_row(schema: pa.Schema, seed: int = 0) -> pa.Table: + rng = random.Random(seed) + columns = [ra.random_array(f.type, 1, f.nullable, rng) for f in schema] + return pa.Table.from_arrays(columns, schema=schema) + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +@pytest.mark.parametrize(("case", "prepared_schema", "input_schema", "kind"), _DIFFERENCES, + ids=[d[0] for d in _DIFFERENCES]) +def test_same_schema_rule(case: str, prepared_schema: pa.Schema, input_schema: pa.Schema, kind: str, + options: dict[str, bool]) -> None: + """Refused exactly when the input's schema hashes differently under the prepared options; + otherwise accepted, hashing byte-identically to the one-shot path.""" + prepared = ArrowDigester.prepare(prepared_schema, **options) + differs = ArrowDigester.hash_schema(input_schema, **options) != prepared.schema_digest + assert differs == _differs_under(kind, options), case + for table in (_empty(input_schema), _one_row(input_schema)): + batch = pa.RecordBatch.from_arrays([c.combine_chunks() for c in table.columns], schema=input_schema) + if differs: + with pytest.raises(ValueError, match=_TABLE_MISMATCH): + prepared.hash_table(table) + with pytest.raises(ValueError, match=_BATCH_MISMATCH): + prepared.hash_record_batch(batch) + d = prepared.digester() + with pytest.raises(ValueError, match=_STREAM_MISMATCH): + d.update(batch) + assert d.finalize() == prepared.hash_table(_empty(prepared_schema)) # left unchanged + else: + _assert_same_as_one_shot(table, options, prepared) + + +# Hash-equivalent but not identical schemas: accepted (today's streaming rule) under every option set +# where they hash alike, through the analysis fallback. +_EQUIVALENT: list[tuple[str, pa.Schema, pa.Schema]] = [ + ("string-vs-large-string", pa.schema([_field("a", pa.string())]), pa.schema([_field("a", pa.large_string())])), + ("binary-vs-large-binary", pa.schema([_field("a", pa.binary())]), pa.schema([_field("a", pa.large_binary())])), + ("list-vs-large-list", pa.schema([_field("l", pa.list_(pa.int8()))]), pa.schema([_field("l", pa.large_list(pa.int8()))])), + ("dictionary-vs-plain", pa.schema([_field("a", pa.string())]), + pa.schema([_field("a", pa.dictionary(pa.int32(), pa.string()))])), + ("dictionary-index-type", pa.schema([_field("a", pa.dictionary(pa.int8(), pa.string()))]), + pa.schema([_field("a", pa.dictionary(pa.int32(), pa.string()))])), + ("list-element-name", pa.schema([_field("l", pa.list_(_field("item", pa.int8())))]), + pa.schema([_field("l", pa.list_(_field("element", pa.int8())))])), + ("metadata-key-order", pa.schema([_field("a", pa.int8(), metadata={"x": "1", "y": "2"})], metadata={"p": "1", "q": "2"}), + pa.schema([_field("a", pa.int8(), metadata={"y": "2", "x": "1"})], metadata={"q": "2", "p": "1"})), +] + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +@pytest.mark.parametrize(("case", "prepared_schema", "input_schema"), _EQUIVALENT, ids=[e[0] for e in _EQUIVALENT]) +def test_hash_equivalent_schemas_are_accepted(case: str, prepared_schema: pa.Schema, input_schema: pa.Schema, + options: dict[str, bool]) -> None: + assert ArrowDigester.hash_schema(input_schema, **options) == ArrowDigester.hash_schema(prepared_schema, **options) + prepared = ArrowDigester.prepare(prepared_schema, **options) + for table in (_empty(input_schema), _one_row(input_schema)): + _assert_same_as_one_shot(table, options, prepared) + + +@pytest.mark.parametrize("options", OPTION_SETS, ids=_OPTION_IDS) +def test_seeded_schema_pairs_accepted_iff_schema_digests_equal(options: dict[str, bool]) -> None: + """Random pairs, many of them hash-equivalent after dropping metadata or reordering.""" + rng = random.Random(1177) + accepted = refused = 0 + for _ in range(150): + a = ra.random_schema(rng) + b = rng.choice([ + ra.random_schema(rng), + a.remove_metadata(), + pa.schema(list(reversed(list(a))), metadata=a.metadata), + pa.schema([f.remove_metadata() if rng.random() < 0.5 else f for f in a], metadata=a.metadata), + a, + ]) + prepared = ArrowDigester.prepare(a, **options) + table = ra.random_table(rng, b) + if ArrowDigester.hash_schema(b, **options) == prepared.schema_digest: + accepted += 1 + assert prepared.hash_table(table) == ArrowDigester.hash_table(table, **options) + else: + refused += 1 + with pytest.raises(ValueError, match=_TABLE_MISMATCH): + prepared.hash_table(table) + assert accepted > 20 and refused > 20 + + +def test_ipc_fast_path_is_deterministic_for_equal_schemas() -> None: + """The fast path compares Arrow IPC schema bytes; equal schemas built in different ways + (pandas, a JSON-like round trip, IPC) serialize identically or fall back to the full check.""" + table = _simple_table() + schema = table.schema + assert schema.serialize().to_pybytes() == schema.serialize().to_pybytes() + via_ipc = pa.ipc.read_schema(pa.py_buffer(schema.serialize().to_pybytes())) + rebuilt = pa.schema([pa.field(f.name, f.type, f.nullable, f.metadata) for f in schema], metadata=schema.metadata) + for options in OPTION_SETS: + prepared = ArrowDigester.prepare(schema, **options) + for other in (via_ipc, rebuilt): + assert other.serialize().to_pybytes() == schema.serialize().to_pybytes() + _assert_same_as_one_shot(pa.Table.from_batches(table.to_batches(), schema=other), options, prepared) + + +# --------------------------------------------------------------------------- +# Differential: golden vectors, seeded random inputs +# --------------------------------------------------------------------------- + +_GOLDEN = Path(__file__).parent / "golden" / "format2_v0.5.json" + + +@cache +def _golden_record_vectors() -> list[dict[str, Any]]: + vectors = json.loads(_GOLDEN.read_text(encoding="utf-8"))["vectors"] + return [v for v in vectors if v["method"] in ("hash_schema", "hash_record_batch", "hash_stream")] + + +def _decode(vector: dict[str, Any]) -> tuple[pa.Schema, list[pa.RecordBatch]]: + reader = pa.ipc.open_stream(io.BytesIO(base64.b64decode(vector["ipc_b64"]))) + return reader.schema, list(reader) + + +def _outcome(call: Any, *args: Any, **kwargs: Any) -> str: + try: + return call(*args, **kwargs).hex() + except (ValueError, TypeError, NotImplementedError) as e: + return f"raises {type(e).__name__}: {e}" + + +def _prepared_schema_digest(schema: pa.Schema, **options: bool) -> bytes: + return ArrowDigester.prepare(schema, **options).schema_digest + + +def _prepared_hash_table(table: pa.Table, **options: bool) -> bytes: + return ArrowDigester.prepare(table.schema, **options).hash_table(table) + + +def test_golden_vectors_prepared_equal_one_shot_for_every_option_set() -> None: + """Every record/schema vector of the Rust golden file, under all 8 option combinations: the + prepared path equals the one-shot path (and the Rust hash for the vector's own options).""" + vectors = _golden_record_vectors() + assert len(vectors) > 300 + for vector in vectors: + schema, batches = _decode(vector) + own = {"include_metadata": vector["include_metadata"], "ordered_columns": vector["ordered_columns"], + "ordered_struct_fields": vector["ordered_struct_fields"]} + for options in OPTION_SETS: + prepared_or_error = _outcome(_prepared_schema_digest, schema, **options) + assert prepared_or_error == _outcome(ArrowDigester.hash_schema, schema, **options) + if prepared_or_error.startswith("raises"): + continue + prepared = ArrowDigester.prepare(schema, **options) + if vector["method"] == "hash_schema": + got = prepared.schema_digest.hex() + else: + table = pa.Table.from_batches(batches, schema=schema) + got = _outcome(prepared.hash_table, table) + assert got == _outcome(ArrowDigester.hash_table, table, **options), vector["id"] + assert got == _outcome(_prepared_stream, prepared, batches), vector["id"] + assert got == _outcome(_one_shot_stream, schema, batches, **options), vector["id"] + if len(batches) == 1: + assert got == _outcome(prepared.hash_record_batch, batches[0]), vector["id"] + if options == own: + assert got == vector["expected_hash"], vector["id"] + + +@pytest.mark.parametrize("chunk", range(8)) +def test_seeded_random_inputs_prepared_equal_one_shot(chunk: int) -> None: + for seed in range(chunk * 40, chunk * 40 + 40): + rng = random.Random(seed) + table = ra.random_table(rng) + for options in OPTION_SETS: + one_shot = _outcome(ArrowDigester.hash_table, table, **options) + if one_shot.startswith("raises"): + assert _outcome(_prepared_hash_table, table, **options) == one_shot + continue + _assert_same_as_one_shot(table, options) From 4eded2db563c7d8574953035379bbadc131c945b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 17:27:39 -0800 Subject: [PATCH 2/4] test: release differential against starfix 0.5.0 and a prepared-schema benchmark (ITL-1177) scripts/release_differential.py hashes seeded random tables under all option combinations, one-shot and prepared, and requires byte-identical results to the released starfix==0.5.0 (run in CI like the legacy differential). scripts/bench_prepared.py times one-row hashes. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 19 ++++ scripts/bench_prepared.py | 94 ++++++++++++++++++ scripts/release_differential.py | 162 ++++++++++++++++++++++++++++++++ 3 files changed, 275 insertions(+) create mode 100644 scripts/bench_prepared.py create mode 100644 scripts/release_differential.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d381b4..bea73a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,6 +94,25 @@ jobs: - name: Compare starfix.legacy.v0_4 with starfix==0.4.0 from PyPI run: uv run --python 3.12 python scripts/legacy_differential.py + release-differential: + # Every hash (one-shot and through a prepared schema) must stay byte-identical to the + # released starfix-python 0.5.0 (PyPI), under every option combination. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Install uv + uses: astral-sh/setup-uv@e58605a9b6da7c637471fab8847a5e5a6b8df081 # v5 + + - name: Set up Python 3.12 + run: uv python install 3.12 + + - name: Install dependencies + run: uv sync --dev --python 3.12 + + - name: Compare this checkout with starfix==0.5.0 from PyPI + run: uv run --python 3.12 python scripts/release_differential.py + license-check: runs-on: ubuntu-latest steps: diff --git a/scripts/bench_prepared.py b/scripts/bench_prepared.py new file mode 100644 index 0000000..989de2c --- /dev/null +++ b/scripts/bench_prepared.py @@ -0,0 +1,94 @@ +"""Time per hash of small tables: one-shot calls against a prepared schema (ITL-1177). + +Usage (from the repository root): + + uv run python scripts/bench_prepared.py [--rows 1] [--repeat 7] + +Hashes tables of a few-column schema (ints, strings, a list, a struct, field and schema +metadata), each built anew as a caller hashing row by row would (so every table carries a new +``pa.Schema`` object). Prints the best-of-``--repeat`` mean time per call in microseconds. +Against a checkout without ``ArrowDigester.prepare`` (0.5.0 and earlier) only the one-shot rows +are printed, so the same script measures before and after: for example + + git worktree add /tmp/starfix-main origin/main + PYTHONPATH=/tmp/starfix-main/src uv run python scripts/bench_prepared.py +""" + +from __future__ import annotations + +import argparse +import platform +import timeit + +import pyarrow as pa + +import starfix +from starfix import ArrowDigester + +SCHEMA = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("name", pa.large_string()), + pa.field("score", pa.float64(), metadata={"unit": "points"}), + pa.field("tags", pa.list_(pa.large_string())), + pa.field("pos", pa.struct([pa.field("x", pa.float32()), pa.field("y", pa.float32())])), + pa.field("path", pa.large_string(), metadata={"semantic": "path"}), + ], + metadata={"producer": "bench"}, +) + + +def _rows(n: int) -> list[dict]: + return [ + {"id": i, "name": f"n{i}", "score": i / 3, "tags": ["a", "b"][: i % 3], "pos": {"x": 1.0, "y": 2.0}, + "path": f"/data/{i}.bin"} + for i in range(n) + ] + + +def _bench_options(bench, tables, batches, options: dict[str, bool], has_prepare: bool) -> None: + bench("one-shot ArrowDigester.hash_table", lambda i: ArrowDigester.hash_table(tables[i], **options)) + bench("one-shot ArrowDigester.hash_record_batch", lambda i: ArrowDigester.hash_record_batch(batches[i], **options)) + if not has_prepare: + return + prepared = ArrowDigester.prepare(SCHEMA, **options) + bench("prepared.hash_table", lambda i: prepared.hash_table(tables[i])) + bench("prepared.hash_record_batch", lambda i: prepared.hash_record_batch(batches[i])) + bench(" of which: ArrowDigester.prepare (once per schema)", lambda i: ArrowDigester.prepare(SCHEMA, **options)) + analysis = prepared._analysis + bench(" of which: same-schema check (identical schema)", lambda i: analysis.matches(tables[i].schema)) + string_schema = pa.schema([f.with_type(pa.string()) if f.type == pa.large_string() else f for f in SCHEMA], + metadata=SCHEMA.metadata) + bench(" same-schema check, hash-equivalent (string) schema", lambda i: analysis.matches(string_schema)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--rows", type=int, default=1) + parser.add_argument("--repeat", type=int, default=7) + parser.add_argument("--number", type=int, default=2000) + args = parser.parse_args() + rows = _rows(args.rows) + tables = [pa.Table.from_pylist(rows, schema=SCHEMA) for _ in range(64)] # distinct Schema objects + batches = [t.to_batches()[0] for t in tables] + has_prepare = hasattr(ArrowDigester, "prepare") + print(f"starfix from {starfix.__file__}; pyarrow {pa.__version__}; Python {platform.python_version()}; " + f"{platform.machine()}; {args.rows} row(s), {len(SCHEMA)} columns") + + def bench(label: str, call) -> None: + it = iter(range(1 << 62)) + + def step() -> None: + call(next(it) % len(tables)) + + best = min(timeit.repeat(step, number=args.number, repeat=args.repeat)) / args.number + print(f" {label:<52} {best * 1e6:8.1f} us") + + for include_metadata in (False, True): + print(f"include_metadata={include_metadata}") + _bench_options(bench, tables, batches, {"include_metadata": include_metadata}, has_prepare) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_differential.py b/scripts/release_differential.py new file mode 100644 index 0000000..ecd89c8 --- /dev/null +++ b/scripts/release_differential.py @@ -0,0 +1,162 @@ +"""Differential test: this checkout (one-shot and prepared) against the released starfix 0.5.0. + +Usage (from the repository root): + + uv run python scripts/release_differential.py [--seeds N] + +The driver hashes seeded random tables (``tests/random_arrow.py``: nested types, dictionaries, +fixed-size lists, extension types, metadata at every depth, empty and multi-batch tables) under +all 8 option combinations, through the one-shot calls and through ``ArrowDigester.prepare`` +(``hash_table``, ``hash_record_batch``, ``digester()`` and ``schema_digest``), and requires the +prepared results to equal the one-shot ones. It then runs this same script with +``--impl release`` in an isolated environment containing the released ``starfix==0.5.0`` from +PyPI and the same pyarrow version, and requires every one-shot result to be byte-identical +(exceptions must match by type). Both processes build the inputs themselves from the seeds. +Exit status 0 means identical. +""" + +from __future__ import annotations + +import argparse +import itertools +import json +import random +import subprocess +import sys +import tempfile +from importlib import metadata +from pathlib import Path + +import pyarrow as pa + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +import random_arrow + +RELEASE = "0.5.0" +OPTION_SETS = [ + {"include_metadata": m, "ordered_columns": c, "ordered_struct_fields": s} + for m, c, s in itertools.product([False, True], repeat=3) +] + + +def _outcome(call, *args, **kwargs) -> str: + try: + return call(*args, **kwargs).hex() + except Exception as e: # noqa: BLE001 - exceptions must match by type + return f"raises {type(e).__name__}" + + +def _stream(digester, batches) -> bytes: + for batch in batches: + digester.update(batch) + return digester.finalize() + + +def _one_shot_stream(D, schema, batches, **options) -> bytes: + return _stream(D(schema, **options), batches) + + +def _prepared_schema(D, schema, **options) -> bytes: + return D.prepare(schema, **options).schema_digest + + +def _prepared_table(D, table, **options) -> bytes: + return D.prepare(table.schema, **options).hash_table(table) + + +def _prepared_batch(D, schema, batch, **options) -> bytes: + return D.prepare(schema, **options).hash_record_batch(batch) + + +def _prepared_stream(D, schema, batches, **options) -> bytes: + return _stream(D.prepare(schema, **options).digester(), batches) + + +def _special_tables(): + """Inputs that raise: schemas whose data does not hash, rejected schemas, nulls in + non-nullable fields.""" + yield "map", pa.table({"m": pa.array([[("k", 1)]], pa.map_(pa.string(), pa.int8()))}) + yield "null", pa.table({"n": pa.nulls(2)}) + yield "union", pa.table({"u": pa.UnionArray.from_sparse(pa.array([0], pa.int8()), [pa.array([1])])}) + yield "duplicate", pa.Table.from_arrays([pa.array([1]), pa.array([2])], names=["a", "a"]) + nulls = pa.StructArray.from_arrays([pa.array([1, None])], fields=[pa.field("x", pa.int64(), nullable=False)]) + yield "non_nullable_null", pa.Table.from_arrays([nulls], names=["s"]) + meta = pa.schema([pa.field("a", pa.int8(), metadata={"ARROW:extension:name": "x"})]) + yield "extension_name_without_type", pa.table({"a": pa.array([1], pa.int8())}, schema=meta) + + +def _tables(seeds: int): + yield from _special_tables() + for seed in range(seeds): + yield seed, random_arrow.random_table(random.Random(seed)) + + +def _run(impl: str, seeds: int) -> tuple[dict[str, str], list[str]]: + from starfix import ArrowDigester as D + + if impl == "release" and metadata.version("starfix") != RELEASE: + raise SystemExit(f"not the starfix {RELEASE} release: {metadata.version('starfix')}") + results: dict[str, str] = {} + prepared_differ: list[str] = [] + for seed, table in _tables(seeds): + schema, batches = table.schema, table.to_batches() + for options in OPTION_SETS: + tag = f"{seed}:m{int(options['include_metadata'])}c{int(options['ordered_columns'])}" \ + f"s{int(options['ordered_struct_fields'])}" + one_shot = { + "schema": _outcome(D.hash_schema, schema, **options), + "table": _outcome(D.hash_table, table, **options), + "stream": _outcome(_one_shot_stream, D, schema, batches, **options), + } + if batches: + one_shot["batch"] = _outcome(D.hash_record_batch, batches[0], **options) + for name, value in one_shot.items(): + results[f"{tag}:{name}"] = value + if impl != "current": + continue + prepared = { + "schema": _outcome(_prepared_schema, D, schema, **options), + "table": _outcome(_prepared_table, D, table, **options), + "stream": _outcome(_prepared_stream, D, schema, batches, **options), + } + if batches: + prepared["batch"] = _outcome(_prepared_batch, D, schema, batches[0], **options) + prepared_differ += [f"{tag}:{k}" for k in one_shot if prepared[k] != one_shot[k]] + return results, prepared_differ + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--impl", choices=["driver", "current", "release"], default="driver") + parser.add_argument("--seeds", type=int, default=1000) + parser.add_argument("--out") + args = parser.parse_args() + if args.impl != "driver": + results, _ = _run(args.impl, args.seeds) + Path(args.out).write_text(json.dumps(results)) + return 0 + + current, prepared_differ = _run("current", args.seeds) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "release.json" + python = f"{sys.version_info.major}.{sys.version_info.minor}" + subprocess.run( + ["uv", "run", "--no-project", "--python", python, "--with", f"starfix=={RELEASE}", + "--with", f"pyarrow=={pa.__version__}", "python", str(Path(__file__).resolve()), + "--impl", "release", "--seeds", str(args.seeds), "--out", str(out)], + check=True, cwd=tmp, + ) + release = json.loads(out.read_text()) + differ = sorted(k for k in current if current[k] != release.get(k)) + sorted(set(release) - set(current)) + raised = sum(1 for v in current.values() if v.startswith("raises")) + print(f"{len(current)} one-shot results compared with starfix {RELEASE} ({raised} raise the same exception " + f"in both); {len(differ)} differ; {len(prepared_differ)} prepared results differ from one-shot") + for key in differ[:20]: + print(f" DIFFER {key}: current={current.get(key)} release={release.get(key)}") + for key in prepared_differ[:20]: + print(f" PREPARED DIFFERS {key}") + return 1 if differ or prepared_differ else 0 + + +if __name__ == "__main__": + sys.exit(main()) From ab8abff8d272dbfae82521fd494b41b2678321cd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 17:27:39 -0800 Subject: [PATCH 3/4] docs: prepared schemas in README and CHANGELOG (ITL-1177) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 31 +++++ README.md | 50 +++++++- .../plans/2026-11-16-prepared-schema.md | 107 ++++++++++++++++++ 3 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 docs/metamorphic/plans/2026-11-16-prepared-schema.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bf6237..9990966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # Changelog +## 0.6.0 (unreleased) + +No hash changes: every hash is byte-identical to 0.5.0 (hash format 2), for every input and +option combination. + +### Added — prepared schemas (ITL-1177) + +- `ArrowDigester.prepare(schema, *, include_metadata=False, ordered_columns=False, + ordered_struct_fields=False)` analyses a schema once and returns a `PreparedSchema` + (exported from `starfix`), with `schema_digest` (equal to `hash_schema`), + `hash_record_batch(batch)`, `hash_table(table)` and `digester()` (a fresh streaming + `ArrowDigester`). Its hashes equal the one-shot calls; hashing a one-row table of a + six-column schema takes about 35 µs instead of about 150 µs (0.5.0, without + `include_metadata`; about 530 µs with it). See the README section "Hashing many inputs of + one schema" and `scripts/bench_prepared.py`. +- A `PreparedSchema` is immutable and owned by the caller; starfix keeps no cache. A table or + batch is accepted exactly when its schema hashes like the prepared one under the prepared + options (the rule streamed batches already follow); otherwise `ValueError`. Identical + schemas are recognized by comparing their Arrow IPC serialization (about 1.5 µs). +- `scripts/release_differential.py` (and a CI job): this checkout, one-shot and prepared, + against the released `starfix==0.5.0`, under all option combinations. + +### Changed — performance + +- One-shot calls (`hash_table`, `hash_record_batch`, `ArrowDigester(...)`) analyse the + schema once instead of three times (plus once per streamed batch): a one-row `hash_table` + takes about 85 µs instead of about 150 µs (about 175 µs instead of about 530 µs with + `include_metadata`). +- `ArrowDigester.update` checks a batch whose schema is identical to the digester's in about + 1.5 µs instead of re-analysing it. + ## 0.5.0 (2026-11-11) ### Changed — hash format 2 (every hash changes) diff --git a/README.md b/README.md index 0b0e45c..cf0ddd2 100644 --- a/README.md +++ b/README.md @@ -55,11 +55,48 @@ ArrowDigester.hash_record_batch(table.to_batches()[0]) ArrowDigester.hash_array(pa.array([1, 2, 3], pa.int32())) ``` +### Hashing many inputs of one schema + +Every call above first analyses the schema (walks its nodes, builds its canonical text and +digest). To hash many small tables or batches of one schema, analyse it once: + +```python +prepared = ArrowDigester.prepare(schema, include_metadata=False) # options as below + +for row_table in tables_of_this_schema: + key = prepared.hash_table(row_table) # == ArrowDigester.hash_table(row_table) +prepared.hash_record_batch(batch) # == ArrowDigester.hash_record_batch(batch) +prepared.schema_digest # == ArrowDigester.hash_schema(schema) +digester = prepared.digester() # a fresh streaming ArrowDigester +``` + +- Every hash equals the one-shot call with the same options, byte for byte. +- A `PreparedSchema` is immutable and belongs to the caller: starfix keeps no cache or other + state between calls. It can be shared between threads and pickled (it is prepared again when + loaded). +- **Same-schema rule.** A table or batch is accepted exactly when its schema hashes like the + prepared one under the prepared options (`hash_schema(its_schema, **options) == + prepared.schema_digest`): the same column names, logical types and nullability at every + depth, the same column order with `ordered_columns`, the same struct field order with + `ordered_struct_fields`, and with `include_metadata` the same field metadata at every node, + schema metadata and extension types. It is the rule `update` applies to streamed batches. + Anything else raises `ValueError` ("Table schema does not match the prepared schema" / + "Record batch schema does not match the prepared schema"); nothing is ever hashed with + another schema's analysis. +- The check costs about 1.5 µs when the input's schema is identical to the prepared one (equal + Arrow IPC serialization, compared as bytes). Any other accepted schema (for example `string` + data for a prepared `large_string` column, or metadata that differs while `include_metadata` + is off) is checked by computing its canonical schema text, as expensive as a one-shot call's + analysis. +- `prepare` raises what `hash_schema` raises. A schema whose data cannot be hashed (see + [Rejected inputs](#rejected-inputs)) still prepares, for its `schema_digest`; hashing data + with it raises `ValueError`. + ## Options Keyword-only, all `False` by default, the same names as the Rust `HasherConfig` fields, accepted by -`ArrowDigester(...)`, `hash_schema`, `hash_record_batch`, `hash_table` and `hash_array` -(`hash_array` has no `include_metadata`). +`ArrowDigester(...)`, `ArrowDigester.prepare`, `hash_schema`, `hash_record_batch`, `hash_table` and +`hash_array` (`hash_array` has no `include_metadata`). | Option | Effect | |---|---| @@ -132,7 +169,8 @@ rebuilds the type from them, so starfix-python hashes what pyarrow can observe: - a non-nullable field holding a null that no enclosing struct null explains; - with `include_metadata`: a duplicate metadata key on a field or the schema, or explicit `ARROW:extension:*` keys that conflict with the field's registered extension type; -- streaming: a batch whose schema differs from the digester's. Physically different but +- streaming, and hashing through a prepared schema: a batch or table whose schema differs from + the digester's or the prepared one (see the same-schema rule above). Physically different but equivalent types (`Utf8`/`LargeUtf8`, `Binary`/`LargeBinary`, `List`/`LargeList`, dictionary and plain values) are accepted; reordered columns or struct fields are accepted unless the matching order option is on. @@ -158,8 +196,10 @@ are materialized (decoded with `take`) before they are traversed. A fixed-size l without null slots is hashed from one zero-copy slice of its child; with null slots, its items are selected in chunks of about one million items. Fixed-width, Boolean and list columns hash at tens of millions of rows per second. String and binary columns hash at about 3 to 5 million rows per second, because each value is -visited in Python. Each `update` call also costs a few microseconds per column, to check the -batch's schema. starfix-python is a pure-Python implementation; the Rust crate is faster on +visited in Python. Each call also analyses the schema, which dominates for small inputs (tens to +hundreds of microseconds, more with `include_metadata`); `ArrowDigester.prepare` does it once +for many inputs. A streamed batch whose schema is identical to the digester's is checked in +about 1.5 µs; `scripts/bench_prepared.py` measures both paths. starfix-python is a pure-Python implementation; the Rust crate is faster on string-heavy data. ## Hash format diff --git a/docs/metamorphic/plans/2026-11-16-prepared-schema.md b/docs/metamorphic/plans/2026-11-16-prepared-schema.md new file mode 100644 index 0000000..ca09fca --- /dev/null +++ b/docs/metamorphic/plans/2026-11-16-prepared-schema.md @@ -0,0 +1,107 @@ +# Prepared Schema Implementation Plan (ITL-1177) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let a caller analyse an Arrow schema once (`ArrowDigester.prepare`) and hash many tables or record +batches of that schema without redoing the analysis, with every digest byte-identical to the one-shot calls. + +**Architecture:** `_format2.analyse_schema` runs the schema analysis once and returns an immutable +`SchemaAnalysis` (node list, canonical schema key, schema digest, entry specs, the data-support verdict, and +the schema's Arrow IPC bytes for the same-schema check). `Digester` is built from an analysis; the one-shot +entry points build one analysis per call and never re-check their own input. The public +`starfix.PreparedSchema` (in `arrow_digester.py`) wraps one analysis and its schema, owned by the caller. +No module-level, global or cached state is added. + +**Tech Stack:** Python >= 3.10, pyarrow >= 19, pytest, uv. + +--- + +## Interface + +```python +ArrowDigester.prepare(schema, *, include_metadata=False, ordered_columns=False, + ordered_struct_fields=False) -> PreparedSchema + +class PreparedSchema: # built only by prepare; immutable; pickles by re-preparing + schema: pa.Schema # read-only properties + include_metadata: bool + ordered_columns: bool + ordered_struct_fields: bool + schema_digest: bytes # == ArrowDigester.hash_schema(schema, same options) + def hash_record_batch(self, record_batch) -> bytes + def hash_table(self, table) -> bytes + def digester(self) -> ArrowDigester +``` + +* `prepare` raises exactly what `hash_schema` raises. Types whose schema hashes but whose data does not + (Null, Map, Interval, view types) do not make `prepare` fail; `hash_record_batch`, `hash_table` and + `digester()` raise the existing `ValueError` for them. +* Same-schema rule: accepted iff the input schema's canonical schema key under the prepared options equals + the prepared one (Phase 1 text; plus Phase 2 text with `include_metadata`), i.e. iff + `hash_schema(input.schema, options) == prepared.schema_digest`. This is today's rule of + `ArrowDigester.update`. +* Check: (1) `input.schema.serialize()` bytes equal the prepared schema's IPC bytes -> identical schema -> + accept; (2) otherwise compute the input's key (today's analysis) and compare; mismatch -> `ValueError` + ("Record batch schema does not match the prepared schema" / "Table schema does not match the prepared + schema"; `digester().update` keeps "Record batch schema does not match ArrowDigester schema"). + `pa.Schema.equals(check_metadata=True)` is not used: it ignores nested metadata of dictionary value types + and the parameters of Python-defined extension types. + +## Files + +* Modify `src/starfix/_format2.py`: `SchemaAnalysis`, `analyse_schema`, `phase2_json(schema, nodes)`, + `_Entry(kind, nullable)`, `Digester.from_analysis`, unchecked ingestion for one-shot calls. +* Modify `src/starfix/arrow_digester.py`: `ArrowDigester.prepare`, `PreparedSchema`. +* Modify `src/starfix/__init__.py`: export `PreparedSchema`. +* Create `tests/random_arrow.py`: seeded random schemas and data (nested, dictionary, fixed-size list, + extension types, metadata, empty and sliced inputs). +* Create `tests/test_prepared_schema.py`: API, immutability, no module state, golden vectors x all option + combinations, the type matrix, accept/refuse rule, seeded differential, analysis-count tests. +* Create `scripts/release_differential.py`: this checkout (one-shot and prepared) against the released + `starfix==0.5.0` from PyPI, byte-identical. Add a CI job for it beside `legacy-differential`. +* Create `scripts/bench_prepared.py`: time per one-row hash, one-shot and prepared. +* Modify `README.md` (usage + same-schema rule), `CHANGELOG.md` (Unreleased, proposed 0.6.0). + +## Tasks + +### Task 1: failing tests for the public interface +- [ ] Write `tests/test_prepared_schema.py` tests: `prepare` returns a `PreparedSchema` whose properties + echo the options; `schema_digest == hash_schema` for all 8 option combinations; `PreparedSchema()` raises + `TypeError`; setting/deleting attributes raises `AttributeError`; pickle round trip hashes the same; + `hash_record_batch` / `hash_table` / `digester()` equal the one-shot results. +- [ ] Run `uv run pytest tests/test_prepared_schema.py -q`; expect failures (no `prepare`). + +### Task 2: analysis once (refactor, no behaviour change) +- [ ] `analyse_schema` computes nodes once, Phase 1 once, Phase 2 once (from the same nodes), the digest + from that key, sorted entry specs, the data-support verdict and the IPC bytes. +- [ ] `Digester.__init__` = `analyse_schema` + `from_analysis`; `update` checks via the analysis; + `hash_record_batch` / `hash_table` analyse once and ingest without re-checking (a table's batches carry + the table's schema). +- [ ] Test: counting wrappers on `schema_nodes`, `phase1_json`, `phase2_json` show one call each per + one-shot `hash_table` / `hash_record_batch`, and none for a prepared hash of an identical schema. +- [ ] Run the full suite: `uv run pytest tests/ -q` (golden vectors pin the bytes). + +### Task 3: `PreparedSchema` and `ArrowDigester.prepare` +- [ ] Implement in `arrow_digester.py`; export from `starfix`. Run Task 1 tests: pass. + +### Task 4: same-schema rule +- [ ] Refusal cases (each with every relevant option combination): column name, type, nullability (top + and nested), column order with `ordered_columns`, struct field order with `ordered_struct_fields`, field + metadata at top level, in a struct child, a list / fixed-size-list element, the struct inside a dictionary + value, schema metadata, a Python extension type's parameter, extension vs storage type, all with + `include_metadata`. Acceptance cases: string vs large_string, dictionary vs plain, reordered columns or + struct fields when unordered, metadata differences without `include_metadata`; accepted results equal the + one-shot hash of the input. +- [ ] Seeded pairs: accepted iff `hash_schema` digests are equal. + +### Task 5: differential proofs +- [ ] Golden vectors (`tests/golden/format2_v0.5.json`) x 8 option combinations: prepared == one-shot; + for the vector's own options == expected. +- [ ] Type matrix (`tests/test_format2_type_matrix.py::_hashes`) also through a prepared schema. +- [ ] Seeded random inputs (`tests/random_arrow.py`), all option combinations, batch / table / stream. +- [ ] `scripts/release_differential.py` against `starfix==0.5.0`; CI job. + +### Task 6: benchmark, docs, changelog, PR +- [ ] `scripts/bench_prepared.py`; numbers before (origin/main) and after. +- [ ] README section "Hashing many inputs of one schema"; CHANGELOG Unreleased. +- [ ] `uv run pytest tests/ -q`; commit (conventional commits); PR against `main`. From 576d23bce7c2fa94216b7cabe095e2823a5c74fb Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Mon, 16 Nov 2026 18:36:54 -0800 Subject: [PATCH 4/4] test: assert every prepared operation skips the schema analysis (ITL-1177) The prepared-schema analysis test reset its counters before checking them, so it passed even if every prepared hash re-analysed the schema. It now asserts zero analysis calls after each prepared operation (digester(), update, finalize, hash_table, hash_record_batch, schema_digest) without resetting, and checks that prepare itself analyses once, so the zero counts cannot pass because the counters observe nothing. The streaming digester's update check moves to its own test. Co-Authored-By: Claude Opus 5.5 --- tests/test_prepared_schema.py | 38 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/tests/test_prepared_schema.py b/tests/test_prepared_schema.py index 520c8f2..31fb1dd 100644 --- a/tests/test_prepared_schema.py +++ b/tests/test_prepared_schema.py @@ -252,23 +252,43 @@ def test_one_shot_calls_analyse_once(analysis_calls: dict[str, int]) -> None: assert analysis_calls == {"schema_nodes": 1, "phase1_json": 1, "phase2_json": 1} +_ANALYSED_ONCE = {"schema_nodes": 1, "phase1_json": 1, "phase2_json": 1} +_NOT_ANALYSED = {"schema_nodes": 0, "phase1_json": 0, "phase2_json": 0} + + def test_prepared_hashes_of_an_identical_schema_do_not_analyse(analysis_calls: dict[str, int]) -> None: table = _simple_table() - prepared = ArrowDigester.prepare(table.schema, include_metadata=True) rebuilt = pa.table(table.to_pydict(), schema=_simple_table().schema) # equal schema, other objects + batch = rebuilt.to_batches()[0] + prepared = ArrowDigester.prepare(table.schema, include_metadata=True) + assert analysis_calls == _ANALYSED_ONCE # the counters see the analysis (else the zeros below are vacuous) for name in analysis_calls: analysis_calls[name] = 0 - prepared.hash_table(table) - prepared.hash_table(rebuilt) - prepared.hash_record_batch(rebuilt.to_batches()[0]) - d = prepared.digester() - d.update(rebuilt.to_batches()[0]) - d.finalize() + # From here on the counts accumulate and are never reset: every prepared operation is checked. + digesters: list[ArrowDigester] = [] + for operation, call in ( + ("digester", lambda: digesters.append(prepared.digester())), + ("digester().update", lambda: digesters[0].update(batch)), + ("digester().finalize", lambda: digesters[0].finalize()), + ("hash_table", lambda: prepared.hash_table(table)), + ("hash_table of an equal schema", lambda: prepared.hash_table(rebuilt)), + ("hash_record_batch", lambda: prepared.hash_record_batch(batch)), + ("schema_digest", lambda: prepared.schema_digest), + ): + call() + assert analysis_calls == _NOT_ANALYSED, f"prepared {operation} analysed the schema" + + +def test_streaming_update_of_an_identical_schema_does_not_analyse(analysis_calls: dict[str, int]) -> None: + table = _simple_table() + batch = pa.table(table.to_pydict(), schema=_simple_table().schema).to_batches()[0] streaming = ArrowDigester(table.schema, include_metadata=True) + assert analysis_calls == _ANALYSED_ONCE # the constructor analyses, once for name in analysis_calls: analysis_calls[name] = 0 - streaming.update(rebuilt.to_batches()[0]) - assert analysis_calls == {"schema_nodes": 0, "phase1_json": 0, "phase2_json": 0} + streaming.update(batch) + streaming.update(batch) + assert analysis_calls == _NOT_ANALYSED # ---------------------------------------------------------------------------