From 272054fedb9aa5d6ed630ade099d47b3301b53ca Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:10:44 -0800 Subject: [PATCH 01/31] docs(specs): v0.3 identity of a user class, implementation spec (ITL-881) Defines the identity record (declared name, kind, order-independent field types), where it enters the hashes, the register_type API proposal, the rebinding and read-path rules, and the 25 golden entries that move. Refs ITL-881 Co-Authored-By: Claude Opus 5.5 --- .../2026-11-10-user-class-identity-v0.3.md | 304 ++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 superpowers/specs/2026-11-10-user-class-identity-v0.3.md diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md new file mode 100644 index 000000000..32d87c730 --- /dev/null +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -0,0 +1,304 @@ +# Identity of a user class in v0.3 (ITL-881) + +**Status:** implementation spec, waiting for approval. No product code changes yet. +**Issue:** ITL-881 (v0.3, a public release; tag target November 16, 2026). +**Builds on:** the findings in PR #288 (`superpowers/specs/2026-11-10-user-class-identity.md`, +experiments E01–E08). "W1"–"W15" are the workshop scenarios from §3.4 of that document. + +Each decision is marked **CONFIRMED** (the developer decided it, November 10, 2026) or +**PENDING** (the default this spec plans with; each pending item is a small, local change). + +| ID | Decision | Status | +|---|---|---| +| D-name | Nominal identity with a declared name; `__main__` classes default to the name only | CONFIRMED: developer decision 1 | +| D-fields | Field names and field types are part of the identity; field order is not | CONFIRMED: developer decision 2 | +| D-computed | Computed fields are not part of the identity in v0.3 | CONFIRMED as deferred: developer decision 2 defers it; v0.3 keeps today's stored fields | +| D-kind | Dataclass and pydantic are distinct kinds | CONFIRMED: developer decision 3 | +| D-rebind | Same identity re-registered: rebind silently | CONFIRMED: developer answer to S1 | +| D-raise | Stored shape disagreeing with registered shape raises, in v0.3 | CONFIRMED: developer answer to S2 | +| D-doc | A page documenting identity | CONFIRMED: developer answer to S4 | +| P1 | A field typed by another user class carries that class's full identity | PENDING: developer asked; default stated by the coordinator | +| P2 | Public API and setting names | PENDING: developer to settle the names | +| D-hash | The 25 golden entries in §e move; nothing else | PENDING: needs the developer's approval of the list (golden rule) | +| D-legacy | Reading a pre-0.3 user-class column directly raises a clear error | PENDING: stored-data behaviour, proposed here | +| D-notice | One-time notice when a `__main__` class is first registered (S3) | PENDING: developer leaned towards a warning; first scope cut | + +## a. The identity definition + +A user class's identity is an **identity record** with three components: + +```json +{"fields": {"x": "int", "y": "float"}, "kind": "dataclass", "name": "Point"} +``` + +### Name component — CONFIRMED (decision 1) + +The name is a string chosen once per class, by the first rule that applies: + +1. **Declared name.** The user declared one (§b). Used verbatim. +2. **`__main__` class** (`cls.__module__` is `"__main__"` or `"__mp_main__"`, the name a + `multiprocessing` spawn child gives the main module, W14): `cls.__qualname__` + (`"Point"`, or `"Outer.Point"` for a nested class), under the default setting + `main="name"`. Setting `main="qualified"` gives `"__main__.Point"` (today's string). +3. **Module class:** `f"{module}.{qualname}"` (`"pkg.models.Point"`), under the default + setting `module="qualified"`. Setting `module="name"` gives `"Point"`. The module is + passed through `canonical_module_name`, which leaves user modules unchanged. + +A name must be non-empty, must not contain `@` or whitespace, and must not start with +`orcapod.` (reserved for built-in types such as `orcapod.path`). A class whose +`__qualname__` contains `` stays rejected in v0.3, even with a declared name +(scope, §g). + +### Kind component — CONFIRMED (decision 3) + +`"dataclass"` (`dataclasses.is_dataclass`, which includes `pydantic.dataclasses.dataclass`) +or `"pydantic"` (`pydantic.BaseModel` subclass). A dataclass and a pydantic model with the +same name and fields have different identity records, so they are different types. + +### Field component — CONFIRMED (decision 2), nested classes PENDING (P1) + +`fields` maps each **stored field's name** to its **type token**. It is a JSON object +serialized with sorted keys, so field order never enters. The stored fields are exactly +the ones stored today: dataclass fields with `init=True`, pydantic `model_fields`. + +Type token of an annotation (from `typing.get_type_hints`, which strips `Annotated` +extras): + +| Annotation | Token | Example | +|---|---|---| +| builtin scalar | its name | `int`, `float`, `str`, `bool`, `bytes` | +| other class without a logical type (stdlib) | `canonical_type_name(cls)` | `datetime.datetime`, `decimal.Decimal` | +| type with a built-in logical type | its logical type name | `orcapod.path`, `orcapod.uuid`, `orcapod.file` | +| generic container | `origin[arg, …]`, origin by its canonical name, args in order | `list[int]`, `dict[str, float]`, `tuple[int, str]` (`typing.List[int]` gives `list[int]`) | +| `Optional` / union | member tokens sorted, joined by ` \| ` | `int \| None` for `Optional[int]`, `int \| None` and `None \| int` | +| nested dataclass / pydantic model (**P1 default**) | that class's identity string (below) | `Inner@3c1d…` | +| nested class (**P1 alternative**) | its fields only: `struct{a: int, b: str}` (sorted, no name, no kind) | — | + +Deliberately **not** part of the identity: defaults and default factories, methods and +validators, properties and pydantic `computed_field`s, `init=False` dataclass fields, +`ClassVar`s, private attributes, field order, docstrings, pydantic field aliases and +constraints (`Field(gt=0)`, `Annotated` metadata), `frozen`/`slots`/config flags, base +classes (only the resulting fields count). Computed fields are deferred (D-computed); a +later decision to include them adds a key that is absent when empty, so it would move +only classes that have computed fields. + +The token is built from the Python annotation, not the Arrow type, because the developer +named field *types* and because the Arrow type cannot tell `str` from `orcapod.path` +(both `large_string`). + +### Identity string + +``` +identity string = f"{name}@{fingerprint}" +fingerprint = sha256(canonical_json(record)).hexdigest()[:16] +canonical_json = json.dumps(record, sort_keys=True, separators=(",", ":")) +``` + +`Point(x: int, y: float)` as a dataclass gives `Point@9117beaaf7ebe3ab`; with `y: int`, +`Point@c0a75f0c39b86873`; as a pydantic model with `y: float`, `Point@703bff8af8d6002c`. +SHA-256 of a documented JSON string, not the Orcapod semantic hasher, so the identity can +be reproduced by hand and does not depend on a hasher version. 16 hex digits (64 bits) +only have to separate classes with the same name and kind. + +### Where each component enters + +The identity string replaces today's `f"{module}.{qualname}"` as the class's logical +type name. Every place that used the old string now uses the new one; there is no +second mechanism. + +| Place | What it carries | How | +|---|---|---| +| Logical type name | identity string | `DataclassLogicalType.logical_type_name` / pydantic equivalent | +| Arrow extension name (`ARROW:extension:name`) | identity string | same string; one Arrow/Polars extension registration per identity | +| Arrow extension metadata | category plus the full record: `{"category":"orcapod.dataclass","identity":{…}}`, canonical JSON | read path uses it to find the class and to print the stored shape | +| Arrow storage type | struct of the stored fields | unchanged (derived from the annotations) | +| Schema hash, source `pipeline_hash`, system-tag column names | identity string | `TypeObjectHandler` hashes a registered class as `type:` (unchanged code) | +| Pod signature hash, node `pipeline_hash`, DB table paths | identity string | via the schema/signature hashes | +| Per-row data hash, `content_hash` | identity string + record + values | the extension column is hashed with its extension name and metadata (unchanged code) | +| `list[Point]`, `Point \| None`, nested fields | identity string | container logical names embed the element's logical name; nested fields via the token (P1) | + +**Same name, different fields → different pipeline identity.** `Point(x: int, y: int)` +and `Point(x: int, y: float)` have different fingerprints, so different logical type +names, different schema hashes, different pod and node pipeline hashes, and different +table paths. They never share a table. That removes the "same table, different shape" +failure (W6, W10: today 0 rows plus log warnings): the changed class computes into its own +tables, and the old tables stay as they were. In one process the two shapes have two +extension names, so Arrow's and Polars' global extension registries (keyed by name) hold +both without conflict; that is why the fields must be in the extension *name*, not only +in its metadata. + +## b. Public API — PENDING (P2: names), behaviour proposed + +```python +import orcapod as op + +@op.register_type # name by the setting: __main__ → "Point", module → "pkg.models.Point" +@dataclass +class Point: ... + +@op.register_type(name="Point") # declared name, same everywhere it is defined +@dataclass +class Point: ... + +op.register_type(Point, name="Point") # call form, for classes you do not own + +op.set_type_naming(main="name", module="qualified") # process-wide defaults (shown) +``` + +- `register_type` records the name for the class object (a process-wide weak-keyed table, + so the class is not modified), validates it, builds its logical type in the default + data context (so an unsupported field type fails at the definition, not at first use) + and returns the class unchanged. Classes that are never registered are registered on + first use, as today, with the name from the setting. +- `set_type_naming` changes the default for classes registered after the call. Values: + `"name"` or `"qualified"`. Defaults: `main="name"`, `module="qualified"`. No TOML key + in v0.3 (scope, §g). +- Registering a class object under a second, different name after it is registered (for + example `register_type(Point, name="Geo")` after `Point` was used) raises `ValueError` + naming both names: one class object has one identity per process. + +**Name collisions (same kind, same name, two different class objects):** + +| Case | What happens | What the user sees | +|---|---|---| +| Same fields, same `module.qualname` (cell re-run, `%autoreload`, W1/W3/W4) | Same identity: the identity is rebound to the new class object (§c) | nothing | +| Same fields, different `module.qualname` (two modules declare `"Point"`, or `module="name"`) | Same identity by definition: they are the same type; values read back are built as the most recently registered class | one `UserWarning` per rebinding naming both classes. Warranted: values come back as a different class object than may be expected | +| Different fields (W2 with changed fields, or two modules) | Different identities: two types that happen to share a display name; no clash | nothing at registration. Reading picks the class whose identity matches (§d) | + +## c. Rebinding rule (S1) — CONFIRMED (developer answer to S1) + +- **Same identity string, new class object → rebind silently.** The registry rebinds the + identity (logical name and extension name) to a logical type built for the new class. + Earlier class objects keep their mapping, so instances created before the re-run still + convert. Arrow's global registration for that extension name is reused (same name, + storage type and metadata by construction). This fixes W1, W3, W4. +- **Different identity under the same name → a different type.** Nothing is rebound; + the new class gets its own logical type and extension name (W2: the next run computes + into new tables). +- **Still raises:** + 1. One class object registered under two names (§b). + 2. An invalid declared name (§a). + 3. A local class (``), as today. + 4. A shape disagreement on read (§d, S2). + +## d. Read path — CONFIRMED behaviour (S2), lookup mechanism proposed + +A stored column carries the extension name `Point@9117beaaf7ebe3ab` and the record in its +metadata. To build values, the reader resolves a class: + +1. **Registered identity:** the extension name is registered in the reading context → use it. +2. **Declared or registered by name:** classes recorded under `(kind, name)` in this + process (by `register_type` or by first use); take the one whose identity string + matches. +3. **`__main__` lookup:** walk the name as an attribute path from + `sys.modules["__main__"]` (`"Point"`, `"Outer.Point"`), accepted if its identity matches. + This makes W12 (read notebook results from a script that defines the same `Point`) + work without a declaration. +4. **Import** (dotted name): import by the longest module prefix, as today, accepted if + its identity matches. + +Outcomes: + +- **No candidate found:** `ImportError` (kept), naming the declared name, the kind, the + stored fields, and how to fix it: "define the class and register it with + `op.register_type(Point, name='Point')` before reading". +- **Candidates found, none with a matching identity (S2):** `TypeIdentityError` + (new, a `ValueError` subclass in `orcapod.errors`) naming the column where the read + path knows it, the class name and kind, the stored fields and the registered fields, + for example "column 'p': stored Point (dataclass) has fields {x: int, y: int}; the + registered Point has {x: int, y: float}". +- **Same identity, different Arrow storage** (possible only if two data contexts map one + annotation to different Arrow types): `TypeIdentityError` naming the column, the class + and both struct types. The existing check in the synthesized + `__arrow_ext_deserialize__` raises it. +- These errors propagate. `DeltaTableDatabase` today catches every read error and logs + "Error reading Delta table"; it re-raises `TypeIdentityError`, so a run can no longer + report success with 0 rows. Other read errors keep their current handling (out of + scope). The misleading "data may have been deleted externally" message is untouched + unless the path is reached for this reason (§g lists it as out of scope). +- **Pre-0.3 column** (D-legacy, PENDING): an extension name without `@` and metadata + without `identity`, read directly (pipelines never reach these tables, §e): raise + `TypeIdentityError` saying the column was written by Orcapod before 0.3 under + `pkg.models.Point`, that v0.3 identifies the class by name and fields, and that the + pipeline recomputes into new tables. Not converted, not dropped. + +## e. Intended hash changes — PENDING (developer approval of the list) + +Every pinned value that involves a dataclass or pydantic type moves, because the identity +string changes for every user class (fields are now in it, for everyone). Measured on +`origin/main` (c5ab1e05) with E08 (`--ref origin/main`): changing the identity string of the +golden fixtures `GoldenPoint` and `GoldenModel` moves exactly these 25 entries, and the +`unchanged` control moves 0. Every identity-string change moves the same set (E08's +`__main__` and module-less variants list identical entries), so the v0.3 string moves +these and only these. The implementation re-runs the golden test and the PR lists the +measured diff; any entry outside this list is a bug. + +| File | Entries | +|---|---| +| `semantic.json` (2) | `type.dataclass`, `type.pydantic` | +| `schema.json` (5) | `multi_field.logical_mix`, `single_field.dataclass.arrow_type`, `single_field.dataclass.schema_hash`, `single_field.pydantic.arrow_type`, `single_field.pydantic.schema_hash` | +| `datagram.json` (10 change) | `{dataclass,pydantic}.content_hash`, `.pipeline_hash`, `.system_tag_columns`, `.row[{"k":"r1"}].data_hash`, `.row[{"k":"r2"}].data_hash` | +| `datagram.json` (8 replaced) | `{dataclass,pydantic}.row[{"k":"r1"/"r2"}]._tag_record_id::` and `._tag_source_id::`: the keys embed the moved pipeline hash, so these 8 keys disappear and 8 keys with the new hash appear | + +`arrow.json` and `pipeline.json` have no user-class entries and do not move. Golden +additions (new keys only): a declared-name dataclass, a nested dataclass (P1), an +`Optional` field, a container field, and permuted-field-order twins that must pin the +same value. + +**For the CHANGELOG (stated plainly):** "Orcapod 0.3 identifies a dataclass or pydantic +model by its name, its kind and its fields. Every pipeline whose data includes such a +class gets new pipeline hashes and table paths, so the first run under 0.3 recomputes +those pod and operator results into new tables. Tables written by earlier versions are +left untouched and are not read by 0.3 pipelines. There is no migration tool. Classes +defined in a notebook or script (`__main__`) are now identified by their name only, so +results survive moving the class into a module if it keeps its name and is registered +with `op.register_type(name=...)`." Results that do not involve user classes are +unaffected. + +## f. Interactions with PR #286 and PR #289 + +- **PR #286** (polars types) touches `builtin_handlers.py`, `logical_types/__init__.py`, + `contexts/data/v0.1.json`, `docs/concepts/function-pods.md`, the golden `cases.py` and + `semantic.json`/`schema.json`/`datagram.json`. This work does not touch + `builtin_handlers.py` or `v0.1.json` (the identity reaches hashing through the + logical type name), adds a new docs page instead of editing `function-pods.md` beyond a + link, and edits the golden files only by regeneration. +- **PR #289** (`orcapod.ndarray`) adds `array_format.py` and tests; no file in common. +- **Preferred order:** #286 and #289 merge first. This branch then merges `origin/main`, + re-runs E08 against the new main to confirm the moved set, and regenerates the golden + files; the PR's golden diff must then show only the §e entries plus the additions. If + this PR merges first, #286 regenerates its own additive entries after merging main; + its user-class-free entries do not move. + +## g. Scope cuts, in order, if time runs short + +1. TOML/config-file setting for the naming defaults (`set_type_naming` stays). +2. The one-time `__main__` notice (S3, D-notice). +3. The collision `UserWarning` for same identity from a different `module.qualname` + (behaviour stays: rebind). +4. The `__main__` lookup on read (§d step 3); a declaration is then required to read + `__main__`-named data in another process. +5. `set_type_naming` itself (defaults fixed: `main="name"`, `module="qualified"`). + +Not cut: the identity record and string, rebinding, `register_type`, the S2 raise, the +read lookup by declared name, the docs page and the CHANGELOG entry. + +Out of scope for v0.3: computed fields; local classes with a declared name; Enum, +NamedTuple and TypedDict support (D6 of the findings); the `DeltaTableDatabase` row-count +and shutdown-hang findings (§9 of the findings). + +## Workshop scenarios after v0.3 (predicted, become the Step 2 tests) + +| # | Scenario | v0.3 | +|---|---|---| +| W1/W3 | Re-run the cell, same fields (dataclass / pydantic) | silent rebind; cache hit | +| W2 | Re-run with changed fields | new type; recompute into new tables; old tables untouched | +| W4 | `%autoreload` of a module class | silent rebind | +| W5/W9 | Restart / second notebook, identical class | cache hit | +| W6/W10 | Restart / second notebook, changed fields | clean recompute, new tables; no 0-row success | +| W7 | Script as `__main__` twice | cache hit | +| W8 | `python -m pkg.pipeline` then `import pkg.pipeline` | miss unless declared (`"Point"` vs `"pkg.pipeline.Point"`); hit with `@op.register_type(name="Point")` | +| W11 | Notebook → module | hit if the module class declares the same name (`"Point"`) | +| W12 | Read notebook results from a script | works if the script defines a matching `Point` (top level or declared); otherwise the `ImportError` of §d | +| W14 | spawn child (`__mp_main__`) | same identity as `__main__` | +| W15 | Local class | `ValueError`, as today | From fae9ba720c2afea7dc92dc9a360b201f9259a7b2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:12:23 -0800 Subject: [PATCH 02/31] docs(specs): record P1, registration order and cycle rules for user class identity (ITL-881) Refs ITL-881 Co-Authored-By: Claude Opus 5.5 --- .../2026-11-10-user-class-identity-v0.3.md | 56 ++++++++++++++----- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md index 32d87c730..d5aeef7fb 100644 --- a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -1,6 +1,6 @@ # Identity of a user class in v0.3 (ITL-881) -**Status:** implementation spec, waiting for approval. No product code changes yet. +**Status:** implementation spec, approved by the coordinator November 10, 2026 (P1 decided by the developer; P2 names and the identity-string format provisional). **Issue:** ITL-881 (v0.3, a public release; tag target November 16, 2026). **Builds on:** the findings in PR #288 (`superpowers/specs/2026-11-10-user-class-identity.md`, experiments E01–E08). "W1"–"W15" are the workshop scenarios from §3.4 of that document. @@ -17,11 +17,12 @@ Each decision is marked **CONFIRMED** (the developer decided it, November 10, 20 | D-rebind | Same identity re-registered: rebind silently | CONFIRMED: developer answer to S1 | | D-raise | Stored shape disagreeing with registered shape raises, in v0.3 | CONFIRMED: developer answer to S2 | | D-doc | A page documenting identity | CONFIRMED: developer answer to S4 | -| P1 | A field typed by another user class carries that class's full identity | PENDING: developer asked; default stated by the coordinator | -| P2 | Public API and setting names | PENDING: developer to settle the names | -| D-hash | The 25 golden entries in §e move; nothing else | PENDING: needs the developer's approval of the list (golden rule) | -| D-legacy | Reading a pre-0.3 user-class column directly raises a clear error | PENDING: stored-data behaviour, proposed here | -| D-notice | One-time notice when a `__main__` class is first registered (S3) | PENDING: developer leaned towards a warning; first scope cut | +| P1 | A field typed by another user class carries that class's identity as that class was registered | CONFIRMED: developer decision, November 10, 2026 | +| D-cycle | A self-referential or cyclic class is rejected with a clear `TypeError` | CONFIRMED by the coordinator from this proposal: Arrow cannot store a recursive struct | +| P2 | Public API names (`register_type`, `set_type_naming`) and the identity-string format | PENDING with the developer; implemented provisionally, each a single-point change | +| D-hash | The 25 golden entries in §e move; nothing else | CONFIRMED by the coordinator: follows from decision 2; the PR lists every moved entry | +| D-legacy | Reading a pre-0.3 user-class column directly raises a clear error | CONFIRMED by the coordinator: a consequence of the identity check; nothing extra is built | +| D-notice | One-time notice when a `__main__` class is first registered (S3) | CONFIRMED in scope by the coordinator; last item before the cuts | ## a. The identity definition @@ -55,7 +56,7 @@ A name must be non-empty, must not contain `@` or whitespace, and must not start or `"pydantic"` (`pydantic.BaseModel` subclass). A dataclass and a pydantic model with the same name and fields have different identity records, so they are different types. -### Field component — CONFIRMED (decision 2), nested classes PENDING (P1) +### Field component — CONFIRMED (decision 2; nested classes: P1) `fields` maps each **stored field's name** to its **type token**. It is a JSON object serialized with sorted keys, so field order never enters. The stored fields are exactly @@ -71,8 +72,30 @@ extras): | type with a built-in logical type | its logical type name | `orcapod.path`, `orcapod.uuid`, `orcapod.file` | | generic container | `origin[arg, …]`, origin by its canonical name, args in order | `list[int]`, `dict[str, float]`, `tuple[int, str]` (`typing.List[int]` gives `list[int]`) | | `Optional` / union | member tokens sorted, joined by ` \| ` | `int \| None` for `Optional[int]`, `int \| None` and `None \| int` | -| nested dataclass / pydantic model (**P1 default**) | that class's identity string (below) | `Inner@3c1d…` | -| nested class (**P1 alternative**) | its fields only: `struct{a: int, b: str}` (sorted, no name, no kind) | — | +| nested dataclass / pydantic model (P1) | that class's identity string (below), as that class is registered | `Inner@3c1d…`, `geo.Inner@…` | + +**Nested classes (P1, CONFIRMED).** The token of a field typed by another user class is +that class's own registered identity string: its name component however *that* class was +registered (declared name, name only, or `module.qualname`), its kind, and its field +component, recursively through the fingerprint. It is never re-derived from +`module.qualname` and never reduced to the storage struct. So renaming the inner class, +declaring it under another name, or changing its fields changes the outer identity. + +**Registration order does not matter.** `register_type` only records the declared name +and checks it (§b); the logical type, and so the identity, is built at first use, and the +outer class's build registers the inner class first (implicitly, by the setting, if it +was never declared). Declaring every class before first use therefore gives the same +identities in any order: outer decorated before inner, inner declared later in the same +cell, inner registered implicitly by the outer's first use. Declaring a class after it +was used under a different name raises (§c). An identical notebook re-run of the inner +class rebinds it (§c) and leaves the outer identity unchanged. + +**Cycles (D-cycle).** A class that refers to itself, directly (`next: Node | None`) or +through other classes, is rejected at first use with a `TypeError` naming the cycle +(`Node.next -> Node`). Today's converter already raises `TypeError: Circular type +dependency detected…`; v0.3 keeps the rejection and names the path. Encoding the +back-reference by name would give a well-defined identity, but Arrow cannot store a +recursive struct, so there is nothing to identify. Deliberately **not** part of the identity: defaults and default factories, methods and validators, properties and pydantic `computed_field`s, `init=False` dataclass fields, @@ -146,10 +169,13 @@ op.set_type_naming(main="name", module="qualified") # process-wide defaults (s ``` - `register_type` records the name for the class object (a process-wide weak-keyed table, - so the class is not modified), validates it, builds its logical type in the default - data context (so an unsupported field type fails at the definition, not at first use) - and returns the class unchanged. Classes that are never registered are registered on - first use, as today, with the name from the setting. + so the class is not modified), checks the kind, the name and that the class is not + local, and returns the class unchanged. The logical type is built at first use, as + today, so declarations may come in any order (§a). Classes that are never declared are + registered on first use with the name from the setting. +- Provisional (P2): the public names live in one place (`orcapod/user_types.py`, + re-exported from `orcapod`) and the identity-string format in one function + (`identity_string(record)`), so a rename or a format change is a single-point edit. - `set_type_naming` changes the default for classes registered after the call. Values: `"name"` or `"qualified"`. Defaults: `main="name"`, `module="qualified"`. No TOML key in v0.3 (scope, §g). @@ -176,7 +202,9 @@ op.set_type_naming(main="name", module="qualified") # process-wide defaults (s the new class gets its own logical type and extension name (W2: the next run computes into new tables). - **Still raises:** - 1. One class object registered under two names (§b). + 1. One class object registered under two names (§b), including a declaration after + the class was already used under its default name. + 5. A self-referential or cyclic class (D-cycle). 2. An invalid declared name (§a). 3. A local class (``), as today. 4. A shape disagreement on read (§d, S2). From 394f60f83803ae069521c37c398af66e020bfd91 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:15:05 -0800 Subject: [PATCH 03/31] docs(plans): implementation plan for v0.3 user class identity (ITL-881) Also records in the spec that a subclass never borrows its parent's identity. Refs ITL-881 Co-Authored-By: Claude Opus 5.5 --- ...-11-10-itl-881-user-class-identity-impl.md | 184 ++++++++++++++++++ .../2026-11-10-user-class-identity-v0.3.md | 6 + 2 files changed, 190 insertions(+) create mode 100644 superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md diff --git a/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md new file mode 100644 index 000000000..caf332c3e --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md @@ -0,0 +1,184 @@ +# User class identity (v0.3) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Identify a dataclass or pydantic model by `{kind, name, fields}` (spec +`superpowers/specs/2026-11-10-user-class-identity-v0.3.md`), with a declared-name API, silent +rebinding of identical re-registrations, and a raise on stored/registered shape mismatch. + +**Architecture:** One new module, `orcapod/logical_types/user_class_identity.py`, owns the +naming policy, declarations, type tokens, the identity record and string (single-point +format), the process-wide `(kind, name)` table and the read-side class lookup. The two +factories share a new base (`user_class_logical_type.py`) that builds the identity string +and uses it as logical type name and Arrow extension name; the metadata carries the record. +`LogicalTypeRegistry` learns to rebind an identical identity to a new class object and stops +matching user classes by subclass. The public API (`register_type`, `set_type_naming`) lives +in `orcapod/user_types.py`. `TypeIdentityError` lives in `orcapod/errors.py`; the Delta read +path re-raises it. + +**Tech Stack:** Python 3.11–3.13, pyarrow, polars, pydantic v2, deltalake, IPython (dev, for +the notebook scenarios), pytest via `uv run`. + +--- + +## File map + +| File | Change | +|---|---| +| `src/orcapod/errors.py` | add `TypeIdentityError(ValueError)` | +| `src/orcapod/logical_types/user_class_identity.py` | **new**: kinds, naming policy, declarations, `type_token`, `identity_record`, `canonical_json`, `identity_string`, known-class table, `find_class`, `describe_fields`, S3 notice | +| `src/orcapod/logical_types/user_class_logical_type.py` | **new**: `UserClassLogicalType`, `UserClassLogicalTypeFactory` (shared write/read path) | +| `src/orcapod/logical_types/dataclass_logical_type_factory.py` | reduce to subclasses: kind, category, `supports_class`, stored fields | +| `src/orcapod/logical_types/pydantic_logical_type_factory.py` | same | +| `src/orcapod/logical_types/registry.py` | rebinding rule; exact-type match for user classes; `TypeIdentityError` from `_deserialize` | +| `src/orcapod/semantic_types/universal_converter.py` | cycle error names the class and says recursive classes cannot be stored | +| `src/orcapod/databases/delta_lake_databases.py` | re-raise `TypeIdentityError` where read errors are logged | +| `src/orcapod/logical_types/database_hooks.py` | add the column name to a `TypeIdentityError` raised while registering a discovered type | +| `src/orcapod/user_types.py` | **new**: `register_type`, `set_type_naming` | +| `src/orcapod/__init__.py` | re-export the two functions | +| tests | `tests/test_user_types/` (new): unit tests of identity, API, registry, read path; `test_workshop_scenarios.py` (subprocess + IPython `InteractiveShell`) | +| existing tests asserting `module.qualname` names | switch to the identity string | +| `tests/test_hashing/golden/cases.py` + `data/*.json` | add new fixture cases; regenerate (25 listed moves + additions) | +| docs | new `docs/concepts/your-own-data-types.md`; section in `docs/concepts/identity.md`; nav in `mkdocs.yml`; link from `function-pods.md` | +| `CHANGELOG.md`, `DESIGN_ISSUES.md` | entry each | + +## Key definitions (used by every task) + +```python +# user_class_identity.py +KIND_DATACLASS = "dataclass" +KIND_PYDANTIC = "pydantic" +_NAMING = {"main": "name", "module": "qualified"} # set_type_naming +_MAIN_MODULES = frozenset({"__main__", "__mp_main__"}) +_DECLARED: weakref.WeakKeyDictionary[type, str] # register_type +_KNOWN: dict[tuple[str, str], list[weakref.ref[type]]] # (kind, name) -> classes + +def validate_name(name: str) -> None: ... # non-empty, no '@', no whitespace, not 'orcapod.' +def class_kind(cls: type) -> str | None: ... +def resolve_name(cls: type) -> str: ... # declared > main policy > module policy +def declare(cls: type, name: str | None) -> None: ... # raises if already built under another name +def set_naming(main: str | None, module: str | None) -> None: ... +def type_token(annotation, converter) -> str: ... +def identity_record(kind, name, fields: dict[str, str]) -> dict: ... +def canonical_json(obj) -> str: ... # sort_keys, (",", ":") +def identity_string(record) -> str: ... # f"{name}@{sha256(canonical_json)[:16]}" (single point) +def describe_fields(fields: dict[str, str]) -> str: ... # "{x: int, y: float}" +def remember(kind, name, cls) -> None: ... # _KNOWN +def candidates(kind, name) -> list[type]: ... # _KNOWN + declared + __main__ walk + import +``` + +Type tokens: builtins bare (`int`); `NoneType` → `None`; other classes without logical type → +`canonical_type_name`; class with logical type → `lt.logical_type_name` (built-ins +`orcapod.path`, user classes their identity string, P1); `list[...]`/`set[...]`/`dict[...]` +→ `origin[args]` with origin by canonical name minus `builtins.`; unions → sorted members +joined by ` | `; `Literal` → `Literal[...]` with sorted `repr`s. + +--- + +### Task 1: Identity primitives (pure functions) + +**Files:** create `src/orcapod/logical_types/user_class_identity.py`, modify +`src/orcapod/errors.py`; test `tests/test_user_types/test_identity_record.py`. + +- [ ] Step 1: failing tests: `identity_string({"kind":"dataclass","name":"Point","fields":{"x":"int","y":"float"}}) == "Point@9117beaaf7ebe3ab"`; field-order independence; `type_token` for `int`, `int | None`, `Optional[int]`, `None | int`, `list[int]`, `typing.List[int]`, `dict[str, float]`, `Path` (`orcapod.path`), `datetime.datetime`, `Literal["b","a"]`; `validate_name` rejects `""`, `"a@b"`, `"a b"`, `"orcapod.x"`; `resolve_name` for a module class (`tests.…Point` qualified), a class with `__module__="__main__"` (`"Point"`), `"__mp_main__"` (same), after `set_naming(main="qualified")` (`"__main__.Point"`), after `set_naming(module="name")`. +- [ ] Step 2: run, expect ImportError (module missing). +- [ ] Step 3: implement the functions above. +- [ ] Step 4: run, pass. Commit `feat(types): identity record and string for user classes (ITL-881)`. + +### Task 2: Shared user-class logical type; factories use the identity + +**Files:** create `src/orcapod/logical_types/user_class_logical_type.py`; reduce both factory +modules to subclasses; test `tests/test_user_types/test_user_class_logical_type.py`. + +`UserClassLogicalType(identity, record, python_type, storage_type, field_annotations, category)`: +logical name = Arrow extension name = identity string; metadata +`canonical_json({"category": category, "identity": record}).encode()`; conversions as today. +Class attribute `exact_python_type_only = True`. + +`UserClassLogicalTypeFactory` (abstract: `kind`, `category`, `supports_class`, `stored_fields(cls, hints)`): +- `create_for_python_type(cls, converter)`: reject ``; name = `resolve_name(cls)`; + for each stored field: `arrow = converter.register_python_class(annotation)` (wrap a + circular `TypeError` as `TypeError(f"{cls.__qualname__}.{field}: recursive classes cannot be stored ({exc})")`); + token = `type_token(annotation, converter)`; record; identity; `remember(kind, name, cls)`; + S3 notice when the name came from the `__main__` policy (once per name per process). +- `reconstruct_from_arrow(ext_name, storage, metadata, converter)`: no `identity` key → + `TypeIdentityError` (pre-0.3 column); else for each candidate class of (kind, name): get its + logical type through `converter.register_python_class(cls)` + + `converter.get_logical_type(cls)`; return the first whose `logical_type_name == ext_name` + (and whose storage equals the stored storage after `normalize_to_large_types`, else + `TypeIdentityError` naming both struct types). No candidate → `ImportError` naming name, + kind, stored fields and `op.register_type(Point, name='Point')`. Candidates but no match → + `TypeIdentityError` naming class, kind, stored fields, registered fields. + +- [ ] Step 1: failing tests: logical name is the identity string; metadata contains the record; two classes with permuted fields share the identity; dataclass vs pydantic same name+fields differ; same name, `y: int` vs `y: float` differ; nested dataclass token is the inner identity string and changes when the inner fields change or the inner is declared under another name (P1); outer built before inner declared gives the same identity as inner-first (order); self-referential class → `TypeError` naming `Node.next`; `` → `ValueError`; subclass with an extra field gets its own identity (exact-type match). +- [ ] Step 2: run, fail. Step 3: implement. Step 4: update the existing factory/roundtrip tests that assert `module.qualname` (listed in the file map) to read the identity from the registry. Step 5: run `uv run pytest tests/test_logical_types tests/test_user_types -q`, pass. Commit. + +### Task 3: Registry rebinding and exact-type match + +**Files:** `src/orcapod/logical_types/registry.py`; test `tests/test_user_types/test_rebinding.py`. + +- `register_logical_type`: when the logical name is bound to a different instance whose + Arrow extension type has the same name, storage type and serialized metadata, and the + Python type differs → rebind: replace the logical-name and Arrow-name bindings, add the new + Python-type binding, keep the old Python-type binding. If the previous class's + `module.qualname` differs from the new one's → one `UserWarning` naming both. Anything + else keeps raising. +- `get_by_python_type`: skip the `issubclass` fallback for logical types with + `exact_python_type_only`. +- `_deserialize`: raise `TypeIdentityError` (subclass of `ValueError`) for storage/metadata mismatch. + +- [ ] Steps: failing tests (rebind same identity silently, `warnings` none; different module.qualname same identity → one `UserWarning`; one class object under two names → `ValueError`), implement, pass, commit. + +### Task 4: Public API + +**Files:** create `src/orcapod/user_types.py`; modify `src/orcapod/__init__.py`; test +`tests/test_user_types/test_register_type.py`. + +```python +def register_type(cls=None, /, *, name=None): # decorator, decorator factory, or call +def set_type_naming(*, main=None, module=None) -> None: +``` + +- [ ] Steps: failing tests through `op.register_type` / `op.set_type_naming` and a `DictSource` + `function_pod` pipeline: declared name gives the identity `Name@…`; `register_type(Point, name="Geo")` after first use under another name → `ValueError`; invalid names; non-dataclass → `TypeError`; returns the class. Implement, pass, commit. + +### Task 5: Read path and S2 guard + +**Files:** `delta_lake_databases.py`, `database_hooks.py`; test +`tests/test_user_types/test_read_path.py` (public interface: pipeline in a subprocess writes +to `DeltaTableDatabase`, a second subprocess reads with a different class). + +- [ ] Steps: failing tests: reader with matching class (declared in another module) reads values; reader without the class → `ImportError` naming `Point` and `register_type`; reader whose `Point` has different fields reading the stored table directly → `TypeIdentityError` naming column, both field sets; re-run of the pipeline with changed fields computes into new tables (calls == rows), old table still readable by the old class. Implement re-raise, pass, commit. + +### Task 6: Workshop scenarios W1–W12 as tests + +**File:** `tests/test_user_types/test_workshop_scenarios.py` (each scenario in a fresh +subprocess; notebook cells through `IPython.core.interactiveshell.InteractiveShell`, as E02; +children end with `os._exit(0)` after printing JSON, as the E-harness does). + +W1 re-run same fields (calls 0 on second run, no error); W2 changed fields (new tables: +calls 2, rows 2, no error); W3 pydantic re-run; W4 `importlib.reload`; W5 restart + re-run +(two processes, cache hit); W6 restart with changed fields (recompute, rows 2); W7 script as +`__main__` twice; W8 `-m` vs import with a declared name (hit); W11 notebook → module with a +declared name: identical pipeline hashes and table paths; W12 read notebook results from a +script defining the same `Point`. Plus: pydantic vs dataclass same name+fields are distinct +(different pod pipeline hashes). + +- [ ] Steps: write, run (must fail before Tasks 1–5 on `main` for W1–W4, W6, W11, W12; record which), pass after, commit. + +### Task 7: Golden values + +- [ ] Add fixture cases to `cases.py` (new names only; nothing moved): declared-name dataclass, nested dataclass, `Optional` field, container field, permuted-order twin pair. +- [ ] Run `uv run pytest tests/test_hashing/golden -q`: exactly the 25 spec entries fail. Record the list. +- [ ] Regenerate; `git diff --stat` and a script comparing old/new JSON: changed keys == the 25, plus only added keys. Commit `test(hashing): golden values for v0.3 user class identity (ITL-881)`. + +### Task 8: Docs, CHANGELOG, DESIGN_ISSUES + +- [ ] `docs/concepts/your-own-data-types.md` (runnable samples: define, declare, re-run, what changes identity), identity section in `docs/concepts/identity.md`, nav entry, link from `function-pods.md`. Run `uv run pytest tests/test_docs -q`. +- [ ] CHANGELOG entry with the hash-change statement from spec §e. +- [ ] DESIGN_ISSUES entry appended at the end (resolved, with Fix note). + +### Task 9: Verification and PR + +- [ ] ruff check/format, basedpyright exit code, full `uv run pytest tests/` on 3.11/3.12/3.13 (`uv run --python 3.1x`). +- [ ] Merge `origin/main` (after #286/#289 land), re-run E08 on the new main, regenerate golden only for the listed entries. +- [ ] PR body: intended hash changes list; mark ready via sensei:create-pr. diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md index d5aeef7fb..96252c9ed 100644 --- a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -56,6 +56,12 @@ A name must be non-empty, must not contain `@` or whitespace, and must not start or `"pydantic"` (`pydantic.BaseModel` subclass). A dataclass and a pydantic model with the same name and fields have different identity records, so they are different types. +**Subclasses.** Each class has its own identity; a subclass never borrows its parent's. +Today the registry matches an unregistered class to a registered *parent* by +`issubclass`, so `class P3(Point)` with an extra field `z` is stored as `Point` and `z` is +silently dropped (reproduced on `main`, November 10, 2026). v0.3 matches dataclass and +pydantic logical types by exact class only. + ### Field component — CONFIRMED (decision 2; nested classes: P1) `fields` maps each **stored field's name** to its **type token**. It is a JSON object From 27d4a9e31262b44719fe512c6db8ed19ecbe7fb1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:16:51 -0800 Subject: [PATCH 04/31] feat(types): identity record, string and naming policy for user classes (ITL-881) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/errors.py | 11 + .../logical_types/user_class_identity.py | 446 ++++++++++++++++++ tests/test_user_types/__init__.py | 0 tests/test_user_types/test_identity_record.py | 143 ++++++ 4 files changed, 600 insertions(+) create mode 100644 src/orcapod/logical_types/user_class_identity.py create mode 100644 tests/test_user_types/__init__.py create mode 100644 tests/test_user_types/test_identity_record.py diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 81c653627..94305e06a 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -20,6 +20,17 @@ class SchemaInconsistencyError(InputValidationError): """ +class TypeIdentityError(ValueError): + """Raised when stored values and a registered user class disagree on identity. + + A stored dataclass or pydantic column records its class's identity (name, kind and + field types). This error is raised when the class found for that name has a + different identity or a different Arrow storage type, or when the column was + written by an Orcapod version that did not record the identity. The message names + the column where it is known, the class, and both shapes. + """ + + class DuplicateTagError(ValueError): """Raised when duplicate tag values are found and skip_duplicates=False""" diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py new file mode 100644 index 000000000..5f95454ee --- /dev/null +++ b/src/orcapod/logical_types/user_class_identity.py @@ -0,0 +1,446 @@ +"""Identity of user classes (dataclasses and pydantic models). + +A user class is identified by an *identity record* with three components (ITL-881, +spec ``superpowers/specs/2026-11-10-user-class-identity-v0.3.md``): + +- ``kind``: ``"dataclass"`` or ``"pydantic"``. +- ``name``: a declared name, or a name chosen by the naming policy (the qualified name + for ``__main__`` classes is just ``__qualname__`` by default; module classes use + ``module.qualname`` by default). +- ``fields``: a mapping from each stored field's name to its *type token*. It is + serialized with sorted keys, so field order never enters the identity. + +The *identity string* ``f"{name}@{fingerprint}"`` is the class's logical type name and +Arrow extension name. ``identity_string`` is the only place that defines its format. + +This module also holds the process-wide state that belongs to classes rather than to a +data context: declared names, the naming policy and the table of classes seen under each +``(kind, name)``, which the read path uses to find a class for a stored column. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import importlib +import json +import re +import sys +import typing +import warnings +import weakref +from typing import TYPE_CHECKING, Any, Literal + +from orcapod.utils.canonical_type_names import canonical_type_name + +if TYPE_CHECKING: + from orcapod.logical_types.protocols import TypeConverterProtocol + +KIND_DATACLASS = "dataclass" +KIND_PYDANTIC = "pydantic" + +NamingStyle = Literal["name", "qualified"] + +_NAMING_STYLES: frozenset[str] = frozenset({"name", "qualified"}) +_DEFAULT_NAMING: dict[str, NamingStyle] = {"main": "name", "module": "qualified"} +_naming: dict[str, NamingStyle] = dict(_DEFAULT_NAMING) + +#: Modules whose classes are "defined in a notebook or script". ``__mp_main__`` is the +#: name a ``multiprocessing`` spawn child gives the main module. +MAIN_MODULES: frozenset[str] = frozenset({"__main__", "__mp_main__"}) + +#: Prefix reserved for Orcapod's own types (``orcapod.path``, ``orcapod.uuid``, ...). +RESERVED_PREFIX = "orcapod." + +#: Number of hex digits of the SHA-256 fingerprint in an identity string. +FINGERPRINT_HEX_DIGITS = 16 + +_NAME_FORBIDDEN = re.compile(r"[@\s]") + +# Declared names, keyed by class object (weak, so classes from re-run cells can go away). +_declared: weakref.WeakKeyDictionary[type, str] = weakref.WeakKeyDictionary() +# The name each class object was built under; a class has one name per process. +_built: weakref.WeakKeyDictionary[type, str] = weakref.WeakKeyDictionary() +# Classes seen under each (kind, name), most recent last. +_known: dict[tuple[str, str], list[weakref.ref[type]]] = {} +# Names that already produced the one-time ``__main__`` notice. +_noticed_main_names: set[str] = set() + + +# --------------------------------------------------------------------------- +# Kinds and names +# --------------------------------------------------------------------------- + + +def class_kind(cls: Any) -> str | None: + """Return the kind of a user class, or ``None`` if it is not one. + + Args: + cls: Any object. + + Returns: + ``"dataclass"`` for a dataclass type (including pydantic dataclasses), + ``"pydantic"`` for a ``pydantic.BaseModel`` subclass, else ``None``. + """ + if not isinstance(cls, type): + return None + if dataclasses.is_dataclass(cls): + return KIND_DATACLASS + from pydantic import BaseModel + + if issubclass(cls, BaseModel): + return KIND_PYDANTIC + return None + + +def validate_name(name: str) -> None: + """Check that ``name`` can be used as the name of a user class. + + Args: + name: The proposed name. + + Raises: + TypeError: If ``name`` is not a string. + ValueError: If ``name`` is empty, contains ``@`` or whitespace, or starts with + the reserved ``orcapod.`` prefix. + """ + if not isinstance(name, str): + raise TypeError(f"A type name must be a string, got {name!r}.") + if not name: + raise ValueError("A type name must not be empty.") + if _NAME_FORBIDDEN.search(name): + raise ValueError( + f"Invalid type name {name!r}: a name must not contain '@' or whitespace." + ) + if name.startswith(RESERVED_PREFIX): + raise ValueError( + f"Invalid type name {name!r}: the prefix {RESERVED_PREFIX!r} is reserved " + f"for Orcapod's own types." + ) + + +def get_naming() -> dict[str, NamingStyle]: + """Return the current naming policy as ``{"main": ..., "module": ...}``.""" + return dict(_naming) + + +def set_naming( + main: NamingStyle | None = None, module: NamingStyle | None = None +) -> None: + """Set how classes that declare no name are named. + + Applies to classes registered after the call. + + Args: + main: ``"name"`` (default) or ``"qualified"`` for classes defined in + ``__main__`` (a notebook or a script run directly). + module: ``"qualified"`` (default) or ``"name"`` for classes defined in an + importable module. + + Raises: + ValueError: If a value is not ``"name"`` or ``"qualified"``. + """ + for key, value in (("main", main), ("module", module)): + if value is None: + continue + if value not in _NAMING_STYLES: + raise ValueError( + f"Invalid naming style for {key}: {value!r}. " + f"Use 'name' or 'qualified'." + ) + _naming[key] = value + + +def is_main_class(cls: type) -> bool: + """Return True if ``cls`` was defined in ``__main__`` (or ``__mp_main__``).""" + return cls.__module__ in MAIN_MODULES + + +def resolve_name(cls: type) -> str: + """Return the name component for ``cls``. + + The first rule that applies wins: a declared name; the naming policy for + ``__main__`` classes; the naming policy for module classes. + + Args: + cls: A dataclass or pydantic model class. + + Returns: + The name component of the class's identity. + """ + declared = _declared.get(cls) + if declared is not None: + return declared + style = _naming["main" if is_main_class(cls) else "module"] + if style == "name": + return cls.__qualname__ + module = cls.__module__ + if not is_main_class(cls): + from orcapod.utils.canonical_type_names import canonical_module_name + + module = canonical_module_name(cls) or module + return f"{module}.{cls.__qualname__}" + + +def declare(cls: type, name: str | None) -> None: + """Record the name a user class is identified by. + + Args: + cls: A dataclass or pydantic model class. + name: The declared name, or ``None`` to use the naming policy. + + Raises: + TypeError: If ``cls`` is not a dataclass or pydantic model class. + ValueError: If the class is local, the name is invalid, or the class was + already registered in this process under a different name. + """ + kind = class_kind(cls) + if kind is None: + raise TypeError( + f"Only dataclasses and pydantic models can be registered as types, " + f"got {cls!r}." + ) + reject_local_class(cls) + if name is not None: + validate_name(name) + effective = name if name is not None else resolve_name(cls) + built_as = _built.get(cls) + if built_as is not None and built_as != effective: + raise ValueError( + f"Cannot register {cls.__module__}.{cls.__qualname__} as {effective!r}: it " + f"is already registered as {built_as!r} in this process, because it was " + f"used before this declaration. Declare the name before the class is first " + f"used (for example with the @op.register_type decorator)." + ) + if name is not None: + _declared[cls] = name + remember(kind, effective, cls) + + +def reject_local_class(cls: type) -> None: + """Raise if ``cls`` is defined inside a function. + + Args: + cls: A class. + + Raises: + ValueError: If ``cls.__qualname__`` contains ````. + """ + if "" in cls.__qualname__: + raise ValueError( + f"Cannot register local class {cls!r}: a class defined inside a function " + f"cannot be found again when stored values are read. Define the class at " + f"module level." + ) + + +def mark_built(cls: type, name: str) -> None: + """Record that ``cls`` was built under ``name`` (one name per class per process).""" + _built[cls] = name + + +# --------------------------------------------------------------------------- +# Known classes per (kind, name) +# --------------------------------------------------------------------------- + + +def remember(kind: str, name: str, cls: type) -> None: + """Record ``cls`` as a class seen under ``(kind, name)``. + + Args: + kind: The class kind. + name: The name component. + cls: The class object. + """ + refs = _known.setdefault((kind, name), []) + refs[:] = [r for r in refs if r() is not None and r() is not cls] + refs.append(weakref.ref(cls)) + + +def known_classes(kind: str, name: str) -> list[type]: + """Return the live classes seen under ``(kind, name)``, most recent first.""" + classes = [r() for r in _known.get((kind, name), [])] + return [c for c in reversed(classes) if c is not None] + + +def candidate_classes(kind: str, name: str) -> list[type]: + """Return the classes that may be the class stored under ``(kind, name)``. + + In order: classes registered or declared under that name in this process, a class + reachable from ``__main__`` by that name, and a class importable by that dotted + name. Each candidate still has to match the stored identity. + + Args: + kind: The stored kind. + name: The stored name component. + + Returns: + Distinct candidate classes of the right kind. + """ + found: list[type] = list(known_classes(kind, name)) + for module_name in MAIN_MODULES: + module = sys.modules.get(module_name) + if module is not None: + obj = _walk_attributes(module, name.split(".")) + if obj is not None: + found.append(obj) + obj = _import_dotted(name) + if obj is not None: + found.append(obj) + result: list[type] = [] + for cls in found: + if class_kind(cls) == kind and cls not in result: + result.append(cls) + return result + + +def _walk_attributes(root: Any, parts: list[str]) -> Any: + obj = root + for part in parts: + obj = getattr(obj, part, None) + if obj is None: + return None + return obj + + +def _import_dotted(name: str) -> Any: + parts = name.split(".") + for split in range(len(parts) - 1, 0, -1): + module_name = ".".join(parts[:split]) + try: + module = importlib.import_module(module_name) + except ImportError: + continue + return _walk_attributes(module, parts[split:]) + return None + + +# --------------------------------------------------------------------------- +# Identity record and string +# --------------------------------------------------------------------------- + + +def identity_record(kind: str, name: str, fields: dict[str, str]) -> dict[str, Any]: + """Build the identity record of a user class. + + Args: + kind: ``"dataclass"`` or ``"pydantic"``. + name: The name component. + fields: Stored field name to type token. + + Returns: + ``{"kind": kind, "name": name, "fields": fields}`` with the fields sorted. + """ + return {"kind": kind, "name": name, "fields": dict(sorted(fields.items()))} + + +def canonical_json(obj: Any) -> str: + """Serialize ``obj`` as canonical JSON (sorted keys, no whitespace).""" + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def identity_string(record: dict[str, Any]) -> str: + """Return the identity string of an identity record. + + The format is ``f"{name}@{fingerprint}"``, where the fingerprint is the first + ``FINGERPRINT_HEX_DIGITS`` hex digits of the SHA-256 of ``canonical_json(record)``. + This is the only definition of the format. + + Args: + record: An identity record from ``identity_record``. + + Returns: + The identity string, e.g. ``"Point@9117beaaf7ebe3ab"``. + """ + digest = hashlib.sha256(canonical_json(record).encode("utf-8")).hexdigest() + return f"{record['name']}@{digest[:FINGERPRINT_HEX_DIGITS]}" + + +def describe_fields(fields: dict[str, str]) -> str: + """Render a field mapping for messages, e.g. ``{x: int, y: float}``.""" + return "{" + ", ".join(f"{k}: {v}" for k, v in sorted(fields.items())) + "}" + + +def describe_record(record: dict[str, Any]) -> str: + """Render an identity record for messages, e.g. ``Point (dataclass) {x: int}``.""" + return f"{record['name']} ({record['kind']}) {describe_fields(record['fields'])}" + + +# --------------------------------------------------------------------------- +# Type tokens +# --------------------------------------------------------------------------- + + +def type_token(annotation: Any, converter: TypeConverterProtocol) -> str: + """Return the type token of a field annotation. + + Builtins are written bare (``int``); classes with a logical type use its logical + type name (``orcapod.path``, or a nested user class's identity string); other + classes use their canonical name; generics are written ``origin[args]``; unions list + their members sorted, with ``None`` last. + + Args: + annotation: A field annotation (``Annotated`` extras already stripped). + converter: The type converter that resolves logical types. + + Returns: + The type token. + + Raises: + TypeError: If the annotation is not a type the token rules cover. + """ + import types as _types + + if annotation is None or annotation is type(None): + return "None" + origin = typing.get_origin(annotation) + args = typing.get_args(annotation) + if origin is typing.Union or origin is _types.UnionType: + members = [type_token(a, converter) for a in args] + non_none = sorted(m for m in members if m != "None") + return " | ".join(non_none + (["None"] if "None" in members else [])) + if origin is Literal: + return f"Literal[{', '.join(sorted(repr(a) for a in args))}]" + if origin is not None: + inner = ", ".join(type_token(a, converter) for a in args) + return f"{_class_token(origin)}[{inner}]" + if isinstance(annotation, type): + converter.register_python_class(annotation) + lt = converter.get_logical_type(annotation) + if lt is not None: + return lt.logical_type_name + return _class_token(annotation) + raise TypeError(f"Cannot build a type token for annotation {annotation!r}.") + + +def _class_token(cls: Any) -> str: + name = canonical_type_name(cls) + return name.removeprefix("builtins.") + + +# --------------------------------------------------------------------------- +# One-time notice for __main__ classes (S3) +# --------------------------------------------------------------------------- + + +def notice_main_class(cls: type, name: str) -> None: + """Warn once per name that a ``__main__`` class is identified by its name only. + + Args: + cls: The class being registered. + name: The name component it was given. + """ + if not is_main_class(cls) or cls in _declared or name in _noticed_main_names: + return + if name != cls.__qualname__: + return + _noticed_main_names.add(name) + warnings.warn( + f"{name} is defined in a notebook or script, so Orcapod identifies it by its " + f"name {name!r} and its fields. Results stored with it are found again by any " + f"class named {name!r} with the same fields, including one moved into a module " + f"and registered with @op.register_type(name={name!r}). Changing its fields " + f"makes a new type and recomputes.", + UserWarning, + stacklevel=2, + ) diff --git a/tests/test_user_types/__init__.py b/tests/test_user_types/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_user_types/test_identity_record.py b/tests/test_user_types/test_identity_record.py new file mode 100644 index 000000000..26f98ea0e --- /dev/null +++ b/tests/test_user_types/test_identity_record.py @@ -0,0 +1,143 @@ +"""Identity record, identity string, type tokens and naming policy of user classes (ITL-881). + +These tests target ``orcapod.logical_types.user_class_identity`` directly: they pin the +documented identity format, which is that module's own interface. +""" + +from __future__ import annotations + +import dataclasses +import datetime +import typing +from collections.abc import Iterator +from pathlib import Path +from typing import Literal, Optional + +import pytest + +from orcapod.contexts import get_default_context +from orcapod.logical_types import user_class_identity as uci + + +@pytest.fixture +def converter(): + return get_default_context().type_converter + + +@pytest.fixture(autouse=True) +def _restore_naming() -> Iterator[None]: + before = uci.get_naming() + yield + uci.set_naming(**before) + + +def _make_class(module: str, qualname: str = "Point") -> type: + cls = dataclasses.make_dataclass(qualname, [("x", int)]) + cls.__module__ = module + cls.__qualname__ = qualname + return cls + + +class TestIdentityString: + def test_documented_example(self) -> None: + record = uci.identity_record("dataclass", "Point", {"x": "int", "y": "float"}) + assert uci.identity_string(record) == "Point@9117beaaf7ebe3ab" + + def test_canonical_json_is_sorted_and_compact(self) -> None: + record = uci.identity_record("pydantic", "P", {"b": "int", "a": "str"}) + assert ( + uci.canonical_json(record) + == '{"fields":{"a":"str","b":"int"},"kind":"pydantic","name":"P"}' + ) + + def test_field_order_does_not_matter(self) -> None: + a = uci.identity_record("dataclass", "P", {"x": "int", "y": "str"}) + b = uci.identity_record("dataclass", "P", {"y": "str", "x": "int"}) + assert uci.identity_string(a) == uci.identity_string(b) + + @pytest.mark.parametrize( + "other", + [ + ("pydantic", "Point", {"x": "int", "y": "float"}), + ("dataclass", "Point2", {"x": "int", "y": "float"}), + ("dataclass", "Point", {"x": "int", "y": "int"}), + ("dataclass", "Point", {"x": "int", "z": "float"}), + ("dataclass", "Point", {"x": "int"}), + ], + ) + def test_each_component_changes_the_identity( + self, other: tuple[str, str, dict[str, str]] + ) -> None: + base = uci.identity_record("dataclass", "Point", {"x": "int", "y": "float"}) + assert uci.identity_string(base) != uci.identity_string( + uci.identity_record(*other) + ) + + def test_describe_fields(self) -> None: + assert uci.describe_fields({"y": "float", "x": "int"}) == "{x: int, y: float}" + + +class TestTypeTokens: + @pytest.mark.parametrize( + ("annotation", "token"), + [ + (int, "int"), + (str, "str"), + (bool, "bool"), + (bytes, "bytes"), + (float, "float"), + (datetime.datetime, "datetime.datetime"), + (Path, "orcapod.path"), + (int | None, "int | None"), + (Optional[int], "int | None"), + (None | int, "int | None"), + (list[int], "list[int]"), + (typing.List[int], "list[int]"), # noqa: UP006 - the typing alias is the case under test + (set[str], "set[str]"), + (dict[str, float], "dict[str, float]"), + (list[int | None], "list[int | None]"), + (Literal["b", "a"], "Literal['a', 'b']"), + ], + ) + def test_token(self, converter, annotation: object, token: str) -> None: + assert uci.type_token(annotation, converter) == token + + +class TestNames: + @pytest.mark.parametrize( + "name", ["", "a@b", "a b", "orcapod.thing", "tab\tname", 3] + ) + def test_invalid_names_rejected(self, name: object) -> None: + with pytest.raises((ValueError, TypeError)): + uci.validate_name(name) # pyright: ignore[reportArgumentType] + + @pytest.mark.parametrize("name", ["Point", "geo.Point", "my-types.Point_2"]) + def test_valid_names_accepted(self, name: str) -> None: + uci.validate_name(name) + + def test_module_class_defaults_to_qualified(self) -> None: + assert uci.resolve_name(_make_class("pkg.models")) == "pkg.models.Point" + + @pytest.mark.parametrize("module", ["__main__", "__mp_main__"]) + def test_main_class_defaults_to_name(self, module: str) -> None: + assert uci.resolve_name(_make_class(module)) == "Point" + + def test_nested_main_class_uses_qualname(self) -> None: + assert uci.resolve_name(_make_class("__main__", "Outer.Point")) == "Outer.Point" + + def test_main_qualified_setting(self) -> None: + uci.set_naming(main="qualified") + assert uci.resolve_name(_make_class("__main__")) == "__main__.Point" + + def test_module_name_setting(self) -> None: + uci.set_naming(module="name") + assert uci.resolve_name(_make_class("pkg.models")) == "Point" + + def test_invalid_setting_rejected(self) -> None: + with pytest.raises(ValueError, match="main"): + uci.set_naming(main="short") # pyright: ignore[reportArgumentType] + + def test_declared_name_wins(self) -> None: + cls = _make_class("pkg.models") + uci.declare(cls, "geo.Point") + assert uci.resolve_name(cls) == "geo.Point" From 5f12c0c151d438bbd5262007a7916fb70a045641 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:22:40 -0800 Subject: [PATCH 05/31] feat(types): identify dataclasses and pydantic models by name, kind and fields (ITL-881) The logical type name and Arrow extension name of a user class become its identity string, Name@, with the record in the extension metadata. Both factories share one write/read path; the read path finds the class by kind and name and checks the identity. The registry rebinds an unchanged identity to a re-defined class and no longer matches a user class's subclass to the parent's type. Golden values move as listed in the spec; regenerated in a later commit. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/errors.py | 9 + .../dataclass_logical_type_factory.py | 354 +---------------- .../pydantic_logical_type_factory.py | 368 ++---------------- src/orcapod/logical_types/registry.py | 66 +++- .../logical_types/user_class_identity.py | 24 +- .../logical_types/user_class_logical_type.py | 354 +++++++++++++++++ .../semantic_types/universal_converter.py | 8 +- .../test_logical_types/test_cache_behavior.py | 16 +- .../test_dataclass_logical_type_factory.py | 82 ++-- .../test_default_context_factories.py | 17 +- .../test_pydantic_logical_type_factory.py | 47 ++- tests/test_logical_types/test_roundtrips.py | 39 +- .../test_schema_compatibility.py | 17 +- tests/test_logical_types/test_type_utils.py | 12 - 14 files changed, 649 insertions(+), 764 deletions(-) create mode 100644 src/orcapod/logical_types/user_class_logical_type.py diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 94305e06a..aae207d0c 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -31,6 +31,15 @@ class TypeIdentityError(ValueError): """ +class RecursiveTypeError(TypeError): + """Raised when a class refers back to itself through its fields. + + Arrow cannot store a recursive struct, so a dataclass or pydantic model whose + fields reach the class again (``next: Node | None``) cannot be stored. The message + names the field path, e.g. ``Node.next -> Node``. + """ + + class DuplicateTagError(ValueError): """Raised when duplicate tag values are found and skip_duplicates=False""" diff --git a/src/orcapod/logical_types/dataclass_logical_type_factory.py b/src/orcapod/logical_types/dataclass_logical_type_factory.py index 16f268786..a73df8ccf 100644 --- a/src/orcapod/logical_types/dataclass_logical_type_factory.py +++ b/src/orcapod/logical_types/dataclass_logical_type_factory.py @@ -1,18 +1,8 @@ """DataclassLogicalType and DataclassLogicalTypeFactory. -Provides the ``DataclassLogicalType`` logical type implementation and the -``DataclassLogicalTypeFactory`` that synthesises and reconstructs ``DataclassLogicalType`` -instances for Python dataclasses. - -Write path (``create_for_python_type``): - Iterates dataclass fields, delegates field Arrow-type resolution to the converter - via ``register_python_class``, and returns a ``DataclassLogicalType`` backed by - a ``pa.struct`` extension type. - -Read path (``reconstruct_from_arrow``): - Imports the dataclass by fully-qualified class name, resolves field annotations - against the (already bottom-up resolved) storage type, and returns a - ``DataclassLogicalType``. +Dataclasses are user classes: their logical type name and Arrow extension name are the +class's identity string (name, kind ``"dataclass"`` and field types). The shared write +and read paths live in ``user_class_logical_type``. Category tag: ``"orcapod.dataclass"`` """ @@ -20,51 +10,23 @@ from __future__ import annotations import dataclasses -import json -import logging -from typing import TYPE_CHECKING, Any -from orcapod.logical_types.base_logical_type import BaseLogicalType -from orcapod.logical_types.registry import ( - make_arrow_extension_type, - make_polars_extension_type, +from orcapod.logical_types import user_class_identity as uci +from orcapod.logical_types.user_class_logical_type import ( + UserClassLogicalType, + UserClassLogicalTypeFactory, ) -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - from orcapod.logical_types.protocols import TypeConverterProtocol -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) #: Category tag embedded in Arrow extension metadata. Used as the factory dispatch key. DATACLASS_CATEGORY = "orcapod.dataclass" -class DataclassLogicalType(BaseLogicalType): - """Logical type binding a Python dataclass to its Arrow extension type representation. - - Stores the dataclass's fully-qualified class name as the Arrow extension name - and a ``pa.struct`` of the dataclass fields as the storage type. - - No Arrow-type reasoning lives here — all field-type resolution is owned by the - converter and completed before this object is constructed. - - Args: - logical_name: Fully-qualified class name (e.g. ``"mymodule.sub.MyData"``). - Used as both the logical type name and the Arrow extension name. - python_type: The Python dataclass ``type`` object. - storage_type: The Arrow ``pa.StructType`` for the dataclass fields. - field_annotations: Ordered list of ``(field_name, python_annotation)`` pairs - matching the fields in ``storage_type``. +class DataclassLogicalType(UserClassLogicalType): + """Logical type binding a dataclass to its Arrow extension type. Example: >>> lt = DataclassLogicalType( - ... "mymod.Point", Point, + ... "Point@c0a75f0c39b86873", Point, ... pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]), ... [("x", int), ("y", int)], ... ) @@ -72,133 +34,13 @@ class DataclassLogicalType(BaseLogicalType): {"x": 1, "y": 2} """ - def __init__( - self, - logical_name: str, - python_type: type, - storage_type: pa.StructType, - field_annotations: list[tuple[str, Any]], - ) -> None: - self._logical_name = logical_name - self._python_type = python_type - self._storage_type = storage_type - self._field_annotations = field_annotations - - _metadata = json.dumps({"category": DATACLASS_CATEGORY}).encode("utf-8") - self._arrow_ext_class = make_arrow_extension_type( - logical_name, storage_type, metadata=_metadata - ) - self._arrow_ext: pa.ExtensionType | None = None - # ``storage_type`` must not contain nested extension types (ET1 in DESIGN_ISSUES.md). - # On the write path, ``DataclassLogicalTypeFactory.create_for_python_type`` strips any - # top-level extension type from each field's Arrow type before inserting it into the - # struct. On the read path, ``reconstruct_from_arrow`` receives a ``storage_type`` - # already guaranteed storage-safe by ``register_storage_type``. - self._polars_ext_class = make_polars_extension_type( - logical_name, - storage_type, - metadata=json.dumps({"category": DATACLASS_CATEGORY}), - ) - self._polars_ext: pl.BaseExtension | None = None - - @property - def logical_type_name(self) -> str: - """Fully-qualified class name used as the logical type identifier.""" - return self._logical_name - - @property - def python_type(self) -> type: - """The Python dataclass type this logical type represents.""" - return self._python_type - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for this dataclass. - - Returns: - A cached ``pa.ExtensionType`` instance with ``extension_name`` equal to - the fully-qualified class name and ``storage_type`` equal to the struct - of the dataclass fields. - """ - if self._arrow_ext is None: - self._arrow_ext = self._arrow_ext_class() - return self._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for this dataclass. + category = DATACLASS_CATEGORY - Returns: - A cached ``pl.BaseExtension`` instance. - """ - if self._polars_ext is None: - self._polars_ext = self._polars_ext_class() - return self._polars_ext - def python_to_storage( - self, value: Any, converter: TypeConverterProtocol | None - ) -> dict[str, Any]: - """Convert a dataclass instance to an Arrow-compatible struct dict. +class DataclassLogicalTypeFactory(UserClassLogicalTypeFactory): + """Factory for dataclass logical types. - Iterates ``_field_annotations`` and delegates each field's conversion to - ``converter.python_to_storage``. - - Args: - value: A dataclass instance of type ``python_type``. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A dict mapping field names to their Arrow storage values. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "DataclassLogicalType.python_to_storage requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - return { - name: converter.python_to_storage(getattr(value, name), annotation) - for name, annotation in self._field_annotations - } - - def storage_to_python( - self, storage_value: Any, converter: TypeConverterProtocol | None - ) -> Any: - """Reconstruct a dataclass instance from an Arrow struct dict. - - Args: - storage_value: A dict mapping field names to Arrow storage values. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A dataclass instance of type ``python_type``. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "DataclassLogicalType.storage_to_python requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - kwargs = { - name: converter.storage_to_python(storage_value[name], annotation) - for name, annotation in self._field_annotations - } - return self._python_type(**kwargs) - - -class DataclassLogicalTypeFactory: - """Stateless factory that synthesises and reconstructs ``DataclassLogicalType`` instances. - - **Write path** (``create_for_python_type``): derives Arrow struct type from the - dataclass fields by delegating to ``converter.register_python_class`` per field. - - **Read path** (``reconstruct_from_arrow``): imports the dataclass by FQCN, matches - fields against the already-resolved ``storage_type``, and returns a - ``DataclassLogicalType``. - - Category tag: ``"orcapod.dataclass"`` + Stored fields are the dataclass fields with ``init=True``. Register with:: @@ -207,168 +49,12 @@ class DataclassLogicalTypeFactory: category="orcapod.dataclass", python_bases=[object], ) - - Example: - >>> factory = DataclassLogicalTypeFactory() - >>> factory.supports_class(MyDataclass) - True - >>> factory.supports_class(str) - False """ - def supports_class(self, python_type: type) -> bool: - """Return True if ``python_type`` is a dataclass. - - Args: - python_type: Any Python type. - - Returns: - True if ``dataclasses.is_dataclass(python_type)`` is True. - """ - return dataclasses.is_dataclass(python_type) and isinstance(python_type, type) - - def create_for_python_type( - self, - python_type: type, - converter: TypeConverterProtocol, - ) -> DataclassLogicalType: - """Synthesise a ``DataclassLogicalType`` for a Python dataclass (write path). - - Derives the FQCN, obtains type hints, and resolves each field's Arrow type - via ``converter.register_python_class``. Rejects local / unnamed classes. - - Args: - python_type: A Python dataclass type. - converter: The active converter for field-type resolution. - - Returns: - A ``DataclassLogicalType`` ready for registration. - - Raises: - ValueError: If ``python_type`` is a local class (``__qualname__`` contains - ``""``). - """ - import typing - - fqcn = f"{python_type.__module__}.{python_type.__qualname__}" - if "" in fqcn: - raise ValueError( - f"Cannot register local class {python_type!r} as a DataclassLogicalType — " - f"local classes have no stable fully-qualified class name and cannot be " - f"reconstructed on read. Define the dataclass at module level." - ) - - try: - hints = typing.get_type_hints(python_type) - except Exception as exc: - raise ValueError( - f"Cannot get type hints for {python_type!r}: {exc}" - ) from exc - - arrow_fields = [] - field_annotations = [] - for field in dataclasses.fields(python_type): - if not field.init: - continue - annotation = hints.get(field.name, Any) - arrow_type = converter.register_python_class(annotation) - # register_python_class returns a storage-safe type: may be extension at the - # top level, but struct fields are always plain. Strip the top-level extension - # type here before inserting into the struct (ET1; see DESIGN_ISSUES.md). - if isinstance(arrow_type, pa.ExtensionType): - arrow_type = arrow_type.storage_type - arrow_fields.append(pa.field(field.name, arrow_type)) - field_annotations.append((field.name, annotation)) - - storage_type = pa.struct(arrow_fields) - logger.debug( - "DataclassLogicalTypeFactory: synthesised %r for %r", fqcn, python_type - ) - return DataclassLogicalType(fqcn, python_type, storage_type, field_annotations) - - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: pa.DataType, - metadata: dict[str, Any], - converter: TypeConverterProtocol, - ) -> DataclassLogicalType: - """Reconstruct a ``DataclassLogicalType`` from Arrow schema metadata (read path). - - Imports the dataclass from its FQCN (``arrow_extension_name``), then matches - the dataclass field annotations against the fields in ``storage_type``. - ``storage_type`` is already bottom-up resolved by ``register_storage_type`` - before this method is called. - - Args: - arrow_extension_name: FQCN of the dataclass (Arrow extension name). - storage_type: Already-resolved ``pa.StructType`` for the dataclass fields. - metadata: Full parsed metadata JSON dict (always contains ``"category"``). - converter: The active converter (not needed here but required by protocol). - - Returns: - A ``DataclassLogicalType`` ready for registration. - - Raises: - ImportError: If the class cannot be imported from ``arrow_extension_name``. - ValueError: If ``storage_type`` is not a struct type. - """ - import typing - - if not pa.types.is_struct(storage_type): - raise ValueError( - f"DataclassLogicalTypeFactory.reconstruct_from_arrow: expected a struct " - f"storage type for {arrow_extension_name!r}, got {storage_type!r}." - ) - - # Import class from FQCN using longest-prefix module walk - cls = _import_from_fqcn(arrow_extension_name) - - try: - hints = typing.get_type_hints(cls) - except Exception as exc: - raise ValueError(f"Cannot get type hints for {cls!r}: {exc}") from exc - - field_annotations = [] - for field in dataclasses.fields(cls): - if not field.init: - continue - annotation = hints.get(field.name, Any) - # Register any logical type the field annotation maps to (registration - # completeness invariant: all nested logical types must be registered when - # the outer type is registered). The return value is discarded; only the - # side effect of registration matters here. - converter.register_python_class(annotation) - field_annotations.append((field.name, annotation)) - - logger.debug( - "DataclassLogicalTypeFactory: reconstructed %r from Arrow", - arrow_extension_name, - ) - return DataclassLogicalType( - arrow_extension_name, cls, storage_type, field_annotations - ) - - -def _import_from_fqcn(fqcn: str) -> type: - """Import a dataclass from its fully-qualified class name. - - Delegates the module-prefix walk to ``type_utils._walk_fqcn``, then - validates the resolved object is a dataclass type. - - Args: - fqcn: Fully-qualified class name, e.g. ``"mypackage.sub.MyClass"``. - - Returns: - The imported dataclass type. - - Raises: - ImportError: If no valid module+attribute split can be found, or if the - resolved object is not a dataclass type. - """ - from orcapod.logical_types.type_utils import _walk_fqcn + kind = uci.KIND_DATACLASS + category = DATACLASS_CATEGORY + logical_type_class = DataclassLogicalType - obj: Any = _walk_fqcn(fqcn) - if not dataclasses.is_dataclass(obj) or not isinstance(obj, type): - raise ImportError(f"{fqcn!r} does not resolve to a dataclass type.") - return obj + def stored_field_names(self, python_type: type) -> list[str]: + """Return the names of the ``init=True`` dataclass fields, in class order.""" + return [f.name for f in dataclasses.fields(python_type) if f.init] diff --git a/src/orcapod/logical_types/pydantic_logical_type_factory.py b/src/orcapod/logical_types/pydantic_logical_type_factory.py index e3f37f090..d714b961a 100644 --- a/src/orcapod/logical_types/pydantic_logical_type_factory.py +++ b/src/orcapod/logical_types/pydantic_logical_type_factory.py @@ -1,379 +1,55 @@ """PydanticLogicalType and PydanticLogicalTypeFactory. -Provides the ``PydanticLogicalType`` logical type implementation and the -``PydanticLogicalTypeFactory`` that synthesises and reconstructs -``PydanticLogicalType`` instances for pydantic v2 ``BaseModel`` subclasses. - -Write path (``create_for_python_type``): - Iterates model fields via ``model_fields`` (pydantic v2 API), delegates - field Arrow-type resolution to the converter via ``register_python_class``, - and returns a ``PydanticLogicalType`` backed by a ``pa.struct`` extension - type. - -Read path (``reconstruct_from_arrow``): - Imports the model by fully-qualified class name, resolves field annotations - against the (already bottom-up resolved) storage type, and returns a - ``PydanticLogicalType``. +Pydantic v2 models are user classes: their logical type name and Arrow extension name +are the class's identity string (name, kind ``"pydantic"`` and field types). A pydantic +model and a dataclass with the same name and fields are different types. The shared +write and read paths live in ``user_class_logical_type``. Category tag: ``"orcapod.pydantic"`` """ from __future__ import annotations -import json -import logging -from typing import TYPE_CHECKING, Any - -from orcapod.logical_types.base_logical_type import BaseLogicalType -from orcapod.logical_types.registry import ( - make_arrow_extension_type, - make_polars_extension_type, +from orcapod.logical_types import user_class_identity as uci +from orcapod.logical_types.user_class_logical_type import ( + UserClassLogicalType, + UserClassLogicalTypeFactory, ) -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - from orcapod.logical_types.protocols import TypeConverterProtocol -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) #: Category tag embedded in Arrow extension metadata. Used as the factory dispatch key. PYDANTIC_CATEGORY = "orcapod.pydantic" -class PydanticLogicalType(BaseLogicalType): +class PydanticLogicalType(UserClassLogicalType): """Logical type binding a pydantic ``BaseModel`` subclass to its Arrow extension type. - Stores the model's fully-qualified class name as the Arrow extension name - and a ``pa.struct`` of the model fields as the storage type. - - No Arrow-type reasoning lives here — all field-type resolution is owned by - the converter and completed before this object is constructed. - - Args: - logical_name: Fully-qualified class name (e.g. ``"mymodule.sub.MyModel"``). - Used as both the logical type name and the Arrow extension name. - python_type: The pydantic ``BaseModel`` subclass. - storage_type: The Arrow ``pa.StructType`` for the model fields. - field_annotations: Ordered list of ``(field_name, python_annotation)`` - pairs matching the fields in ``storage_type``. - - Example: - >>> lt = PydanticLogicalType( - ... "mymod.Point", Point, - ... pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]), - ... [("x", int), ("y", int)], - ... ) - >>> lt.python_to_storage(Point(x=1, y=2), converter) - {"x": 1, "y": 2} + ``storage_to_python`` constructs the model, so pydantic validation runs on read. """ - def __init__( - self, - logical_name: str, - python_type: type, - storage_type: pa.StructType, - field_annotations: list[tuple[str, Any]], - ) -> None: - self._logical_name = logical_name - self._python_type = python_type - self._storage_type = storage_type - self._field_annotations = field_annotations - - _metadata = json.dumps({"category": PYDANTIC_CATEGORY}).encode("utf-8") - self._arrow_ext_class = make_arrow_extension_type( - logical_name, storage_type, metadata=_metadata - ) - self._arrow_ext: pa.ExtensionType | None = None - # ``storage_type`` must not contain nested extension types (ET1 in DESIGN_ISSUES.md). - # On the write path, ``PydanticLogicalTypeFactory.create_for_python_type`` strips any - # top-level extension type from each field's Arrow type before inserting it into the - # struct. On the read path, ``reconstruct_from_arrow`` receives a ``storage_type`` - # already guaranteed storage-safe by ``register_storage_type``. - self._polars_ext_class = make_polars_extension_type( - logical_name, - storage_type, - metadata=json.dumps({"category": PYDANTIC_CATEGORY}), - ) - self._polars_ext: pl.BaseExtension | None = None - - @property - def logical_type_name(self) -> str: - """Fully-qualified class name used as the logical type identifier.""" - return self._logical_name - - @property - def python_type(self) -> type: - """The pydantic ``BaseModel`` subclass this logical type represents.""" - return self._python_type - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for this model. - - Returns: - A cached ``pa.ExtensionType`` instance with ``extension_name`` equal to - the fully-qualified class name and ``storage_type`` equal to the struct - of the model fields. - """ - if self._arrow_ext is None: - self._arrow_ext = self._arrow_ext_class() - return self._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for this model. + category = PYDANTIC_CATEGORY - Returns: - A cached ``pl.BaseExtension`` instance. - """ - if self._polars_ext is None: - self._polars_ext = self._polars_ext_class() - return self._polars_ext - def python_to_storage( - self, value: Any, converter: TypeConverterProtocol | None - ) -> dict[str, Any]: - """Convert a pydantic model instance to an Arrow-compatible struct dict. +class PydanticLogicalTypeFactory(UserClassLogicalTypeFactory): + """Factory for pydantic model logical types. - Iterates ``_field_annotations`` and delegates each field's conversion to - ``converter.python_to_storage``. - - Args: - value: A pydantic model instance of type ``python_type``. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A dict mapping field names to their Arrow storage values. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "PydanticLogicalType.python_to_storage requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - return { - name: converter.python_to_storage(getattr(value, name), annotation) - for name, annotation in self._field_annotations - } - - def storage_to_python( - self, storage_value: Any, converter: TypeConverterProtocol | None - ) -> Any: - """Reconstruct a pydantic model instance from an Arrow struct dict. - - Args: - storage_value: A dict mapping field names to Arrow storage values. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A pydantic model instance of type ``python_type``. Pydantic validation - runs on construction, ensuring the model is always in a valid state. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "PydanticLogicalType.storage_to_python requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - kwargs = { - name: converter.storage_to_python(storage_value[name], annotation) - for name, annotation in self._field_annotations - } - return self._python_type(**kwargs) - - -class PydanticLogicalTypeFactory: - """Stateless factory that synthesises and reconstructs ``PydanticLogicalType`` instances. - - **Write path** (``create_for_python_type``): derives Arrow struct type from the - model fields by delegating to ``converter.register_python_class`` per field. - Only fields in ``model_fields`` are stored — computed fields and private - attributes are excluded. - - **Read path** (``reconstruct_from_arrow``): imports the model by FQCN, matches - fields against the already-resolved ``storage_type``, and returns a - ``PydanticLogicalType``. - - Category tag: ``"orcapod.pydantic"`` + Stored fields are the model's ``model_fields``; computed fields and private + attributes are not stored. Register with:: from pydantic import BaseModel + converter.register_logical_type_factory( PydanticLogicalTypeFactory(), category="orcapod.pydantic", python_bases=[BaseModel], ) - - Example: - >>> factory = PydanticLogicalTypeFactory() - >>> factory.supports_class(MyModel) - True - >>> factory.supports_class(str) - False """ - def supports_class(self, python_type: type) -> bool: - """Return True if ``python_type`` is a pydantic ``BaseModel`` subclass. - - Args: - python_type: Any Python type. - - Returns: - True if ``python_type`` is a ``BaseModel`` subclass. - """ - from pydantic import BaseModel - - return isinstance(python_type, type) and issubclass(python_type, BaseModel) - - def create_for_python_type( - self, - python_type: type, - converter: TypeConverterProtocol, - ) -> PydanticLogicalType: - """Synthesise a ``PydanticLogicalType`` for a pydantic model (write path). - - Derives the FQCN, obtains type hints, and resolves each field's Arrow type - via ``converter.register_python_class``. Only fields present in - ``model_fields`` are stored — computed fields and private attributes are - excluded. Rejects local / unnamed classes. - - Args: - python_type: A pydantic ``BaseModel`` subclass. - converter: The active converter for field-type resolution. - - Returns: - A ``PydanticLogicalType`` ready for registration. - - Raises: - ValueError: If ``python_type`` is a local class (``__qualname__`` contains - ``""``). - """ - import typing - - fqcn = f"{python_type.__module__}.{python_type.__qualname__}" - if "" in fqcn: - raise ValueError( - f"Cannot register local class {python_type!r} as a PydanticLogicalType — " - f"local classes have no stable fully-qualified class name and cannot be " - f"reconstructed on read. Define the model at module level." - ) - - try: - hints = typing.get_type_hints(python_type) - except Exception as exc: - raise ValueError( - f"Cannot get type hints for {python_type!r}: {exc}" - ) from exc - - arrow_fields = [] - field_annotations = [] - for field_name in python_type.model_fields: - annotation = hints.get(field_name, Any) - arrow_type = converter.register_python_class(annotation) - # Strip top-level extension type before inserting into the struct (ET1; - # see DESIGN_ISSUES.md): Arrow cannot represent extension types inside - # struct field types. - if isinstance(arrow_type, pa.ExtensionType): - arrow_type = arrow_type.storage_type - arrow_fields.append(pa.field(field_name, arrow_type)) - field_annotations.append((field_name, annotation)) - - storage_type = pa.struct(arrow_fields) - logger.debug( - "PydanticLogicalTypeFactory: synthesised %r for %r", fqcn, python_type - ) - return PydanticLogicalType(fqcn, python_type, storage_type, field_annotations) - - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: pa.DataType, - metadata: dict[str, Any], - converter: TypeConverterProtocol, - ) -> PydanticLogicalType: - """Reconstruct a ``PydanticLogicalType`` from Arrow schema metadata (read path). - - Imports the model from its FQCN (``arrow_extension_name``), then matches - the model field annotations against the fields in ``storage_type``. - ``storage_type`` is already bottom-up resolved by ``register_storage_type`` - before this method is called. - - Args: - arrow_extension_name: FQCN of the pydantic model (Arrow extension name). - storage_type: Already-resolved ``pa.StructType`` for the model fields. - metadata: Full parsed metadata JSON dict (always contains ``"category"``). - converter: The active converter (used for registration completeness invariant). - - Returns: - A ``PydanticLogicalType`` ready for registration. + kind = uci.KIND_PYDANTIC + category = PYDANTIC_CATEGORY + logical_type_class = PydanticLogicalType - Raises: - ImportError: If the class cannot be imported from ``arrow_extension_name``. - ValueError: If ``storage_type`` is not a struct type. - """ - import typing - - if not pa.types.is_struct(storage_type): - raise ValueError( - f"PydanticLogicalTypeFactory.reconstruct_from_arrow: expected a struct " - f"storage type for {arrow_extension_name!r}, got {storage_type!r}." - ) - - cls = _import_pydantic_model_from_fqcn(arrow_extension_name) - - try: - hints = typing.get_type_hints(cls) - except Exception as exc: - raise ValueError(f"Cannot get type hints for {cls!r}: {exc}") from exc - - field_annotations = [] - for field_name in cls.model_fields: - annotation = hints.get(field_name, Any) - # Register any logical type the field annotation maps to (registration - # completeness invariant: all nested logical types must be registered when - # the outer type is registered). The return value is discarded. - converter.register_python_class(annotation) - field_annotations.append((field_name, annotation)) - - logger.debug( - "PydanticLogicalTypeFactory: reconstructed %r from Arrow", - arrow_extension_name, - ) - return PydanticLogicalType( - arrow_extension_name, cls, storage_type, field_annotations - ) - - -def _import_pydantic_model_from_fqcn(fqcn: str) -> type: - """Import a pydantic ``BaseModel`` subclass from its fully-qualified class name. - - Delegates the module-prefix walk to ``type_utils._walk_fqcn``, then - validates the resolved object is a ``BaseModel`` subclass. - - Args: - fqcn: Fully-qualified class name, e.g. ``"mypackage.sub.MyModel"``. - - Returns: - The imported ``BaseModel`` subclass. - - Raises: - ImportError: If no valid module+attribute split can be found, or if the - resolved object is not a ``BaseModel`` subclass. - """ - from pydantic import BaseModel - from orcapod.logical_types.type_utils import _walk_fqcn - - obj: Any = _walk_fqcn(fqcn) - if not (isinstance(obj, type) and issubclass(obj, BaseModel)): - raise ImportError( - f"{fqcn!r} does not resolve to a pydantic BaseModel subclass." - ) - return obj + def stored_field_names(self, python_type: type) -> list[str]: + """Return the names in ``model_fields``, in class order.""" + return list(python_type.model_fields) # pyright: ignore[reportAttributeAccessIssue] # supports_class guarantees a BaseModel subclass diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index a33790422..e3729f647 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -8,6 +8,7 @@ import logging import re +import warnings from typing import TYPE_CHECKING, Iterable from orcapod.logical_types.protocols import ( @@ -128,13 +129,15 @@ def _deserialize( else: # Non-canonical _storage — require exact match. storage_ok = storage_type == _storage + from orcapod.errors import TypeIdentityError + if not storage_ok: - raise ValueError( + raise TypeIdentityError( f"Arrow extension type '{_name}': expected storage_type " f"{_storage!r} but got {storage_type!r}." ) if serialized != _metadata: - raise ValueError( + raise TypeIdentityError( f"Arrow extension type '{_name}': expected metadata " f"{_metadata!r} but got {serialized!r}." ) @@ -219,6 +222,15 @@ def ext_from_params( ) +def _same_arrow_extension(a: pa.ExtensionType, b: pa.ExtensionType) -> bool: + """Return True if two extension types have the same name, storage and metadata.""" + return ( + a.extension_name == b.extension_name + and a.storage_type == b.storage_type + and a.__arrow_ext_serialize__() == b.__arrow_ext_serialize__() + ) + + class LogicalTypeRegistry: """Registry for ``LogicalType`` instances. @@ -294,6 +306,19 @@ def register_logical_type(self, logical_type: LogicalTypeProtocol) -> None: existing_by_arrow = self._by_arrow_name.get(arrow_ext_name) existing_by_python = self._by_python_type.get(py_type) + if ( + existing_by_python is None + and existing_by_logical is not None + and existing_by_logical is existing_by_arrow + and existing_by_logical is not logical_type + and getattr(logical_type, "exact_python_type_only", False) + and _same_arrow_extension( + existing_by_logical.get_arrow_extension_type(), arrow_ext + ) + ): + self._rebind(existing_by_logical, logical_type) + return + # Triplet conflict check: raise if any key is bound to a different instance. for existing, label, key in [ (existing_by_logical, "logical_type_name", logical_name), @@ -344,6 +369,41 @@ def register_logical_type(self, logical_type: LogicalTypeProtocol) -> None: self._by_arrow_name[arrow_ext_name] = logical_type self._by_python_type[py_type] = logical_type + def _rebind( + self, previous: LogicalTypeProtocol, logical_type: LogicalTypeProtocol + ) -> None: + """Rebind an identity to a logical type built for a new class object. + + Used when a user class with an unchanged identity is defined again (a notebook + cell re-run, a module reload). The logical-name and Arrow-name bindings move to + the new logical type; the previous class object keeps its binding, so instances + created before the re-definition still convert. Arrow's and Polars' global + registrations are reused: name, storage type and metadata are identical. + + Args: + previous: The logical type currently bound to the identity. + logical_type: The logical type for the new class object. + """ + old_cls = previous.python_type + new_cls = logical_type.python_type + old_path = f"{old_cls.__module__}.{old_cls.__qualname__}" + new_path = f"{new_cls.__module__}.{new_cls.__qualname__}" + if old_path != new_path: + warnings.warn( + f"{new_path} has the same identity as {old_path} " + f"({logical_type.logical_type_name}: same name, kind and fields), so " + f"Orcapod treats them as one type. Stored values of this type are now " + f"read back as {new_path}.", + UserWarning, + stacklevel=4, + ) + name = logical_type.logical_type_name + self._by_logical_name[name] = logical_type + self._by_arrow_name[logical_type.get_arrow_extension_type().extension_name] = ( + logical_type + ) + self._by_python_type[new_cls] = logical_type + def get_by_logical_name(self, name: str) -> LogicalTypeProtocol | None: """Return the logical type registered under *name*, or ``None``.""" return self._by_logical_name.get(name) @@ -359,6 +419,8 @@ def get_by_python_type(self, python_type: type) -> LogicalTypeProtocol | None: if result is not None: return result for registered_type, registered_lt in self._by_python_type.items(): + if getattr(registered_lt, "exact_python_type_only", False): + continue try: if issubclass(python_type, registered_type): return registered_lt diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index 5f95454ee..ae8b91f3a 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -22,7 +22,6 @@ import dataclasses import hashlib -import importlib import json import re import sys @@ -145,8 +144,7 @@ def set_naming( continue if value not in _NAMING_STYLES: raise ValueError( - f"Invalid naming style for {key}: {value!r}. " - f"Use 'name' or 'qualified'." + f"Invalid naming style for {key}: {value!r}. Use 'name' or 'qualified'." ) _naming[key] = value @@ -234,6 +232,11 @@ def reject_local_class(cls: type) -> None: ) +def built_name(cls: type) -> str | None: + """Return the name ``cls`` was built under in this process, or ``None``.""" + return _built.get(cls) + + def mark_built(cls: type, name: str) -> None: """Record that ``cls`` was built under ``name`` (one name per class per process).""" _built[cls] = name @@ -304,15 +307,12 @@ def _walk_attributes(root: Any, parts: list[str]) -> Any: def _import_dotted(name: str) -> Any: - parts = name.split(".") - for split in range(len(parts) - 1, 0, -1): - module_name = ".".join(parts[:split]) - try: - module = importlib.import_module(module_name) - except ImportError: - continue - return _walk_attributes(module, parts[split:]) - return None + from orcapod.logical_types.type_utils import _walk_fqcn + + try: + return _walk_fqcn(name) + except ImportError: + return None # --------------------------------------------------------------------------- diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py new file mode 100644 index 000000000..f7e4bcd10 --- /dev/null +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -0,0 +1,354 @@ +"""Shared logical type and factory for user classes (dataclasses and pydantic models). + +A user class's logical type name and Arrow extension name are its *identity string* +(``user_class_identity.identity_string``): the name, kind and field types, so two +classes with the same name and different fields are different types with different +tables (ITL-881). The Arrow extension metadata carries the full identity record, which +the read path uses to find the class and to describe a mismatch. + +Write path (``create_for_python_type``): + Resolves the name (declared name or naming policy), resolves each stored field's + Arrow type and type token through the converter, and builds the identity. + +Read path (``reconstruct_from_arrow``): + Reads the identity record from the metadata, finds candidate classes by kind and + name, and accepts the one whose identity string equals the stored extension name. +""" + +from __future__ import annotations + +import logging +import typing +from typing import TYPE_CHECKING, Any + +from orcapod.errors import RecursiveTypeError, TypeIdentityError +from orcapod.logical_types import user_class_identity as uci +from orcapod.logical_types.base_logical_type import BaseLogicalType +from orcapod.logical_types.registry import ( + make_arrow_extension_type, + make_polars_extension_type, +) +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import polars as pl + import pyarrow as pa + + from orcapod.logical_types.protocols import TypeConverterProtocol +else: + pa = LazyModule("pyarrow") + pl = LazyModule("polars") + +logger = logging.getLogger(__name__) + + +class UserClassLogicalType(BaseLogicalType): + """Logical type binding a user class to its Arrow extension type. + + The logical type name and Arrow extension name are the class's identity string; + the storage type is a ``pa.struct`` of the stored fields. Subclasses set + ``category``. + + Args: + logical_name: The identity string (e.g. ``"Point@9117beaaf7ebe3ab"``). + python_type: The class. + storage_type: The Arrow ``pa.StructType`` of the stored fields. + field_annotations: ``(field_name, python_annotation)`` pairs matching the + fields in ``storage_type``. + identity_record: The identity record, embedded in the Arrow extension metadata. + ``None`` embeds only the category (for low-level tests). + """ + + category: str = "" + + #: A subclass of the class is a different class with its own identity, so the + #: registry must not match it to this logical type by ``issubclass``. + exact_python_type_only: bool = True + + def __init__( + self, + logical_name: str, + python_type: type, + storage_type: pa.StructType, + field_annotations: list[tuple[str, Any]], + identity_record: dict[str, Any] | None = None, + ) -> None: + self._logical_name = logical_name + self._python_type = python_type + self._storage_type = storage_type + self._field_annotations = field_annotations + self._identity_record = identity_record + + metadata: dict[str, Any] = {"category": self.category} + if identity_record is not None: + metadata["identity"] = identity_record + metadata_json = uci.canonical_json(metadata) + # ``storage_type`` never contains nested extension types (ET1 in DESIGN_ISSUES.md): + # the factory strips each field's top-level extension type before building it. + self._arrow_ext_class = make_arrow_extension_type( + logical_name, storage_type, metadata=metadata_json.encode("utf-8") + ) + self._arrow_ext: pa.ExtensionType | None = None + self._polars_ext_class = make_polars_extension_type( + logical_name, storage_type, metadata=metadata_json + ) + self._polars_ext: pl.BaseExtension | None = None + + @property + def logical_type_name(self) -> str: + """The class's identity string.""" + return self._logical_name + + @property + def python_type(self) -> type: + """The class this logical type represents.""" + return self._python_type + + @property + def identity_record(self) -> dict[str, Any] | None: + """The identity record, or ``None`` for a logical type built without one.""" + return self._identity_record + + @property + def storage_type(self) -> pa.StructType: + """The Arrow struct type of the stored fields.""" + return self._storage_type + + def get_arrow_extension_type(self) -> pa.ExtensionType: + """Return the cached Arrow extension type (name = identity string).""" + if self._arrow_ext is None: + self._arrow_ext = self._arrow_ext_class() + return self._arrow_ext + + def get_polars_extension_type(self) -> pl.BaseExtension: + """Return the cached Polars extension type.""" + if self._polars_ext is None: + self._polars_ext = self._polars_ext_class() + return self._polars_ext + + def python_to_storage( + self, value: Any, converter: TypeConverterProtocol | None + ) -> dict[str, Any]: + """Convert an instance to an Arrow-compatible struct dict. + + Args: + value: An instance of ``python_type``. + converter: The active converter for per-field conversion. + + Returns: + A dict mapping field names to their Arrow storage values. + + Raises: + ValueError: If ``converter`` is ``None``. + """ + if converter is None: + raise ValueError( + f"{type(self).__name__}.python_to_storage requires a converter — " + "pass a TypeConverterProtocol instance for field-level conversion." + ) + return { + name: converter.python_to_storage(getattr(value, name), annotation) + for name, annotation in self._field_annotations + } + + def storage_to_python( + self, storage_value: Any, converter: TypeConverterProtocol | None + ) -> Any: + """Build an instance from an Arrow struct dict. + + Args: + storage_value: A dict mapping field names to Arrow storage values. + converter: The active converter for per-field conversion. + + Returns: + An instance of ``python_type``. + + Raises: + ValueError: If ``converter`` is ``None``. + """ + if converter is None: + raise ValueError( + f"{type(self).__name__}.storage_to_python requires a converter — " + "pass a TypeConverterProtocol instance for field-level conversion." + ) + kwargs = { + name: converter.storage_to_python(storage_value[name], annotation) + for name, annotation in self._field_annotations + } + return self._python_type(**kwargs) + + +class UserClassLogicalTypeFactory: + """Base factory for user-class logical types. + + Subclasses set ``kind``, ``category`` and ``logical_type_class`` and implement + ``supports_class`` and ``stored_field_names``. + """ + + kind: str = "" + category: str = "" + logical_type_class: type[UserClassLogicalType] = UserClassLogicalType + + def supports_class(self, python_type: type) -> bool: + """Return True if this factory handles ``python_type``.""" + return uci.class_kind(python_type) == self.kind + + def stored_field_names(self, python_type: type) -> list[str]: + """Return the names of the fields stored for ``python_type``, in class order.""" + raise NotImplementedError + + # ------------------------------------------------------------------ + # Write path + # ------------------------------------------------------------------ + + def create_for_python_type( + self, + python_type: type, + converter: TypeConverterProtocol, + ) -> UserClassLogicalType: + """Build the logical type of a user class (write path). + + Args: + python_type: The class. + converter: The active converter for field-type resolution. + + Returns: + A logical type whose name is the class's identity string. + + Raises: + ValueError: If the class is local or its type hints cannot be resolved. + TypeError: If the class refers back to itself (directly or through other + classes), naming the field path. + """ + uci.reject_local_class(python_type) + name = uci.built_name(python_type) or uci.resolve_name(python_type) + hints = _type_hints(python_type) + + arrow_fields = [] + field_annotations: list[tuple[str, Any]] = [] + tokens: dict[str, str] = {} + for field_name in self.stored_field_names(python_type): + annotation = hints.get(field_name, Any) + try: + arrow_type = converter.register_python_class(annotation) + except TypeError as exc: + if isinstance(exc, RecursiveTypeError): + raise RecursiveTypeError( + f"{python_type.__qualname__}.{field_name} -> {exc}" + ) from exc + raise + # Struct fields cannot hold extension types (ET1; see DESIGN_ISSUES.md). + if isinstance(arrow_type, pa.ExtensionType): + arrow_type = arrow_type.storage_type + arrow_fields.append(pa.field(field_name, arrow_type)) + field_annotations.append((field_name, annotation)) + tokens[field_name] = uci.type_token(annotation, converter) + + record = uci.identity_record(self.kind, name, tokens) + identity = uci.identity_string(record) + uci.mark_built(python_type, name) + uci.remember(self.kind, name, python_type) + uci.notice_main_class(python_type, name) + logger.debug("%s: built %r for %r", type(self).__name__, identity, python_type) + return self.logical_type_class( + identity, python_type, pa.struct(arrow_fields), field_annotations, record + ) + + # ------------------------------------------------------------------ + # Read path + # ------------------------------------------------------------------ + + def reconstruct_from_arrow( + self, + arrow_extension_name: str, + storage_type: pa.DataType, + metadata: dict[str, Any], + converter: TypeConverterProtocol, + ) -> UserClassLogicalType: + """Find the class of a stored column and return its logical type (read path). + + Args: + arrow_extension_name: The stored identity string. + storage_type: The stored struct type (already resolved). + metadata: The parsed extension metadata, with ``category`` and ``identity``. + converter: The active converter. + + Returns: + The logical type of the class whose identity matches the stored one. + + Raises: + ImportError: If no class with the stored kind and name can be found. + TypeIdentityError: If the column has no identity record (written before + Orcapod 0.3), or the classes found have a different identity or a + different storage type. + """ + record = metadata.get("identity") + if not isinstance(record, dict): + raise TypeIdentityError( + f"The stored {self.kind} column type {arrow_extension_name!r} was written " + f"by an Orcapod version before 0.3, which identified classes by module " + f"and qualified name only. Orcapod 0.3 identifies a class by its name, " + f"kind and fields and does not read such columns; re-running the " + f"pipeline computes the results into new tables." + ) + name = record.get("name") + kind = record.get("kind") + if kind != self.kind or not isinstance(name, str): + raise TypeIdentityError( + f"Stored column type {arrow_extension_name!r} has an invalid identity " + f"record for category {self.category!r}: {record!r}." + ) + + candidates = uci.candidate_classes(kind, name) + if not candidates: + raise ImportError( + f"Cannot find the {kind} {name!r} to read stored values of type " + f"{uci.describe_record(record)}. Define the class and register it " + f"under that name before reading, for example " + f"op.register_type({name.rsplit('.', 1)[-1]}, name={name!r})." + ) + + found: list[UserClassLogicalType] = [] + for cls in candidates: + converter.register_python_class(cls) + lt = converter.get_logical_type(cls) + if not isinstance(lt, UserClassLogicalType): + continue + if lt.logical_type_name == arrow_extension_name: + _check_storage(lt, storage_type, record) + return lt + found.append(lt) + + registered = "; ".join( + uci.describe_record(lt.identity_record) + if lt.identity_record is not None + else lt.logical_type_name + for lt in found + ) + raise TypeIdentityError( + f"Stored values of {uci.describe_record(record)} do not match the " + f"registered class: {registered}. A class with the same name but different " + f"fields is a different type; restore the stored fields, or use the class " + f"with the new fields to compute new results." + ) + + +def _type_hints(python_type: type) -> dict[str, Any]: + try: + return typing.get_type_hints(python_type) + except Exception as exc: + raise ValueError(f"Cannot get type hints for {python_type!r}: {exc}") from exc + + +def _check_storage( + lt: UserClassLogicalType, storage_type: pa.DataType, record: dict[str, Any] +) -> None: + from orcapod.utils.arrow_utils import normalize_to_large_types + + if normalize_to_large_types(lt.storage_type) != normalize_to_large_types( + storage_type + ): + raise TypeIdentityError( + f"Stored values of {uci.describe_record(record)} have Arrow storage type " + f"{storage_type}, but the registered class stores {lt.storage_type}." + ) diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index b68a6f39c..2fae197e7 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -429,9 +429,11 @@ def _register_python_class_impl( # Cycle detection (via the shared ContextVar-backed in_progress set) if annotation in in_progress: - raise TypeError( - f"Circular type dependency detected while synthesising " - f"LogicalType for {annotation!r}." + from orcapod.errors import RecursiveTypeError + + raise RecursiveTypeError( + f"{annotation.__qualname__} (a class that refers back to itself " + f"cannot be stored: Arrow has no recursive struct type)" ) # Factory dispatch via MRO walk diff --git a/tests/test_logical_types/test_cache_behavior.py b/tests/test_logical_types/test_cache_behavior.py index 90bac4471..52416f3e3 100644 --- a/tests/test_logical_types/test_cache_behavior.py +++ b/tests/test_logical_types/test_cache_behavior.py @@ -28,6 +28,15 @@ ) +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return lt.logical_type_name + + # Module-level dataclass — local classes cannot be reconstructed from FQCN. @@ -74,18 +83,19 @@ def test_cache_populated_after_first_read(tmp_path): parquet_path = _write_parquet(tmp_path, write_converter) read_converter = _fresh_converter() - fqcn = f"{_CachePoint.__module__}.{_CachePoint.__qualname__}" + type_name = _identity(_CachePoint) # Before read: not registered assert ( - read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn) is None + read_converter._logical_type_registry.get_by_arrow_extension_name(type_name) + is None ) read_converter.load_logical_types(pq.read_table(parquet_path)) # After read: registered (cache populated) assert ( - read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn) + read_converter._logical_type_registry.get_by_arrow_extension_name(type_name) is not None ) diff --git a/tests/test_logical_types/test_dataclass_logical_type_factory.py b/tests/test_logical_types/test_dataclass_logical_type_factory.py index 70b098e8f..c300c8bc5 100644 --- a/tests/test_logical_types/test_dataclass_logical_type_factory.py +++ b/tests/test_logical_types/test_dataclass_logical_type_factory.py @@ -9,6 +9,37 @@ import pytest +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return lt.logical_type_name + + +def _metadata(cls: type) -> dict: + """Return the parsed Arrow extension metadata of a user class (ITL-881).""" + import json + + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return json.loads(lt.get_arrow_extension_type().__arrow_ext_serialize__()) + + +def _metadata(cls: type) -> dict: + """Return the parsed Arrow extension metadata of a user class (ITL-881).""" + import json + + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return json.loads(lt.get_arrow_extension_type().__arrow_ext_serialize__()) + + # ── Helpers ───────────────────────────────────────────────────────────────── @@ -382,16 +413,18 @@ def test_factory_reconstruct_from_arrow(): ) storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - metadata = {"category": "orcapod.dataclass"} - fqcn = f"{_RoundTripPoint.__module__}.{_RoundTripPoint.__qualname__}" + metadata = _metadata(_RoundTripPoint) + type_name = _identity(_RoundTripPoint) factory = DataclassLogicalTypeFactory() converter = _make_full_converter() - lt = factory.reconstruct_from_arrow(fqcn, storage, metadata, converter=converter) + lt = factory.reconstruct_from_arrow( + type_name, storage, metadata, converter=converter + ) assert isinstance(lt, DataclassLogicalType) assert lt.python_type is _RoundTripPoint - assert lt.logical_type_name == fqcn + assert lt.logical_type_name == type_name def test_factory_reconstruct_from_arrow_invalid_fqcn(): @@ -408,7 +441,14 @@ def test_factory_reconstruct_from_arrow_invalid_fqcn(): factory.reconstruct_from_arrow( "nonexistent.module.NoSuchClass", storage, - {"category": "orcapod.dataclass"}, + { + "category": "orcapod.dataclass", + "identity": { + "kind": "dataclass", + "name": "nonexistent.module.NoSuchClass", + "fields": {"x": "int"}, + }, + }, converter, ) @@ -429,7 +469,7 @@ def test_reconstruct_from_arrow_registers_nested_types(): pa.field("label", pa.large_string()), ] ) - outer_fqcn = f"{_OuterForRegistrationTest.__module__}.{_OuterForRegistrationTest.__qualname__}" + outer_type_name = _identity(_OuterForRegistrationTest) factory = DataclassLogicalTypeFactory() converter = _make_full_converter() @@ -442,7 +482,7 @@ def test_reconstruct_from_arrow_registers_nested_types(): # reconstruct_from_arrow for Outer should trigger registration of Inner as a side effect factory.reconstruct_from_arrow( - outer_fqcn, outer_storage, {"category": "orcapod.dataclass"}, converter + outer_type_name, outer_storage, _metadata(_OuterForRegistrationTest), converter ) # Inner must now be registered @@ -499,12 +539,12 @@ def test_dataclass_with_uuid_round_trip(): assert reconstructed.label == "hello" -# ── _import_from_fqcn nested class tests ───────────────────────────────────── +# ── nested class lookup on read ─────────────────────────────────────────────── @dataclasses.dataclass class _OuterForNestedTest: - """Module-level outer class for testing nested-class FQCN import.""" + """Module-level outer class for testing the read-side lookup of a nested class.""" @dataclasses.dataclass class Inner: @@ -512,20 +552,18 @@ class Inner: y: str -def test_import_from_fqcn_nested_class(): - """_import_from_fqcn resolves module-level nested dataclasses via attribute walk.""" - from orcapod.logical_types.dataclass_logical_type_factory import _import_from_fqcn - - # _OuterForNestedTest.Inner lives in this test module; its FQCN uses '.' for nesting - module = _OuterForNestedTest.__module__ - inner_qualname = ( - _OuterForNestedTest.Inner.__qualname__ - ) # e.g. "_OuterForNestedTest.Inner" +def test_reconstruct_finds_class_nested_in_a_class(): + """The read path imports a class nested in another class by its dotted name.""" + from orcapod.logical_types.dataclass_logical_type_factory import ( + DataclassLogicalTypeFactory, + ) - fqcn = f"{module}.{inner_qualname}" - cls = _import_from_fqcn(fqcn) - assert cls is _OuterForNestedTest.Inner - assert dataclasses.is_dataclass(cls) + inner = _OuterForNestedTest.Inner + storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.large_string())]) + lt = DataclassLogicalTypeFactory().reconstruct_from_arrow( + _identity(inner), storage, _metadata(inner), _make_full_converter() + ) + assert lt.python_type is inner def test_python_to_storage_raises_when_converter_none(): diff --git a/tests/test_logical_types/test_default_context_factories.py b/tests/test_logical_types/test_default_context_factories.py index 2c880b16b..550d5c185 100644 --- a/tests/test_logical_types/test_default_context_factories.py +++ b/tests/test_logical_types/test_default_context_factories.py @@ -20,6 +20,15 @@ from orcapod.logical_types.registry import LogicalTypeRegistry +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return lt.logical_type_name + + # ── Module-level dataclasses (local classes cannot be registered) ──────────── @@ -115,8 +124,8 @@ def test_default_context_dataclass_auto_registered_on_use(): converter = create_registry().get_context().type_converter arrow_type = converter.register_python_class(_SimplePoint) assert isinstance(arrow_type, pa.ExtensionType) - fqcn = f"{_SimplePoint.__module__}.{_SimplePoint.__qualname__}" - assert arrow_type.extension_name == fqcn + type_name = _identity(_SimplePoint) + assert arrow_type.extension_name == type_name def test_default_context_pydantic_auto_registered_on_use(): @@ -124,8 +133,8 @@ def test_default_context_pydantic_auto_registered_on_use(): converter = create_registry().get_context().type_converter arrow_type = converter.register_python_class(_SimpleModel) assert isinstance(arrow_type, pa.ExtensionType) - fqcn = f"{_SimpleModel.__module__}.{_SimpleModel.__qualname__}" - assert arrow_type.extension_name == fqcn + type_name = _identity(_SimpleModel) + assert arrow_type.extension_name == type_name # ── Parquet round-trip tests ───────────────────────────────────────────────── diff --git a/tests/test_logical_types/test_pydantic_logical_type_factory.py b/tests/test_logical_types/test_pydantic_logical_type_factory.py index f7f6d2cdc..f5087722e 100644 --- a/tests/test_logical_types/test_pydantic_logical_type_factory.py +++ b/tests/test_logical_types/test_pydantic_logical_type_factory.py @@ -11,6 +11,26 @@ from pydantic import BaseModel, PrivateAttr, computed_field, field_validator +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return lt.logical_type_name + + +def _metadata(cls: type) -> dict: + """Return the parsed Arrow extension metadata of a user class (ITL-881).""" + import json + + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return json.loads(lt.get_arrow_extension_type().__arrow_ext_serialize__()) + + # ── Helpers ────────────────────────────────────────────────────────────────── @@ -410,16 +430,18 @@ def test_factory_reconstruct_from_arrow(): ) storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - metadata = {"category": "orcapod.pydantic"} - fqcn = f"{_RoundTripPoint.__module__}.{_RoundTripPoint.__qualname__}" + metadata = _metadata(_RoundTripPoint) + type_name = _identity(_RoundTripPoint) factory = PydanticLogicalTypeFactory() converter = _make_full_converter() - lt = factory.reconstruct_from_arrow(fqcn, storage, metadata, converter=converter) + lt = factory.reconstruct_from_arrow( + type_name, storage, metadata, converter=converter + ) assert isinstance(lt, PydanticLogicalType) assert lt.python_type is _RoundTripPoint - assert lt.logical_type_name == fqcn + assert lt.logical_type_name == type_name def test_factory_reconstruct_from_arrow_invalid_fqcn(): @@ -436,7 +458,14 @@ def test_factory_reconstruct_from_arrow_invalid_fqcn(): factory.reconstruct_from_arrow( "nonexistent.module.NoSuchModel", storage, - {"category": "orcapod.pydantic"}, + { + "category": "orcapod.pydantic", + "identity": { + "kind": "pydantic", + "name": "nonexistent.module.NoSuchClass", + "fields": {"x": "int"}, + }, + }, converter, ) @@ -454,7 +483,7 @@ def test_reconstruct_from_arrow_registers_nested_types(): pa.field("label", pa.large_string()), ] ) - outer_fqcn = f"{_OuterModel.__module__}.{_OuterModel.__qualname__}" + outer_type_name = _identity(_OuterModel) factory = PydanticLogicalTypeFactory() converter = _make_full_converter() @@ -463,7 +492,7 @@ def test_reconstruct_from_arrow_registers_nested_types(): assert converter._logical_type_registry.get_by_python_type(_InnerModel) is None factory.reconstruct_from_arrow( - outer_fqcn, outer_storage, {"category": "orcapod.pydantic"}, converter + outer_type_name, outer_storage, _metadata(_OuterModel), converter ) # Inner must now be registered as a side effect @@ -737,9 +766,9 @@ def test_literal_model_polars_dataframe_round_trip(): df = pl.from_arrow(table) # Polars column must carry the correct extension type - fqcn = f"{_LiteralRoundTripModel.__module__}.{_LiteralRoundTripModel.__qualname__}" + type_name = _identity(_LiteralRoundTripModel) assert isinstance(df.dtypes[0], pl.BaseExtension) - assert df.dtypes[0].ext_name() == fqcn + assert df.dtypes[0].ext_name() == type_name # Convert back to Arrow and reconstruct Python objects table_back = df.to_arrow() diff --git a/tests/test_logical_types/test_roundtrips.py b/tests/test_logical_types/test_roundtrips.py index c576836f9..4fe911fb7 100644 --- a/tests/test_logical_types/test_roundtrips.py +++ b/tests/test_logical_types/test_roundtrips.py @@ -34,6 +34,15 @@ from orcapod.semantic_types.universal_converter import UniversalTypeConverter +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return lt.logical_type_name + + # ── Module-level dataclasses ────────────────────────────────────────────────── # DataclassLogicalTypeFactory rejects local (in-function) classes because they # have no stable fully-qualified class name for reconstruction from Arrow schema. @@ -338,12 +347,12 @@ def test_simple_dataclass_round_trip( tmp_path, ) - fqcn = f"{_PointA.__module__}.{_PointA.__qualname__}" + type_name = _identity(_PointA) field = result.schema.field("point") assert hasattr(field.type, "extension_name"), ( f"Expected extension type on field 'point', got {field.type!r}" ) - assert field.type.extension_name == fqcn + assert field.type.extension_name == type_name rows = read_converter.arrow_table_to_python_dicts(result) assert len(rows) == 1 @@ -370,13 +379,13 @@ def test_two_dataclasses_same_shape_distinct_extension_names( tmp_path, ) - fqcn_a = f"{_PointA.__module__}.{_PointA.__qualname__}" - fqcn_b = f"{_PointB.__module__}.{_PointB.__qualname__}" + type_name_a = _identity(_PointA) + type_name_b = _identity(_PointB) field = result.schema.field("point") assert hasattr(field.type, "extension_name") - assert field.type.extension_name == fqcn_a - assert field.type.extension_name != fqcn_b # distinct from _PointB + assert field.type.extension_name == type_name_a + assert field.type.extension_name != type_name_b # distinct from _PointB rows = read_converter.arrow_table_to_python_dicts(result) reconstructed = rows[0]["point"] @@ -401,15 +410,19 @@ def test_nested_dataclass_round_trip( tmp_path, ) - fqcn_outer = f"{_Outer.__module__}.{_Outer.__qualname__}" - fqcn_inner = f"{_Inner.__module__}.{_Inner.__qualname__}" + type_name_outer = _identity(_Outer) + type_name_inner = _identity(_Inner) assert ( - read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn_outer) + read_converter._logical_type_registry.get_by_arrow_extension_name( + type_name_outer + ) is not None ), "_Outer should be registered after read" assert ( - read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn_inner) + read_converter._logical_type_registry.get_by_arrow_extension_name( + type_name_inner + ) is not None ), "_Inner should be registered transitively after read" @@ -444,7 +457,7 @@ def test_delta_polars_read_delta(tmp_path: Path) -> None: import polars as pl delta_path = str(tmp_path / "polars_delta") - fqcn = f"{_PointA.__module__}.{_PointA.__qualname__}" + type_name = _identity(_PointA) # Write — registers _PointA in PyArrow + Polars global registries. write_converter = _fresh_converter() @@ -463,8 +476,8 @@ def test_delta_polars_read_delta(tmp_path: Path) -> None: assert col_dtype.is_extension(), ( f"Expected a Polars extension type on column 'point', got {col_dtype!r}" ) - assert col_dtype.ext_name() == fqcn, ( - f"Expected extension name {fqcn!r}, got {col_dtype.ext_name()!r}" + assert col_dtype.ext_name() == type_name, ( + f"Expected extension name {type_name!r}, got {col_dtype.ext_name()!r}" ) diff --git a/tests/test_logical_types/test_schema_compatibility.py b/tests/test_logical_types/test_schema_compatibility.py index 860519348..b7c527010 100644 --- a/tests/test_logical_types/test_schema_compatibility.py +++ b/tests/test_logical_types/test_schema_compatibility.py @@ -28,6 +28,15 @@ from orcapod.utils.schema_utils import check_schema_compatibility +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + + lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + assert lt is not None + return lt.logical_type_name + + # Module-level dataclasses — DataclassLogicalTypeFactory rejects local classes # because they have no stable fully-qualified class name for reconstruction. @@ -64,10 +73,10 @@ def test_arrow_schema_distinct_extension_names_for_same_shape(): assert isinstance(type_a, pa.ExtensionType) assert isinstance(type_b, pa.ExtensionType) - fqcn_a = f"{_PointA.__module__}.{_PointA.__qualname__}" - fqcn_b = f"{_PointB.__module__}.{_PointB.__qualname__}" - assert type_a.extension_name == fqcn_a - assert type_b.extension_name == fqcn_b + type_name_a = _identity(_PointA) + type_name_b = _identity(_PointB) + assert type_a.extension_name == type_name_a + assert type_b.extension_name == type_name_b assert type_a.extension_name != type_b.extension_name diff --git a/tests/test_logical_types/test_type_utils.py b/tests/test_logical_types/test_type_utils.py index e8e52864a..c83c61dc5 100644 --- a/tests/test_logical_types/test_type_utils.py +++ b/tests/test_logical_types/test_type_utils.py @@ -167,15 +167,3 @@ def _patched(name: str, *args, **kwargs): with pytest.raises(ModuleNotFoundError, match="'path'"): _walk_fqcn("pathlib.Path") - - -# ── _import_from_fqcn tests ────────────────────────────────────────────────── - - -def test_import_from_fqcn_raises_for_non_dataclass(): - """_import_from_fqcn raises ImportError when FQCN resolves to a non-dataclass.""" - from orcapod.logical_types.dataclass_logical_type_factory import _import_from_fqcn - - # pathlib.Path is importable via _walk_fqcn but is not a dataclass - with pytest.raises(ImportError): - _import_from_fqcn("pathlib.Path") From b4104a455de35caf97ea70158abbd33424de6ba1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:39:24 -0800 Subject: [PATCH 06/31] feat(types): register_type API, rebinding and read-side identity checks (ITL-881) - op.register_type declares the name a dataclass or pydantic model is identified by; op.set_type_naming sets the defaults (__main__ classes by name, module classes by module path). - Stored fields are kept in name order, so field order changes neither the identity nor the Arrow struct. - A pod defined before a class cell re-run accepts the re-defined class (schema compatibility compares user classes by identity). - DeltaTableDatabase re-raises TypeIdentityError instead of logging a read failure, and read-time identity errors name the column. - Tests: workshop scenarios W1-W12 in fresh processes (IPython InteractiveShell for notebook cells), public API, S2 storage guard. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/__init__.py | 6 + src/orcapod/databases/delta_lake_databases.py | 21 + src/orcapod/logical_types/database_hooks.py | 32 +- .../dataclass_logical_type_factory.py | 2 +- .../pydantic_logical_type_factory.py | 2 +- .../logical_types/user_class_identity.py | 56 ++- .../logical_types/user_class_logical_type.py | 17 +- src/orcapod/user_types.py | 97 +++++ src/orcapod/utils/schema_utils.py | 6 +- .../test_logical_types/test_cache_behavior.py | 5 +- .../test_dataclass_logical_type_factory.py | 23 +- .../test_default_context_factories.py | 5 +- .../test_pydantic_logical_type_factory.py | 12 +- tests/test_logical_types/test_roundtrips.py | 7 +- .../test_schema_compatibility.py | 4 +- tests/test_user_types/child_harness.py | 149 +++++++ tests/test_user_types/test_read_path.py | 52 +++ tests/test_user_types/test_register_type.py | 361 +++++++++++++++++ .../test_workshop_scenarios.py | 365 ++++++++++++++++++ 19 files changed, 1177 insertions(+), 45 deletions(-) create mode 100644 src/orcapod/user_types.py create mode 100644 tests/test_user_types/child_harness.py create mode 100644 tests/test_user_types/test_read_path.py create mode 100644 tests/test_user_types/test_register_type.py create mode 100644 tests/test_user_types/test_workshop_scenarios.py diff --git a/src/orcapod/__init__.py b/src/orcapod/__init__.py index bc229f096..59b3f6571 100644 --- a/src/orcapod/__init__.py +++ b/src/orcapod/__init__.py @@ -60,6 +60,9 @@ # Extension registration API (ITL-473) from .extensions import OrcapodExtension, register_extension +# Your own data types: dataclass / pydantic identity (ITL-881) +from .user_types import register_type, set_type_naming + # Post-run hook types (ITL-523) from .hooks import ( HookConfig, @@ -106,6 +109,9 @@ # Extension registration API (ITL-473) "OrcapodExtension", "register_extension", + # Your own data types (ITL-881) + "register_type", + "set_type_naming", # Post-run hook types (ITL-523) "HookConfig", "InvocationStatus", diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index c7dec5dcf..fed945012 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast from orcapod.databases.utils import coerce_record_id +from orcapod.errors import TypeIdentityError from orcapod.databases.storage_utils import is_cloud_uri, parse_base_path from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule @@ -574,6 +575,10 @@ def _handle_delta_schema_compatibility( # Get Delta table schema and convert from arro3 to pyarrow arro3_schema = delta_table.schema().to_arrow() delta_schema = pa.schema(arro3_schema) # type: ignore + except TypeIdentityError: + # Stored and registered user-class shapes disagree: never report + # this as missing data (ITL-881). + raise except Exception as e: logger.warning(f"Could not get Delta table schema: {e}") return table @@ -644,6 +649,10 @@ def get_all_records( ).to_table() if delta_table_data.num_rows > 0: tables_to_combine.append(delta_table_data) + except TypeIdentityError: + # Stored and registered user-class shapes disagree: never report + # this as missing data (ITL-881). + raise except Exception as e: logger.warning(f"Error reading Delta table: {e}") @@ -710,6 +719,10 @@ def get_records_with_column_value( # Handle (remove/rename) the record id column before returning return self._handle_record_id_column(result, record_id_column) + except TypeIdentityError: + # Stored and registered user-class shapes disagree: never report + # this as missing data (ITL-881). + raise except Exception as e: logger.error( f"Error getting record with {column_values} from {'/'.join(record_path)}: {e}" @@ -769,6 +782,10 @@ def get_record_by_id( # Handle (remove/rename) the record id column before returning return self._handle_record_id_column(result, record_id_column) + except TypeIdentityError: + # Stored and registered user-class shapes disagree: never report + # this as missing data (ITL-881). + raise except Exception as e: logger.error( f"Error getting record {record_id} from {'/'.join(record_path)}: {e}" @@ -826,6 +843,10 @@ def get_records_by_ids( # Handle record_id column based on parameter return self._handle_record_id_column(result, record_id_column) + except TypeIdentityError: + # Stored and registered user-class shapes disagree: never report + # this as missing data (ITL-881). + raise except Exception as e: logger.error( f"Error getting records by IDs from {'/'.join(record_path)}: {e}" diff --git a/src/orcapod/logical_types/database_hooks.py b/src/orcapod/logical_types/database_hooks.py index 65bf04065..845b88bd5 100644 --- a/src/orcapod/logical_types/database_hooks.py +++ b/src/orcapod/logical_types/database_hooks.py @@ -24,8 +24,9 @@ import logging from typing import TYPE_CHECKING +from orcapod.errors import TypeIdentityError from orcapod.logical_types.registry import LogicalTypeRegistry -from orcapod.logical_types.schema_walker import walk_schema +from orcapod.logical_types.schema_walker import walk_field, walk_schema if TYPE_CHECKING: import pyarrow as pa @@ -56,6 +57,10 @@ def register_discovered_logical_types( Raises: ValueError: Propagated from the converter if an extension type's metadata has no registered factory or is malformed. + TypeIdentityError: If a stored user-class column does not match the class + found for it; the message starts with the column name. + ImportError: If no class can be found for a stored user-class column; the + message starts with the column name. """ if converter is None: logger.debug( @@ -73,13 +78,24 @@ def register_discovered_logical_types( [info.extension_name for info in found], ) for info in found: - # Bottom-up resolve the storage type first, then register the extension - resolved_storage = converter.register_storage_type(info.storage_type) - converter.register_logical_type_from_arrow_metadata( - info.extension_name, - info.extension_metadata, - resolved_storage, - ) + try: + # Bottom-up resolve the storage type first, then register the extension + resolved_storage = converter.register_storage_type(info.storage_type) + converter.register_logical_type_from_arrow_metadata( + info.extension_name, + info.extension_metadata, + resolved_storage, + ) + except (TypeIdentityError, ImportError) as exc: + # Name the column(s) holding the type: the factory only knows the type. + columns = ", ".join( + repr(field.name) + for field in schema + if any( + i.extension_name == info.extension_name for i in walk_field(field) + ) + ) + raise type(exc)(f"Column {columns}: {exc}") from exc def apply_logical_types( diff --git a/src/orcapod/logical_types/dataclass_logical_type_factory.py b/src/orcapod/logical_types/dataclass_logical_type_factory.py index a73df8ccf..978cad3f4 100644 --- a/src/orcapod/logical_types/dataclass_logical_type_factory.py +++ b/src/orcapod/logical_types/dataclass_logical_type_factory.py @@ -11,7 +11,7 @@ import dataclasses -from orcapod.logical_types import user_class_identity as uci +import orcapod.logical_types.user_class_identity as uci from orcapod.logical_types.user_class_logical_type import ( UserClassLogicalType, UserClassLogicalTypeFactory, diff --git a/src/orcapod/logical_types/pydantic_logical_type_factory.py b/src/orcapod/logical_types/pydantic_logical_type_factory.py index d714b961a..baa66fc93 100644 --- a/src/orcapod/logical_types/pydantic_logical_type_factory.py +++ b/src/orcapod/logical_types/pydantic_logical_type_factory.py @@ -10,7 +10,7 @@ from __future__ import annotations -from orcapod.logical_types import user_class_identity as uci +import orcapod.logical_types.user_class_identity as uci from orcapod.logical_types.user_class_logical_type import ( UserClassLogicalType, UserClassLogicalTypeFactory, diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index ae8b91f3a..56486bfd1 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -60,6 +60,8 @@ _declared: weakref.WeakKeyDictionary[type, str] = weakref.WeakKeyDictionary() # The name each class object was built under; a class has one name per process. _built: weakref.WeakKeyDictionary[type, str] = weakref.WeakKeyDictionary() +# The identity string each class object was built with. +_identity_of: weakref.WeakKeyDictionary[type, str] = weakref.WeakKeyDictionary() # Classes seen under each (kind, name), most recent last. _known: dict[tuple[str, str], list[weakref.ref[type]]] = {} # Names that already produced the one-time ``__main__`` notice. @@ -146,7 +148,7 @@ def set_naming( raise ValueError( f"Invalid naming style for {key}: {value!r}. Use 'name' or 'qualified'." ) - _naming[key] = value + _naming[key] = typing.cast(NamingStyle, value) def is_main_class(cls: type) -> bool: @@ -237,9 +239,53 @@ def built_name(cls: type) -> str | None: return _built.get(cls) -def mark_built(cls: type, name: str) -> None: - """Record that ``cls`` was built under ``name`` (one name per class per process).""" +def mark_built(cls: type, name: str, identity: str) -> None: + """Record that ``cls`` was built under ``name`` with ``identity``. + + A class has one name per process. + + Args: + cls: The class. + name: Its name component. + identity: Its identity string. + """ _built[cls] = name + _identity_of[cls] = identity + + +def equivalent_hints(a: Any, b: Any) -> bool: + """Return True if two type hints are equal up to user-class identity. + + Two user classes are equivalent when they were built with the same identity string, + for example a notebook class and its re-definition from a re-run cell. Generic hints + are compared by origin and arguments; union members in any order. + + Args: + a: A type hint. + b: A type hint. + + Returns: + True if the hints denote the same types. + """ + import types as _types + + if a == b: + return True + if isinstance(a, type) and isinstance(b, type): + ident = _identity_of.get(a) + return ident is not None and ident == _identity_of.get(b) + origin_a, origin_b = typing.get_origin(a), typing.get_origin(b) + if origin_a is None or origin_b is None: + return False + args_a, args_b = typing.get_args(a), typing.get_args(b) + unions = (typing.Union, _types.UnionType) + if origin_a in unions and origin_b in unions: + return all(any(equivalent_hints(x, y) for y in args_b) for x in args_a) and all( + any(equivalent_hints(x, y) for x in args_a) for y in args_b + ) + if origin_a != origin_b or len(args_a) != len(args_b): + return False + return all(equivalent_hints(x, y) for x, y in zip(args_a, args_b)) # --------------------------------------------------------------------------- @@ -405,8 +451,8 @@ def type_token(annotation: Any, converter: TypeConverterProtocol) -> str: inner = ", ".join(type_token(a, converter) for a in args) return f"{_class_token(origin)}[{inner}]" if isinstance(annotation, type): - converter.register_python_class(annotation) - lt = converter.get_logical_type(annotation) + _ = converter.register_python_class(annotation) # raises for unsupported types + lt = converter.get_logical_type_for_python_type(annotation) if lt is not None: return lt.logical_type_name return _class_token(annotation) diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index f7e4bcd10..745755bdf 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Any from orcapod.errors import RecursiveTypeError, TypeIdentityError -from orcapod.logical_types import user_class_identity as uci +import orcapod.logical_types.user_class_identity as uci from orcapod.logical_types.base_logical_type import BaseLogicalType from orcapod.logical_types.registry import ( make_arrow_extension_type, @@ -117,13 +117,13 @@ def storage_type(self) -> pa.StructType: def get_arrow_extension_type(self) -> pa.ExtensionType: """Return the cached Arrow extension type (name = identity string).""" if self._arrow_ext is None: - self._arrow_ext = self._arrow_ext_class() + self._arrow_ext = self._arrow_ext_class() # pyright: ignore[reportCallIssue] # synthesized class takes no arguments return self._arrow_ext def get_polars_extension_type(self) -> pl.BaseExtension: """Return the cached Polars extension type.""" if self._polars_ext is None: - self._polars_ext = self._polars_ext_class() + self._polars_ext = self._polars_ext_class() # pyright: ignore[reportCallIssue] # synthesized class takes no arguments return self._polars_ext def python_to_storage( @@ -227,7 +227,9 @@ def create_for_python_type( arrow_fields = [] field_annotations: list[tuple[str, Any]] = [] tokens: dict[str, str] = {} - for field_name in self.stored_field_names(python_type): + # Sorted by name: field order is not part of the identity, so it must not + # change the storage type either (same identity, same struct). + for field_name in sorted(self.stored_field_names(python_type)): annotation = hints.get(field_name, Any) try: arrow_type = converter.register_python_class(annotation) @@ -246,7 +248,7 @@ def create_for_python_type( record = uci.identity_record(self.kind, name, tokens) identity = uci.identity_string(record) - uci.mark_built(python_type, name) + uci.mark_built(python_type, name, identity) uci.remember(self.kind, name, python_type) uci.notice_main_class(python_type, name) logger.debug("%s: built %r for %r", type(self).__name__, identity, python_type) @@ -299,6 +301,7 @@ def reconstruct_from_arrow( f"record for category {self.category!r}: {record!r}." ) + kind = self.kind candidates = uci.candidate_classes(kind, name) if not candidates: raise ImportError( @@ -310,8 +313,8 @@ def reconstruct_from_arrow( found: list[UserClassLogicalType] = [] for cls in candidates: - converter.register_python_class(cls) - lt = converter.get_logical_type(cls) + _ = converter.register_python_class(cls) # raises for unsupported classes + lt = converter.get_logical_type_for_python_type(cls) if not isinstance(lt, UserClassLogicalType): continue if lt.logical_type_name == arrow_extension_name: diff --git a/src/orcapod/user_types.py b/src/orcapod/user_types.py new file mode 100644 index 000000000..d9b4cceaf --- /dev/null +++ b/src/orcapod/user_types.py @@ -0,0 +1,97 @@ +"""Registering your own data types (dataclasses and pydantic models). + +Orcapod identifies a dataclass or pydantic model by its **name**, its **kind** +(dataclass or pydantic) and its **fields** (names and types, in any order). Two classes +that agree on all three are the same type: results stored with one are found again with +the other. See ``docs/concepts/your-own-data-types.md``. + +``register_type`` declares the name a class is identified by. ``set_type_naming`` sets +how classes that declare no name are named. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TypeVar, overload + +import orcapod.logical_types.user_class_identity as _uci + +T = TypeVar("T", bound=type) + +__all__ = ["register_type", "set_type_naming"] + + +@overload +def register_type(cls: T, /, *, name: str | None = None) -> T: ... + + +@overload +def register_type(*, name: str | None = None) -> Callable[[T], T]: ... + + +def register_type( + cls: T | None = None, /, *, name: str | None = None +) -> T | Callable[[T], T]: + """Declare the name a dataclass or pydantic model is identified by. + + Use it as a decorator, with or without a name, or call it on a class you do not + own. Without a name, the class is named by the naming setting (see + ``set_type_naming``): a class defined in a notebook or script by its class name, + a class defined in a module by its module path and class name. + + Declare the name before the class is first used in a source or pod. A class + defined again with the same name and fields (a re-run notebook cell) is the same + type; with different fields it is a different type. + + Args: + cls: The class, when called directly. + name: The declared name, e.g. ``"Point"`` or ``"geo.Point"``. It must not be + empty, contain ``@`` or whitespace, or start with ``orcapod.``. + + Returns: + The class unchanged, or a decorator that returns it unchanged. + + Raises: + TypeError: If the class is not a dataclass or pydantic model. + ValueError: If the name is invalid, the class is defined inside a function, or + the class was already used under a different name. + + Example: + >>> import dataclasses + >>> import orcapod as op + >>> @op.register_type(name="Point") + ... @dataclasses.dataclass + ... class Point: + ... x: int + ... y: int + """ + + def decorate(target: T) -> T: + _uci.declare(target, name) + return target + + if cls is None: + return decorate + return decorate(cls) + + +def set_type_naming( + *, main: _uci.NamingStyle | None = None, module: _uci.NamingStyle | None = None +) -> None: + """Set how dataclasses and pydantic models that declare no name are named. + + Applies to classes first used after the call, so call it before defining or using + your classes. + + Args: + main: Classes defined in a notebook or a script run directly (``__main__``): + ``"name"`` (default) uses the class name (``"Point"``); ``"qualified"`` + uses ``"__main__.Point"``. + module: Classes defined in an importable module: ``"qualified"`` (default) + uses the module path and class name (``"pkg.models.Point"``); ``"name"`` + uses the class name only. + + Raises: + ValueError: If a value is not ``"name"`` or ``"qualified"``. + """ + _uci.set_naming(main=main, module=module) diff --git a/src/orcapod/utils/schema_utils.py b/src/orcapod/utils/schema_utils.py index f6146a6b3..43efab3ef 100644 --- a/src/orcapod/utils/schema_utils.py +++ b/src/orcapod/utils/schema_utils.py @@ -41,11 +41,15 @@ def check_schema_compatibility( ) -> bool: from beartype.door import is_subhint + from orcapod.logical_types.user_class_identity import equivalent_hints + for key, type_info in incoming_types.items(): if key not in receiving_types: logger.warning(f"Key '{key}' not found in parameter types.") return False - if not is_subhint(type_info, receiving_types[key]): + if not is_subhint(type_info, receiving_types[key]) and not equivalent_hints( + type_info, receiving_types[key] + ): logger.warning( f"Type mismatch for key '{key}': expected {receiving_types[key]}, got {type_info}." ) diff --git a/tests/test_logical_types/test_cache_behavior.py b/tests/test_logical_types/test_cache_behavior.py index 52416f3e3..92e168ceb 100644 --- a/tests/test_logical_types/test_cache_behavior.py +++ b/tests/test_logical_types/test_cache_behavior.py @@ -31,8 +31,11 @@ def _identity(cls: type) -> str: """Return the identity string (logical type name) of a user class (ITL-881).""" from orcapod.contexts import get_default_context + from orcapod.types import Schema - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) assert lt is not None return lt.logical_type_name diff --git a/tests/test_logical_types/test_dataclass_logical_type_factory.py b/tests/test_logical_types/test_dataclass_logical_type_factory.py index c300c8bc5..415103d35 100644 --- a/tests/test_logical_types/test_dataclass_logical_type_factory.py +++ b/tests/test_logical_types/test_dataclass_logical_type_factory.py @@ -12,30 +12,25 @@ def _identity(cls: type) -> str: """Return the identity string (logical type name) of a user class (ITL-881).""" from orcapod.contexts import get_default_context + from orcapod.types import Schema - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) assert lt is not None return lt.logical_type_name -def _metadata(cls: type) -> dict: +def _metadata(cls: type) -> dict[str, object]: """Return the parsed Arrow extension metadata of a user class (ITL-881).""" import json from orcapod.contexts import get_default_context + from orcapod.types import Schema - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) - assert lt is not None - return json.loads(lt.get_arrow_extension_type().__arrow_ext_serialize__()) - - -def _metadata(cls: type) -> dict: - """Return the parsed Arrow extension metadata of a user class (ITL-881).""" - import json - - from orcapod.contexts import get_default_context - - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) assert lt is not None return json.loads(lt.get_arrow_extension_type().__arrow_ext_serialize__()) diff --git a/tests/test_logical_types/test_default_context_factories.py b/tests/test_logical_types/test_default_context_factories.py index 550d5c185..48b5a8f8d 100644 --- a/tests/test_logical_types/test_default_context_factories.py +++ b/tests/test_logical_types/test_default_context_factories.py @@ -23,8 +23,11 @@ def _identity(cls: type) -> str: """Return the identity string (logical type name) of a user class (ITL-881).""" from orcapod.contexts import get_default_context + from orcapod.types import Schema - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) assert lt is not None return lt.logical_type_name diff --git a/tests/test_logical_types/test_pydantic_logical_type_factory.py b/tests/test_logical_types/test_pydantic_logical_type_factory.py index f5087722e..a3bce9a06 100644 --- a/tests/test_logical_types/test_pydantic_logical_type_factory.py +++ b/tests/test_logical_types/test_pydantic_logical_type_factory.py @@ -14,19 +14,25 @@ def _identity(cls: type) -> str: """Return the identity string (logical type name) of a user class (ITL-881).""" from orcapod.contexts import get_default_context + from orcapod.types import Schema - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) assert lt is not None return lt.logical_type_name -def _metadata(cls: type) -> dict: +def _metadata(cls: type) -> dict[str, object]: """Return the parsed Arrow extension metadata of a user class (ITL-881).""" import json from orcapod.contexts import get_default_context + from orcapod.types import Schema - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) assert lt is not None return json.loads(lt.get_arrow_extension_type().__arrow_ext_serialize__()) diff --git a/tests/test_logical_types/test_roundtrips.py b/tests/test_logical_types/test_roundtrips.py index 4fe911fb7..fafbf21f2 100644 --- a/tests/test_logical_types/test_roundtrips.py +++ b/tests/test_logical_types/test_roundtrips.py @@ -37,8 +37,11 @@ def _identity(cls: type) -> str: """Return the identity string (logical type name) of a user class (ITL-881).""" from orcapod.contexts import get_default_context + from orcapod.types import Schema - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) assert lt is not None return lt.logical_type_name @@ -477,7 +480,7 @@ def test_delta_polars_read_delta(tmp_path: Path) -> None: f"Expected a Polars extension type on column 'point', got {col_dtype!r}" ) assert col_dtype.ext_name() == type_name, ( - f"Expected extension name {type_name!r}, got {col_dtype.ext_name()!r}" + f"Expected extension name {type_name!r}, got {col_dtype.ext_name()!r}" # pyright: ignore[reportAttributeAccessIssue] # polars stubs lack ext_name ) diff --git a/tests/test_logical_types/test_schema_compatibility.py b/tests/test_logical_types/test_schema_compatibility.py index b7c527010..6289f6b09 100644 --- a/tests/test_logical_types/test_schema_compatibility.py +++ b/tests/test_logical_types/test_schema_compatibility.py @@ -32,7 +32,9 @@ def _identity(cls: type) -> str: """Return the identity string (logical type name) of a user class (ITL-881).""" from orcapod.contexts import get_default_context - lt = get_default_context().type_converter.get_logical_type_for_python_type(cls) + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) assert lt is not None return lt.logical_type_name diff --git a/tests/test_user_types/child_harness.py b/tests/test_user_types/child_harness.py new file mode 100644 index 000000000..c28b944a2 --- /dev/null +++ b/tests/test_user_types/child_harness.py @@ -0,0 +1,149 @@ +"""Run workshop scenarios in fresh Python processes (ITL-881 tests). + +A user class's identity depends on process-wide state (where the class is defined, what +is registered, Arrow's global extension registry), so the scenarios run each "kernel" or +"script" in its own interpreter. A child reports by printing one ``JSON:`` line +through ``report``. + +``WSJOB`` is a module the scenarios write next to their scripts: it runs one function pod +over a ``DictSource`` in a ``PipelineJob`` stored in a ``DeltaTableDatabase``. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +JSON_MARKER = "JSON:" +HARNESS_DIR = Path(__file__).parent + + +def write_files(root: Path, files: Mapping[str, str]) -> None: + """Write ``files`` (relative path to dedented content) under ``root``.""" + for rel, content in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(textwrap.dedent(content).lstrip(), encoding="utf-8") + + +def run_child( + argv: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str] | None = None, + timeout: float = 300.0, +) -> Any: + """Run ``python `` in ``cwd`` and return the payload it reported. + + Args: + argv: Arguments after the interpreter, e.g. ``["-c", code]``. + cwd: Working directory; also put on ``PYTHONPATH`` with this harness. + env: Extra environment variables. + timeout: Seconds before the child is killed. + + Returns: + The reported payload. + + Raises: + AssertionError: If the child exits non-zero or reports nothing. + """ + child_env = dict(os.environ) + child_env["PYTHONPATH"] = os.pathsep.join([str(cwd), str(HARNESS_DIR)]) + child_env.update(env or {}) + proc = subprocess.run( + [sys.executable, *argv], + cwd=cwd, + env=child_env, + capture_output=True, + text=True, + check=False, + timeout=timeout, + ) + for line in proc.stdout.splitlines(): + if line.startswith(JSON_MARKER): + return json.loads(line[len(JSON_MARKER) :]) + raise AssertionError( + f"child reported nothing (exit {proc.returncode}):\n{proc.stderr[-4000:]}" + ) + + +def report(payload: Any) -> None: + """Print ``payload`` as the JSON line the parent reads, then exit at once. + + ``os._exit`` skips interpreter shutdown: a process that read Delta tables sometimes + hangs in Arrow's thread-pool destructor at shutdown (seen in the ITL-881 + experiments; out of scope here). + """ + print(JSON_MARKER + json.dumps(payload, default=str)) + sys.stdout.flush() + sys.stderr.flush() + os._exit(0) + + +def error_text(exc: BaseException) -> str: + """Return ``": "`` for an exception.""" + first = str(exc).splitlines()[0] if str(exc) else "" + return f"{type(exc).__name__}: {first}" + + +#: Job module written next to the scenario scripts. +WSJOB = ''' +import os +from pathlib import Path + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import DictSource + + +def table_paths(root): + return sorted( + str(p.parent.relative_to(root)) for p in Path(root).rglob("_delta_log") if p.is_dir() + ) + + +def make_pod(point_cls): + """Return ``(pod, calls)`` for ``shift(p: point_cls) -> point_cls``.""" + calls = [] + + def shift(p): + calls.append(p) + return p + + shift.__annotations__ = {"p": point_cls, "return": point_cls} + return function_pod(output_keys="q")(shift), calls + + +def run_job(point_cls, pod_and_calls=None, rows=None): + """Run the shift pod over two rows; report calls, rows and identity.""" + pod, calls = pod_and_calls or make_pod(point_cls) + if rows is None: + names = list(getattr(point_cls, "model_fields", None) or point_cls.__dataclass_fields__) + rows = [{n: i for n in names} for i in (1, 2)] + data = [{"k": f"r{i}", "p": point_cls(**row)} for i, row in enumerate(rows, 1)] + before = len(calls) + db_root = os.environ["WS_DB"] + job = PipelineJob(name="ws", store=DeltaTableDatabase(Path(db_root))) + with job: + pod.pod(DictSource(data=data, tag_columns=["k"], source_id="s"), label="shift") + job.run() + node = job.nodes["shift"] + values = [d.as_dict()["q"] for _, d in node.iter_data()] + return { + "calls": len(calls) - before, + "rows": node.as_table().num_rows, + "q_class_is_point": all(type(v) is point_cls for v in values), + "pod_pipeline_hash": pod.pod.pipeline_hash().to_string(), + "node_identity_path": list(node.node_identity_path), + "table_paths": table_paths(db_root), + "extension_name": getattr( + node.as_table().schema.field("q").type, "extension_name", None + ), + } +''' diff --git a/tests/test_user_types/test_read_path.py b/tests/test_user_types/test_read_path.py new file mode 100644 index 000000000..7c085b1f9 --- /dev/null +++ b/tests/test_user_types/test_read_path.py @@ -0,0 +1,52 @@ +"""Stored and registered shapes of a user class must agree on read (ITL-881, S2). + +A stored column whose extension name matches a registered class but whose Arrow storage +differs (possible only if two data contexts map a field differently, or a table was +written by other means) must raise ``TypeIdentityError`` from ``DeltaTableDatabase`` +instead of being logged and read as "no records". +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import Any + +import deltalake +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import TypeIdentityError +from orcapod.sources import DictSource + + +@op.register_type(name="itl881.StoredPoint") +@dataclasses.dataclass +class StoredPoint: + x: int + y: int + + +def _extension_field_metadata() -> dict[bytes | str, bytes | str]: + value: Any = StoredPoint(1, 2) + source = DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) + ext = source.as_table().schema.field("p").type + return { + b"ARROW:extension:name": ext.extension_name.encode(), + b"ARROW:extension:metadata": ext.__arrow_ext_serialize__(), + } + + +def test_storage_mismatch_raises_instead_of_reading_nothing(tmp_path: Path) -> None: + wrong = pa.struct([pa.field("x", pa.float64()), pa.field("y", pa.int64())]) + field = pa.field("q", wrong).with_metadata(_extension_field_metadata()) + table = pa.table( + {"q": pa.array([{"x": 1.5, "y": 2}], type=wrong)}, + schema=pa.schema([field]), + ) + deltalake.write_deltalake(str(tmp_path / "points"), table) + + with pytest.raises(TypeIdentityError, match="expected storage_type"): + DeltaTableDatabase(tmp_path).get_all_records(("points",)) diff --git a/tests/test_user_types/test_register_type.py b/tests/test_user_types/test_register_type.py new file mode 100644 index 000000000..204c78ac3 --- /dev/null +++ b/tests/test_user_types/test_register_type.py @@ -0,0 +1,361 @@ +"""Public API for user class identity: ``op.register_type`` and ``op.set_type_naming`` (ITL-881). + +Identity is observed through the public interface: the Arrow extension name of a +``DictSource`` column (``as_table``), the source's ``pipeline_hash`` and values read back +with ``iter_data``. Every test declares names under ``itl881.`` that no other test uses, +because declarations are process-wide. +""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Iterator +from typing import Any + +import pydantic +import pytest + +import orcapod as op +from orcapod.errors import RecursiveTypeError +from orcapod.logical_types import ( + user_class_identity as uci, +) # get_naming: restore the policy +from orcapod.sources import DictSource + + +def _source(value: Any) -> DictSource: + return DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) + + +def _type_name(value: Any) -> str: + return _source(value).as_table().schema.field("p").type.extension_name + + +def _name_part(value: Any) -> str: + return _type_name(value).split("@")[0] + + +# --------------------------------------------------------------------------- +# Names +# --------------------------------------------------------------------------- + + +@op.register_type(name="itl881.Declared") +@dataclasses.dataclass +class Declared: + x: int + + +@dataclasses.dataclass +class Undeclared: + x: int + + +@op.register_type +@dataclasses.dataclass +class BareDecorated: + x: int + + +@dataclasses.dataclass +class CalledLater: + x: int + + +def test_declared_name_is_the_name_component() -> None: + assert _name_part(Declared(1)) == "itl881.Declared" + + +def test_module_class_defaults_to_module_path() -> None: + assert _name_part(Undeclared(1)) == f"{__name__}.Undeclared" + + +def test_bare_decorator_uses_the_naming_setting() -> None: + assert _name_part(BareDecorated(1)) == f"{__name__}.BareDecorated" + + +def test_call_form_returns_the_class() -> None: + assert op.register_type(CalledLater, name="itl881.CalledLater") is CalledLater + assert _name_part(CalledLater(1)) == "itl881.CalledLater" + + +@pytest.mark.parametrize("name", ["", "a@b", "a b", "orcapod.Point"]) +def test_invalid_names_raise(name: str) -> None: + with pytest.raises(ValueError): + op.register_type(name=name)(Undeclared) + + +def test_non_dataclass_raises_type_error() -> None: + class Plain: + x: int = 0 + + with pytest.raises(TypeError, match="dataclasses and pydantic models"): + op.register_type(Plain, name="itl881.Plain") + + +def test_local_class_raises() -> None: + @dataclasses.dataclass + class Local: + x: int + + with pytest.raises(ValueError, match="inside a function"): + op.register_type(Local, name="itl881.Local") + + +@dataclasses.dataclass +class UsedFirst: + x: int + + +def test_declaring_after_use_under_another_name_raises() -> None: + _type_name(UsedFirst(1)) + with pytest.raises(ValueError, match="already registered as"): + op.register_type(UsedFirst, name="itl881.UsedFirst") + + +@dataclasses.dataclass +class RenamedByPolicy: + x: int + + +def test_set_type_naming_module_name() -> None: + before = uci.get_naming() + op.set_type_naming(module="name") + try: + assert _name_part(RenamedByPolicy(1)) == "RenamedByPolicy" + finally: + op.set_type_naming(**before) + + +def test_set_type_naming_rejects_unknown_style() -> None: + with pytest.raises(ValueError, match="'name' or 'qualified'"): + op.set_type_naming(main="short") # pyright: ignore[reportArgumentType] + + +# --------------------------------------------------------------------------- +# Fields +# --------------------------------------------------------------------------- + + +@op.register_type(name="itl881.Shape") +@dataclasses.dataclass +class ShapeInt: + x: int + y: int + + +@op.register_type(name="itl881.Shape") +@dataclasses.dataclass +class ShapeFloat: + x: int + y: float + + +@op.register_type(name="itl881.Shape") +class ShapeModel(pydantic.BaseModel): + x: int + y: int + + +def test_same_name_different_field_type_is_a_different_type() -> None: + a, b = _source(ShapeInt(1, 2)), _source(ShapeFloat(1, 2.0)) + assert _type_name(ShapeInt(1, 2)) != _type_name(ShapeFloat(1, 2.0)) + assert a.pipeline_hash() != b.pipeline_hash() + + +def test_dataclass_and_pydantic_model_are_different_types() -> None: + assert _type_name(ShapeInt(1, 2)) != _type_name(ShapeModel(x=1, y=2)) + + +@op.register_type(name="itl881.Defaults") +@dataclasses.dataclass +class DefaultsA: + x: int = 0 + + def method(self) -> int: + return self.x + + @property + def doubled(self) -> int: + return 2 * self.x + + +@op.register_type(name="itl881.Defaults") +@dataclasses.dataclass +class DefaultsB: + """Same fields as DefaultsA; different default, no methods.""" + + x: int = 5 + + +def test_defaults_methods_and_properties_are_not_identity() -> None: + first = _type_name(DefaultsA(1)) + with pytest.warns(UserWarning, match="same identity"): + second = _type_name(DefaultsB(1)) + assert first == second + + +@op.register_type(name="itl881.Computed") +class ComputedA(pydantic.BaseModel): + x: int + + @pydantic.computed_field + @property + def doubled(self) -> int: + return 2 * self.x + + +@op.register_type(name="itl881.Computed") +class ComputedB(pydantic.BaseModel): + x: int + + +def test_pydantic_computed_fields_are_not_identity() -> None: + first = _type_name(ComputedA(x=1)) + with pytest.warns(UserWarning, match="same identity"): + second = _type_name(ComputedB(x=1)) + assert first == second + + +# --------------------------------------------------------------------------- +# Nested classes (P1) and registration order +# --------------------------------------------------------------------------- + + +@op.register_type(name="itl881.InnerA") +@dataclasses.dataclass +class InnerA: + value: int + + +@op.register_type(name="itl881.InnerB") +@dataclasses.dataclass +class InnerB: + value: int + + +@op.register_type(name="itl881.InnerA") +@dataclasses.dataclass +class InnerAFloat: + value: float + + +@op.register_type(name="itl881.Outer") +@dataclasses.dataclass +class OuterA: + inner: InnerA + + +@op.register_type(name="itl881.Outer") +@dataclasses.dataclass +class OuterB: + inner: InnerB + + +@op.register_type(name="itl881.Outer") +@dataclasses.dataclass +class OuterAFloat: + inner: InnerAFloat + + +def test_nested_class_contributes_its_name() -> None: + assert _type_name(OuterA(InnerA(1))) != _type_name(OuterB(InnerB(1))) + + +def test_nested_class_contributes_its_fields() -> None: + assert _type_name(OuterA(InnerA(1))) != _type_name(OuterAFloat(InnerAFloat(1.0))) + + +@dataclasses.dataclass +class LateInner: + value: int + + +@op.register_type(name="itl881.OrderOuter") +@dataclasses.dataclass +class EarlyOuter: + inner: LateInner + + +# Declared after the outer class was decorated, before either is used. +op.register_type(LateInner, name="itl881.OrderInner") + + +@op.register_type(name="itl881.OrderInner") +@dataclasses.dataclass +class FirstInner: + value: int + + +@op.register_type(name="itl881.OrderOuter") +@dataclasses.dataclass +class SecondOuter: + inner: FirstInner + + +def test_registration_order_does_not_matter() -> None: + late_order = _type_name(EarlyOuter(LateInner(1))) + with pytest.warns(UserWarning, match="same identity"): + natural_order = _type_name(SecondOuter(FirstInner(1))) + assert late_order == natural_order + + +def test_nested_values_round_trip() -> None: + _, data = next(iter(_source(OuterA(InnerA(3))).iter_data())) + value = data.as_dict()["p"] + assert value == OuterA(InnerA(3)) + assert type(value.inner) is InnerA + + +@dataclasses.dataclass +class Node: + value: int + next: Node | None = None + + +@dataclasses.dataclass +class CycleA: + b: CycleB | None = None + + +@dataclasses.dataclass +class CycleB: + a: CycleA | None = None + + +def test_self_referential_class_is_rejected_naming_the_field() -> None: + with pytest.raises(RecursiveTypeError, match=r"Node\.next -> Node"): + _source(Node(1)) + + +def test_cyclic_classes_are_rejected_naming_the_path() -> None: + with pytest.raises(RecursiveTypeError, match=r"CycleA\.b -> CycleB\.a -> CycleA"): + _source(CycleA()) + + +# --------------------------------------------------------------------------- +# Subclasses +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Base: + x: int + + +@dataclasses.dataclass +class Child(Base): + z: int = 0 + + +def test_subclass_has_its_own_identity_and_keeps_its_fields() -> None: + assert _type_name(Base(1)) != _type_name(Child(1, 2)) + _, data = next(iter(_source(Child(1, 2)).iter_data())) + assert data.as_dict()["p"] == Child(1, 2) + + +@pytest.fixture(autouse=True) +def _no_policy_leak() -> Iterator[None]: + before = uci.get_naming() + yield + assert uci.get_naming() == before diff --git a/tests/test_user_types/test_workshop_scenarios.py b/tests/test_user_types/test_workshop_scenarios.py new file mode 100644 index 000000000..98231c023 --- /dev/null +++ b/tests/test_user_types/test_workshop_scenarios.py @@ -0,0 +1,365 @@ +"""Workshop scenarios W1–W12 for user class identity (ITL-881). + +Each scenario runs notebook cells (IPython ``InteractiveShell``, as a Jupyter kernel runs +them) or scripts in fresh processes, against one ``DeltaTableDatabase``, and drives the +pipeline only through the public API (``DictSource``, ``function_pod``, ``PipelineJob``, +``ExtensionAwareDatabase``). Scenario numbers follow the findings in PR #288. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.test_user_types.child_harness import WSJOB, run_child, write_files + +pytestmark = pytest.mark.integration + +KERNEL = r""" +import warnings +from IPython.core.interactiveshell import InteractiveShell +from child_harness import error_text, report + +shell = InteractiveShell.instance() +CELLS = { + "setup": "import wsjob\nfrom dataclasses import dataclass\nimport orcapod as op\n", + "define": "@dataclass\nclass Point:\n x: int\n y: int\n", + "define_reordered": "@dataclass\nclass Point:\n y: int\n x: int\n", + "define_float": "@dataclass\nclass Point:\n x: int\n y: float\n", + "define_model": "import pydantic\nclass Point(pydantic.BaseModel):\n x: int\n y: int\n", + "import_mod": "from wsmodels import Point\n", + "reload_mod": "import importlib, wsmodels\nimportlib.reload(wsmodels)\nfrom wsmodels import Point\n", + "pod": "shift = wsjob.make_pod(Point)\n", + "run": "_result = wsjob.run_job(Point)\n", + "run_pod": "_result = wsjob.run_job(Point, shift)\n", +} +log = [] +for step in SCENARIO: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + res = shell.run_cell(CELLS[step], store_history=False, silent=True) + entry = { + "cell": step, + "warnings": [str(w.message) for w in caught if issubclass(w.category, UserWarning)], + } + err = res.error_in_exec or res.error_before_exec + if err is not None: + entry["error"] = error_text(err) + elif step.startswith("run"): + entry.update(shell.user_ns["_result"]) + log.append(entry) +report(log) +""" + +WSMODELS = """ +from dataclasses import dataclass + + +@dataclass +class Point: + x: int + y: int +""" + + +@pytest.fixture +def workdir(tmp_path: Path) -> Path: + write_files(tmp_path, {"wsjob.py": WSJOB, "wsmodels.py": WSMODELS}) + (tmp_path / "db").mkdir() + return tmp_path + + +def _kernel(workdir: Path, cells: list[str]) -> list[dict[str, Any]]: + code = f"SCENARIO = {cells!r}\n" + KERNEL + return run_child(["-c", code], cwd=workdir, env={"WS_DB": str(workdir / "db")}) + + +def _runs(log: list[dict[str, Any]]) -> list[dict[str, Any]]: + errors = [e for e in log if "error" in e] + assert not errors, errors + return [e for e in log if e["cell"].startswith("run")] + + +def _script(workdir: Path, code: str, **env: str) -> Any: + return run_child( + ["-c", code], cwd=workdir, env={"WS_DB": str(workdir / "db"), **env} + ) + + +# --------------------------------------------------------------------------- +# Notebook cell re-runs in one kernel +# --------------------------------------------------------------------------- + + +def test_w1_rerun_define_cell_same_fields_rebinds(workdir: Path) -> None: + log = _kernel(workdir, ["setup", "define", "run", "define", "run"]) + first, second = _runs(log) + assert (first["calls"], first["rows"]) == (2, 2) + assert (second["calls"], second["rows"]) == (0, 2) + assert second["q_class_is_point"] + assert first["pod_pipeline_hash"] == second["pod_pipeline_hash"] + + +def test_w1_old_pod_cell_with_redefined_class(workdir: Path) -> None: + """Re-running only the class cell keeps the earlier pod cell usable.""" + log = _kernel(workdir, ["setup", "define", "pod", "run_pod", "define", "run_pod"]) + first, second = _runs(log) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + + +def test_w1_run_all_twice(workdir: Path) -> None: + log = _kernel(workdir, ["setup", "define", "pod", "run_pod"] * 2) + first, second = _runs(log) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + + +def test_field_order_does_not_change_identity(workdir: Path) -> None: + log = _kernel(workdir, ["setup", "define", "run", "define_reordered", "run"]) + first, second = _runs(log) + assert second["calls"] == 0 + assert first["extension_name"] == second["extension_name"] + + +def test_w2_rerun_with_changed_fields_is_a_new_type(workdir: Path) -> None: + log = _kernel(workdir, ["setup", "define", "run", "define_float", "run"]) + first, second = _runs(log) + assert (second["calls"], second["rows"]) == (2, 2) + assert second["q_class_is_point"] + assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] + assert first["extension_name"].split("@")[0] == "Point" + assert second["extension_name"].split("@")[0] == "Point" + assert first["extension_name"] != second["extension_name"] + assert set(first["table_paths"]) < set(second["table_paths"]) + + +def test_w3_pydantic_rerun_rebinds(workdir: Path) -> None: + log = _kernel(workdir, ["setup", "define_model", "run", "define_model", "run"]) + first, second = _runs(log) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + + +def test_dataclass_and_pydantic_with_same_name_and_fields_are_distinct( + workdir: Path, +) -> None: + log = _kernel(workdir, ["setup", "define", "run", "define_model", "run"]) + first, second = _runs(log) + assert second["calls"] == 2 + assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] + assert first["extension_name"] != second["extension_name"] + + +def test_w4_module_reload_rebinds(workdir: Path) -> None: + log = _kernel(workdir, ["setup", "import_mod", "run", "reload_mod", "run"]) + first, second = _runs(log) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + assert first["extension_name"].startswith("wsmodels.Point@") + + +def test_main_class_notice_once(workdir: Path) -> None: + log = _kernel(workdir, ["setup", "define", "run", "define", "run"]) + notices = [w for e in log for w in e["warnings"] if "notebook or script" in w] + assert len(notices) == 1 + assert "'Point'" in notices[0] + + +# --------------------------------------------------------------------------- +# Across processes +# --------------------------------------------------------------------------- + + +def test_w5_restart_same_class_hits_cache(workdir: Path) -> None: + first = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + second = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + + +def test_w6_restart_with_changed_fields_recomputes(workdir: Path) -> None: + """Changed fields after a restart compute into new tables instead of 0 rows.""" + first = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + changed = _runs(_kernel(workdir, ["setup", "define_float", "run"]))[0] + back = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + assert (changed["calls"], changed["rows"]) == (2, 2) + assert first["node_identity_path"] != changed["node_identity_path"] + assert (back["calls"], back["rows"]) == (0, 2) + + +def test_w7_script_as_main_twice(workdir: Path) -> None: + write_files( + workdir, + { + "pkg/__init__.py": "", + "pkg/pipeline.py": """ + from dataclasses import dataclass + import wsjob + from child_harness import report + + @dataclass + class Point: + x: int + y: int + + def main(): + report(wsjob.run_job(Point)) + + if __name__ == "__main__": + main() + """, + }, + ) + env = {"WS_DB": str(workdir / "db")} + first = run_child(["pkg/pipeline.py"], cwd=workdir, env=env) + second = run_child(["-m", "pkg.pipeline"], cwd=workdir, env=env) + assert (first["calls"], second["calls"]) == (2, 0) + + +def test_w8_script_then_import_hits_with_declared_name(workdir: Path) -> None: + write_files( + workdir, + { + "pkg/__init__.py": "", + "pkg/pipeline.py": """ + from dataclasses import dataclass + import orcapod as op + import wsjob + from child_harness import report + + @op.register_type(name="Point") + @dataclass + class Point: + x: int + y: int + + def main(): + report(wsjob.run_job(Point)) + + if __name__ == "__main__": + main() + """, + }, + ) + env = {"WS_DB": str(workdir / "db")} + first = run_child(["-m", "pkg.pipeline"], cwd=workdir, env=env) + second = run_child( + ["-c", "import pkg.pipeline; pkg.pipeline.main()"], cwd=workdir, env=env + ) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + assert first["node_identity_path"] == second["node_identity_path"] + + +def test_w11_notebook_to_module_with_declared_name(workdir: Path) -> None: + """A class moved from a notebook into a module keeps its results under its name.""" + write_files( + workdir, + { + "pkg/__init__.py": "", + "pkg/models.py": """ + from dataclasses import dataclass + import orcapod as op + + @op.register_type(name="Point") + @dataclass + class Point: + y: int + x: int + """, + }, + ) + notebook = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + module = _script( + workdir, + "import wsjob\nfrom child_harness import report\nfrom pkg.models import Point\n" + "report(wsjob.run_job(Point))\n", + ) + assert (module["calls"], module["rows"]) == (0, 2) + assert module["pod_pipeline_hash"] == notebook["pod_pipeline_hash"] + assert module["node_identity_path"] == notebook["node_identity_path"] + assert module["table_paths"] == notebook["table_paths"] + assert module["extension_name"] == notebook["extension_name"] + + +def test_w11_module_without_declared_name_is_a_different_type(workdir: Path) -> None: + notebook = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + module = _runs(_kernel(workdir, ["setup", "import_mod", "run"]))[0] + assert module["calls"] == 2 + assert module["pod_pipeline_hash"] != notebook["pod_pipeline_hash"] + + +# --------------------------------------------------------------------------- +# Reading stored values +# --------------------------------------------------------------------------- + +READER = """ +import os +from pathlib import Path +from child_harness import error_text, report +from orcapod.contexts import get_default_context +from orcapod.databases import DeltaTableDatabase, ExtensionAwareDatabase +import wsjob + +{define} + +root = Path(os.environ["WS_DB"]) +path = tuple(next(p for p in wsjob.table_paths(root) if "/_result/" in p).split("/")) +converter = get_default_context().type_converter +try: + table = ExtensionAwareDatabase(DeltaTableDatabase(root), converter=converter).get_all_records(path) + rows = converter.arrow_table_to_python_dicts(table.select(["q"])) + report({{"values": sorted(repr(r["q"]) for r in rows), "classes": sorted({{type(r["q"]).__module__ + "." + type(r["q"]).__qualname__ for r in rows}})}}) +except Exception as exc: + report({{"error": error_text(exc), "message": str(exc)}}) +""" + + +def _store_from_notebook(workdir: Path) -> None: + _runs(_kernel(workdir, ["setup", "define", "run"])) + + +def test_w12_read_notebook_results_from_script_with_same_class(workdir: Path) -> None: + _store_from_notebook(workdir) + define = "from dataclasses import dataclass\n@dataclass\nclass Point:\n x: int\n y: int\n" + out = _script(workdir, READER.format(define=define)) + assert out.get("error") is None, out + assert out["values"] == ["Point(x=1, y=1)", "Point(x=2, y=2)"] + assert out["classes"] == ["__main__.Point"] + + +def test_read_with_declared_class_from_a_module(workdir: Path) -> None: + _store_from_notebook(workdir) + write_files( + workdir, + { + "geo.py": """ + from dataclasses import dataclass + import orcapod as op + + @op.register_type(name="Point") + @dataclass + class Point: + x: int + y: int + """ + }, + ) + out = _script(workdir, READER.format(define="import geo")) + assert out.get("error") is None, out + assert out["classes"] == ["geo.Point"] + + +def test_read_without_the_class_names_it_and_how_to_register(workdir: Path) -> None: + _store_from_notebook(workdir) + out = _script(workdir, READER.format(define="")) + assert out["error"].startswith("ImportError: Column 'q'") + assert "'Point'" in out["message"] + assert "register_type" in out["message"] + assert "{x: int, y: int}" in out["message"] + + +def test_read_with_different_fields_raises_naming_both_shapes(workdir: Path) -> None: + _store_from_notebook(workdir) + define = "from dataclasses import dataclass\n@dataclass\nclass Point:\n x: int\n y: float\n" + out = _script(workdir, READER.format(define=define)) + assert out["error"].startswith("TypeIdentityError: Column 'q'"), out + assert "{x: int, y: int}" in out["message"] + assert "{x: int, y: float}" in out["message"] From a7d0044170054581263d80b095b835ba567e997f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:41:01 -0800 Subject: [PATCH 07/31] test(hashing): golden values for v0.3 user class identity (ITL-881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intended hash changes (approved): exactly the 25 entries listed in the spec move — 17 values change and 8 per-row system-tag keys are replaced because they embed the moved pipeline hash. arrow.json and pipeline.json are unchanged. New fixtures pin a declared-name dataclass and pydantic model (same name and fields, distinct types), a nested dataclass with a container field, list[GoldenPoint], and field-order twins that pin equal identity values. Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/golden/cases.py | 97 ++++++++++++++++++ tests/test_hashing/golden/data/datagram.json | 100 +++++++++++++++---- tests/test_hashing/golden/data/schema.json | 22 +++- tests/test_hashing/golden/data/semantic.json | 10 +- 4 files changed, 202 insertions(+), 27 deletions(-) diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index a2e12d808..c7bb02029 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -197,11 +197,82 @@ class GoldenModel(pydantic.BaseModel): count: int +@dataclasses.dataclass +class GoldenDeclared: + """Dataclass declared under a name (ITL-881), with an optional field.""" + + label: str + score: float | None + + +class GoldenDeclaredModel(pydantic.BaseModel): + """Pydantic model declared under the same name and fields as ``GoldenDeclared``.""" + + label: str + score: float | None + + +@dataclasses.dataclass +class GoldenInner: + """Dataclass nested in ``GoldenOuter``; its identity enters the outer one.""" + + value: int + + +@dataclasses.dataclass +class GoldenOuter: + """Dataclass with a nested dataclass field and a container field.""" + + inner: GoldenInner + tags: list[str] + + +@dataclasses.dataclass +class GoldenTwinAB: + """Declared ``golden.Twin``; same fields as ``GoldenTwinBA`` in another order.""" + + a: int + b: str + + +@dataclasses.dataclass +class GoldenTwinBA: + """Declared ``golden.Twin``; must pin the same values as ``GoldenTwinAB``.""" + + b: str + a: int + + +def _declare_golden_names() -> None: + """Declare the names of the ITL-881 fixtures (releases before v0.3 have no names).""" + try: + from orcapod import register_type + except ImportError: + return + register_type(GoldenDeclared, name="golden.Declared") + register_type(GoldenDeclaredModel, name="golden.Declared") + register_type(GoldenInner, name="golden.Inner") + register_type(GoldenOuter, name="golden.Outer") + register_type(GoldenTwinAB, name="golden.Twin") + register_type(GoldenTwinBA, name="golden.Twin") + + +_declare_golden_names() + + def register_logical_classes() -> None: """Register the dataclass and pydantic fixtures with the default type converter.""" tc = _ctx().type_converter tc.register_python_class(GoldenPoint) tc.register_python_class(GoldenModel) + for cls in ( + GoldenDeclared, + GoldenDeclaredModel, + GoldenOuter, + GoldenTwinAB, + GoldenTwinBA, + ): + tc.register_python_class(cls) PATH_A = Path("/golden/data/a.txt") @@ -280,6 +351,12 @@ def _semantic_types() -> dict[str, Any]: "set_uuid": set[uuid.UUID], "dataclass": GoldenPoint, "pydantic": GoldenModel, + "dataclass_declared": GoldenDeclared, + "pydantic_declared": GoldenDeclaredModel, + "dataclass_nested": GoldenOuter, + "dataclass_twin_ab": GoldenTwinAB, + "dataclass_twin_ba": GoldenTwinBA, + "list_dataclass": list[GoldenPoint], } @@ -586,6 +663,26 @@ def _datagram_rows() -> dict[str, list[dict[str, Any]]]: ], "list_path": [{"k": "r1", "c": [PATH_A, PATH_B]}, {"k": "r2", "c": []}], "set_uuid": [{"k": "r1", "c": {UUID_A, UUID_B}}, {"k": "r2", "c": {UUID_A}}], + "dataclass_declared": [ + {"k": "r1", "c": GoldenDeclared("a", 0.5)}, + {"k": "r2", "c": GoldenDeclared("b", None)}, + ], + "pydantic_declared": [ + {"k": "r1", "c": GoldenDeclaredModel(label="a", score=0.5)}, + {"k": "r2", "c": GoldenDeclaredModel(label="b", score=None)}, + ], + "dataclass_nested": [ + {"k": "r1", "c": GoldenOuter(GoldenInner(1), ["x", "y"])}, + {"k": "r2", "c": GoldenOuter(GoldenInner(2), [])}, + ], + "dataclass_twin_ab": [ + {"k": "r1", "c": GoldenTwinAB(1, "x")}, + {"k": "r2", "c": GoldenTwinAB(2, "y")}, + ], + "dataclass_twin_ba": [ + {"k": "r1", "c": GoldenTwinBA("x", 1)}, + {"k": "r2", "c": GoldenTwinBA("y", 2)}, + ], } diff --git a/tests/test_hashing/golden/data/datagram.json b/tests/test_hashing/golden/data/datagram.json index e9bf5b544..4a93fe769 100644 --- a/tests/test_hashing/golden/data/datagram.json +++ b/tests/test_hashing/golden/data/datagram.json @@ -89,17 +89,65 @@ "_tag_source_id::b58c50c5ff232cdb71eb0272547485074c9784600f5a34218434f7c3de419e8e", "_tag_record_id::b58c50c5ff232cdb71eb0272547485074c9784600f5a34218434f7c3de419e8e" ], - "dataclass.content_hash": "semantic_v0.1:018c841dd6f3d3df7f15689df0ac43adf3b8086655f315311fd53fd98abae218", - "dataclass.pipeline_hash": "semantic_v0.1:c16022c03e2749f6a4f1d7ac2b255e87d1ccce6895c5328f2489103558b2b14c", - "dataclass.row[{\"k\":\"r1\"}]._tag_record_id::c16022c03e2749f6a4f1d7ac2b255e87d1ccce6895c5328f2489103558b2b14c": "26db93a5eab45b0485c2699148b0600d", - "dataclass.row[{\"k\":\"r1\"}]._tag_source_id::c16022c03e2749f6a4f1d7ac2b255e87d1ccce6895c5328f2489103558b2b14c": "golden_dataclass", - "dataclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000015e306dd402af59e45e6f5a9113e71e691f96006cb8cf63246504800071050f55", - "dataclass.row[{\"k\":\"r2\"}]._tag_record_id::c16022c03e2749f6a4f1d7ac2b255e87d1ccce6895c5328f2489103558b2b14c": "71d5c6ce2f7b5b5b9dfdc7a61a7df366", - "dataclass.row[{\"k\":\"r2\"}]._tag_source_id::c16022c03e2749f6a4f1d7ac2b255e87d1ccce6895c5328f2489103558b2b14c": "golden_dataclass", - "dataclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b088edbc3232e6d836e6036a75d5c0a0d83864166d21a16ad113608e21afe445", + "dataclass.content_hash": "semantic_v0.1:30806932620b0715e691bb1eaa74f58e587f5e6eeef08833cff14c9d305411f4", + "dataclass.pipeline_hash": "semantic_v0.1:32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635", + "dataclass.row[{\"k\":\"r1\"}]._tag_record_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635": "26db93a5eab45b0485c2699148b0600d", + "dataclass.row[{\"k\":\"r1\"}]._tag_source_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635": "golden_dataclass", + "dataclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000135da7ac200e567d06cc849537f7726196f576b8498fc7e46bf0849740f67c692", + "dataclass.row[{\"k\":\"r2\"}]._tag_record_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635": "71d5c6ce2f7b5b5b9dfdc7a61a7df366", + "dataclass.row[{\"k\":\"r2\"}]._tag_source_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635": "golden_dataclass", + "dataclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:00000106c5a4a7b5a8d235e66b3aad694a92cdc826c5b416f3148ba93e22767407f564", "dataclass.system_tag_columns": [ - "_tag_source_id::c16022c03e2749f6a4f1d7ac2b255e87d1ccce6895c5328f2489103558b2b14c", - "_tag_record_id::c16022c03e2749f6a4f1d7ac2b255e87d1ccce6895c5328f2489103558b2b14c" + "_tag_source_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635", + "_tag_record_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635" + ], + "dataclass_declared.content_hash": "semantic_v0.1:7e97c35e68612f04276d8de55d9a5232c377aec9ff1ae10e5fa2dd3bc5507f16", + "dataclass_declared.pipeline_hash": "semantic_v0.1:6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c", + "dataclass_declared.row[{\"k\":\"r1\"}]._tag_record_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c": "570e538241085842a7994cad88c5a26b", + "dataclass_declared.row[{\"k\":\"r1\"}]._tag_source_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c": "golden_dataclass_declared", + "dataclass_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001c4cb62165dfd97cc7f0c1949354ba602436ffa8e2f0517fd922e87588cde5782", + "dataclass_declared.row[{\"k\":\"r2\"}]._tag_record_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c": "0e35aec1a9605d428b2741d9f13012d5", + "dataclass_declared.row[{\"k\":\"r2\"}]._tag_source_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c": "golden_dataclass_declared", + "dataclass_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b2cf415f684673e3fa593084372b3b8b0bb165bc82619fde3ba44e7e78ef2001", + "dataclass_declared.system_tag_columns": [ + "_tag_source_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c", + "_tag_record_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c" + ], + "dataclass_nested.content_hash": "semantic_v0.1:8887fd4cfaf912c84ccbf49786814c789b8ea972774056c132dadf0106016a22", + "dataclass_nested.pipeline_hash": "semantic_v0.1:839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665", + "dataclass_nested.row[{\"k\":\"r1\"}]._tag_record_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665": "3b11b0ab021d59f28c8efaf02ad69aee", + "dataclass_nested.row[{\"k\":\"r1\"}]._tag_source_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665": "golden_dataclass_nested", + "dataclass_nested.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000014be5231f5cdac2ab183a8616ac8e7390385a21c5ee2efdcc951f4feb81c05ddd", + "dataclass_nested.row[{\"k\":\"r2\"}]._tag_record_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665": "178f8da0e9ed5520827e648f5a890193", + "dataclass_nested.row[{\"k\":\"r2\"}]._tag_source_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665": "golden_dataclass_nested", + "dataclass_nested.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:00000111562233d9e99e74ad15355aafd40b2e377e4a0799db72330416a082490552d9", + "dataclass_nested.system_tag_columns": [ + "_tag_source_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665", + "_tag_record_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665" + ], + "dataclass_twin_ab.content_hash": "semantic_v0.1:313370fb33d9bea1af19c4fb2c8206a97a3e26186f686930e51bd1514bb63616", + "dataclass_twin_ab.pipeline_hash": "semantic_v0.1:95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e", + "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "f35be6f60b275f2c81fbc49366c2db67", + "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "golden_dataclass_twin_ab", + "dataclass_twin_ab.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001a4f0170a2d724ce2395c6ab6595053a9482d58e05eac5b3311d9e6b56e4f80f4", + "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "7307d67233695c9ab4952175e9735bc9", + "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "golden_dataclass_twin_ab", + "dataclass_twin_ab.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001721c4768261afb271b69fef8680f8df75e5ab8d3494bc8da048ec1322a18fd25", + "dataclass_twin_ab.system_tag_columns": [ + "_tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e", + "_tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e" + ], + "dataclass_twin_ba.content_hash": "semantic_v0.1:9dd09b862ed11da8904ae2df1d7224394adea497e46773b2aa5778448583aeda", + "dataclass_twin_ba.pipeline_hash": "semantic_v0.1:95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e", + "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "65d4fb8402a75d059b3fbfa686a22420", + "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "golden_dataclass_twin_ba", + "dataclass_twin_ba.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001a4f0170a2d724ce2395c6ab6595053a9482d58e05eac5b3311d9e6b56e4f80f4", + "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "3d1c61df9063598ca8178d55c4a47ff9", + "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "golden_dataclass_twin_ba", + "dataclass_twin_ba.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001721c4768261afb271b69fef8680f8df75e5ab8d3494bc8da048ec1322a18fd25", + "dataclass_twin_ba.system_tag_columns": [ + "_tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e", + "_tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e" ], "dataframe.content_hash": "semantic_v0.1:f18422d163082bd23e621e586c3d484ab3a25b6c1ad959b08daa4d63ef461e68", "dataframe.pipeline_hash": "semantic_v0.1:38f74931045e006f752450c2d26dbf9096c5bff65fbe952591dec83b04296a14", @@ -173,17 +221,29 @@ "_tag_source_id::bf2528f33e2c72825ecca9151c2302d77df0e6e7544e955e56101468f9b2bd09", "_tag_record_id::bf2528f33e2c72825ecca9151c2302d77df0e6e7544e955e56101468f9b2bd09" ], - "pydantic.content_hash": "semantic_v0.1:8e5ec65a6c6e45a672c6abf5dbf1e3332e8efebef128be193704364769780612", - "pydantic.pipeline_hash": "semantic_v0.1:aeb2c7a55a50242b1c5916876b4004e77108a32ac2b97bd974184ad4091ed967", - "pydantic.row[{\"k\":\"r1\"}]._tag_record_id::aeb2c7a55a50242b1c5916876b4004e77108a32ac2b97bd974184ad4091ed967": "cb9bbf1a138d5c848e86f99bab7c0acb", - "pydantic.row[{\"k\":\"r1\"}]._tag_source_id::aeb2c7a55a50242b1c5916876b4004e77108a32ac2b97bd974184ad4091ed967": "golden_pydantic", - "pydantic.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000012280b233c56771fa4f901eff64090067273d718ac8d36b881b78d79d4128ab15", - "pydantic.row[{\"k\":\"r2\"}]._tag_record_id::aeb2c7a55a50242b1c5916876b4004e77108a32ac2b97bd974184ad4091ed967": "10c1dbaba95a5c5fa7c14661231224eb", - "pydantic.row[{\"k\":\"r2\"}]._tag_source_id::aeb2c7a55a50242b1c5916876b4004e77108a32ac2b97bd974184ad4091ed967": "golden_pydantic", - "pydantic.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:00000138d77ff36d7b2f3c326a128b325d8219e31f55be9264a21abf51638993059c7e", + "pydantic.content_hash": "semantic_v0.1:cdcdbc5ee86513eb5cb5a5ac65597283624cba0fdcaff90160ba2944e1bf1ca2", + "pydantic.pipeline_hash": "semantic_v0.1:1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad", + "pydantic.row[{\"k\":\"r1\"}]._tag_record_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad": "cb9bbf1a138d5c848e86f99bab7c0acb", + "pydantic.row[{\"k\":\"r1\"}]._tag_source_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad": "golden_pydantic", + "pydantic.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000017138b5a4d4d4ebc1cfafd17ad95a997c673a0ec614a1fb453b0b0adca1a698c9", + "pydantic.row[{\"k\":\"r2\"}]._tag_record_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad": "10c1dbaba95a5c5fa7c14661231224eb", + "pydantic.row[{\"k\":\"r2\"}]._tag_source_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad": "golden_pydantic", + "pydantic.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b32cd5f30c4ee4b7a38bce31ac7d94419a9212d33ad07e5d08757ce8e94e9118", "pydantic.system_tag_columns": [ - "_tag_source_id::aeb2c7a55a50242b1c5916876b4004e77108a32ac2b97bd974184ad4091ed967", - "_tag_record_id::aeb2c7a55a50242b1c5916876b4004e77108a32ac2b97bd974184ad4091ed967" + "_tag_source_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad", + "_tag_record_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad" + ], + "pydantic_declared.content_hash": "semantic_v0.1:ec413e1781f85a762bb5fe2a55794d766e3390954c7606b9d6a1e3e47f71b1af", + "pydantic_declared.pipeline_hash": "semantic_v0.1:922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13", + "pydantic_declared.row[{\"k\":\"r1\"}]._tag_record_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13": "fec6f63bdc695be9aeddfdd100112d42", + "pydantic_declared.row[{\"k\":\"r1\"}]._tag_source_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13": "golden_pydantic_declared", + "pydantic_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001b2d14e386eb560940d8d9f54e03045b5f20f2e8cf560e22fd724fad309d233b5", + "pydantic_declared.row[{\"k\":\"r2\"}]._tag_record_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13": "89983c8f835d51c3aaf2bbca0d7f3d55", + "pydantic_declared.row[{\"k\":\"r2\"}]._tag_source_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13": "golden_pydantic_declared", + "pydantic_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f215ba71a9109b04f48d2627489608d2d83b599f0a4dd97ddc8862f298a9a0a0", + "pydantic_declared.system_tag_columns": [ + "_tag_source_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13", + "_tag_record_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13" ], "scalars.content_hash": "semantic_v0.1:6b123e343edb881e71967e118f1e90b2dc46d20fb60e43b9b7991e3ef822a1d8", "scalars.pipeline_hash": "semantic_v0.1:1fdf145196d8e5dd7b38887cc8798325147df1f5c6f2af80952488a76f9aec07", diff --git a/tests/test_hashing/golden/data/schema.json b/tests/test_hashing/golden/data/schema.json index a33e88f3e..c2502b47d 100644 --- a/tests/test_hashing/golden/data/schema.json +++ b/tests/test_hashing/golden/data/schema.json @@ -2,7 +2,7 @@ "compute_schema_hash.chars_16": "a3e81ce4fbd701a0", "compute_schema_hash.full": "a3e81ce4fbd701a0fb97cae134f236953f7f66a8e7b7f4741339b45fe9f485ba", "multi_field.empty": "semantic_v0.1:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - "multi_field.logical_mix": "semantic_v0.1:29372233b68192c5ff9e48524aaf6073c904eac64479d1c43a6f37e3819a6f1b", + "multi_field.logical_mix": "semantic_v0.1:db02d466882c5127e1e6ab0e264865450102d30773579e666514eba27a72a1ca", "multi_field.nullable_type": "semantic_v0.1:560844c33bb686eddb98de8d5b608f38d845ee5bf64e6a094f9a8326679aafe9", "multi_field.optional_field": "semantic_v0.1:bc030299c9a955ea2f18a2c746121180b5ed0d1fd6265c95ef243088e392b843", "multi_field.two_scalars": "semantic_v0.1:bc030299c9a955ea2f18a2c746121180b5ed0d1fd6265c95ef243088e392b843", @@ -11,8 +11,16 @@ "single_field.bool.schema_hash": "semantic_v0.1:d0845939265975cb588b4821498b4844a6ae18d9613e6bb9bd8c274db521eab0", "single_field.bytes.arrow_type": "large_binary", "single_field.bytes.schema_hash": "semantic_v0.1:c691ab0e3a0ce67c3f8d7b9065d60e2a14afcdce983b362c62de85c9dcf19a52", - "single_field.dataclass.arrow_type": "extension[struct]", - "single_field.dataclass.schema_hash": "semantic_v0.1:a65f5d99a90ffeb981faabef6dbdb0fa03a547287f882e5330b449294580e4e7", + "single_field.dataclass.arrow_type": "extension[struct]", + "single_field.dataclass.schema_hash": "semantic_v0.1:b57fab5e732720af0d9263f2e2885591790c4760d35675acaf084670ef2c4545", + "single_field.dataclass_declared.arrow_type": "extension[struct]", + "single_field.dataclass_declared.schema_hash": "semantic_v0.1:ec33ba184eb852e3b75c7c9e7541b8695f259aec588f541ac6a119d3198b1215", + "single_field.dataclass_nested.arrow_type": "extension[struct, tags: large_list>]", + "single_field.dataclass_nested.schema_hash": "semantic_v0.1:1aee0c1efd5f33da7984bfa837a50b7706ea65a5dc68ad05ba844efffda262d5", + "single_field.dataclass_twin_ab.arrow_type": "extension[struct]", + "single_field.dataclass_twin_ab.schema_hash": "semantic_v0.1:38f6e2b2c6dc3d14de182f2938128a353f056222e184caa202413e3b42edde1e", + "single_field.dataclass_twin_ba.arrow_type": "extension[struct]", + "single_field.dataclass_twin_ba.schema_hash": "semantic_v0.1:38f6e2b2c6dc3d14de182f2938128a353f056222e184caa202413e3b42edde1e", "single_field.dataframe.arrow_type": "extension[large_binary]", "single_field.dataframe.schema_hash": "semantic_v0.1:2544430b3979b1e0ad8cecb9ab45c93b91dda546d5a02a59b5587c73aa04ecd8", "single_field.dict_str_int.arrow_type": "large_list>", @@ -21,6 +29,8 @@ "single_field.float.schema_hash": "semantic_v0.1:1d165501d30136c6e1309c58db930165e0cb51aae8469323283a95297a880191", "single_field.int.arrow_type": "int64", "single_field.int.schema_hash": "semantic_v0.1:2faa1297b96c1248edfafb2422fded3963672852acbe2973155351859eab83d2", + "single_field.list_dataclass.arrow_type": "extension[large_list>]", + "single_field.list_dataclass.schema_hash": "semantic_v0.1:d078161a5775aafe2afde86369e2cc57dcdf00ed010292015a60147b8eee1c47", "single_field.list_int.arrow_type": "large_list", "single_field.list_int.schema_hash": "semantic_v0.1:906c18997d1191d374db82ea5cc763d3d04dc3ceccb4f3fde82cf6d53d0f8dc7", "single_field.list_list_int.arrow_type": "large_list>", @@ -33,8 +43,10 @@ "single_field.optional_int.schema_hash": "semantic_v0.1:16d9f93d8da8f8c5a582352f7022b1191ab1e4593722bbd37fdc1238d34cdc13", "single_field.path.arrow_type": "extension[large_string]", "single_field.path.schema_hash": "semantic_v0.1:84e095c4212af644cc68357daa75a3b9094b953c7fb9030fb4af86f8310b30c3", - "single_field.pydantic.arrow_type": "extension[struct]", - "single_field.pydantic.schema_hash": "semantic_v0.1:a62df3085a2efc4667d0e216148c6898991129df452b12c12cff79900503ce72", + "single_field.pydantic.arrow_type": "extension[struct]", + "single_field.pydantic.schema_hash": "semantic_v0.1:259566a29cf776cf97fd5c6a0201a8026b3d41644a24c2cb1d3b7544507be4ad", + "single_field.pydantic_declared.arrow_type": "extension[struct]", + "single_field.pydantic_declared.schema_hash": "semantic_v0.1:d05e37ea7d328efbd2833d604e9cff59e1b1ca5f9b09133bb37e9ba524c94896", "single_field.set_uuid.arrow_type": "extension[large_list]", "single_field.set_uuid.schema_hash": "semantic_v0.1:73501f29f5a260ab092f67ea9d88c8c3d5d8d5c90bba8d9aea88b3b5400c6272", "single_field.str.arrow_type": "large_string", diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/semantic.json index 7e29628ae..ea3156065 100644 --- a/tests/test_hashing/golden/data/semantic.json +++ b/tests/test_hashing/golden/data/semantic.json @@ -1,18 +1,24 @@ { "type.bool": "semantic_v0.1:cd8f971312b872f0e9a3ef40aa58d6bb537258e72a9a142e0cf712f0a35020e3", "type.bytes": "semantic_v0.1:f65cea8d1f29ca5b96d67caf1ced2f630b509e32574a7b78b9071958818a1c59", - "type.dataclass": "semantic_v0.1:84b8714a03d2a6784736ae38435d9ffbe87c397ea64af275d91e8bdd8281adbc", + "type.dataclass": "semantic_v0.1:814071362ffe7481bde7e4173907a59d5bc6df940f81c9838764d0f83973abbe", + "type.dataclass_declared": "semantic_v0.1:fee6ecce526f9b6791a66e6bb638a4ba0934e79beff9fe247bf71807224fc786", + "type.dataclass_nested": "semantic_v0.1:214904a85c3d6015ee0b2e83dceba5852d3196dbb295b0f4550198a707c23ad1", + "type.dataclass_twin_ab": "semantic_v0.1:921bd905da2fce576b9a8d0274b9716e5cba49edefd123b955c8ebcffe24835c", + "type.dataclass_twin_ba": "semantic_v0.1:921bd905da2fce576b9a8d0274b9716e5cba49edefd123b955c8ebcffe24835c", "type.dataframe": "semantic_v0.1:51e3c34453121e01086e882959cd12746b3236ff7b259a40a42d43493bf26dff", "type.dict_str_int": "semantic_v0.1:f0d36b085fa65f3368d1d98687ffcb6c80099a418419e45a77d434b37e4393dd", "type.float": "semantic_v0.1:0059c08fc7dfe19dc8d095b7e95d56e8d377bbe49ac30376a00105eca2e7c75f", "type.int": "semantic_v0.1:24be397c58c33686e96b4f85a9f443fd2483564ec7dee1c14b96da3e60ec16ac", + "type.list_dataclass": "semantic_v0.1:9568f2ffe6c7dd3c492522b02c06d59b6f57a6888e0db1d59db41dcac1bcf363", "type.list_int": "semantic_v0.1:5c376e29b0cfebd85277c71c07c9996afaf06fab35f81681b82f705c9feafa3e", "type.list_list_int": "semantic_v0.1:ba62756844df6c82b48a739fadd6bb568a3507896dc5dba192dd3e0285635c48", "type.list_path": "semantic_v0.1:e2029c1032fad111f798363e1f718ecde715573c690c3cc78f33824a246a82ce", "type.ndarray": "semantic_v0.1:b6f37b2016cf3ee3062c3e715bab25fa02b68e92931bce243684300ba1dda5b6", "type.optional_int": "semantic_v0.1:91bc2bd8f4f083d6cbf6157350e693dd52b0bc06c57b7719db8581712098cd73", "type.path": "semantic_v0.1:02a12969616ecd278958ffca715be2865fe4a32c27523059df37245c5048a880", - "type.pydantic": "semantic_v0.1:01ca376f8a57dd68d9645036cb5fe247b410f6dfc6ef3cc09661a9e71d2eb4d6", + "type.pydantic": "semantic_v0.1:76a4fd9c0acc945dfba19bb621c91b9eb9f6361df50e6f17a38a3ae7b0b947bc", + "type.pydantic_declared": "semantic_v0.1:a18f664d19b35c92b7a12ac983973d9aedb5e8c86f23a168207f3eee3225cc24", "type.set_uuid": "semantic_v0.1:0ac1b6672ab001f6ee64f406e6d1af3a21024d6f78ef0c7414014cdfad96eaf6", "type.str": "semantic_v0.1:309d607de6633b959ae5118171f079b2830f4b1340064fac065652e305b038ad", "type.tuple_int_str": "semantic_v0.1:65ac71aa155fbfa28072adba04ed55d8d99d89e9c87a0d5ed45ccd15fcafc099", From 285c884c979e01ad02dbadeac32dd564bdebe3ab Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:45:59 -0800 Subject: [PATCH 08/31] feat(types): full-length identity hash, per-class qualified option, no naming switch (ITL-881) Developer decisions (Nov 10, 2026): the identity string carries the full 64-hex SHA-256; the process-wide naming setter is dropped in favour of the fixed location rule (__main__ -> class name, module -> module path) plus register_type(qualified=False) per class. Registration never modifies the class, is idempotent for the same name and raises for a different one. Golden values regenerated: the same 25 listed entries move, plus additions. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/__init__.py | 3 +- .../logical_types/user_class_identity.py | 115 +++++++------- src/orcapod/user_types.py | 69 ++++----- tests/test_hashing/golden/data/datagram.json | 140 +++++++++--------- tests/test_hashing/golden/data/schema.json | 34 ++--- tests/test_hashing/golden/data/semantic.json | 16 +- tests/test_user_types/test_identity_record.py | 27 +--- tests/test_user_types/test_register_type.py | 57 ++++--- 8 files changed, 220 insertions(+), 241 deletions(-) diff --git a/src/orcapod/__init__.py b/src/orcapod/__init__.py index 59b3f6571..95fa89776 100644 --- a/src/orcapod/__init__.py +++ b/src/orcapod/__init__.py @@ -61,7 +61,7 @@ from .extensions import OrcapodExtension, register_extension # Your own data types: dataclass / pydantic identity (ITL-881) -from .user_types import register_type, set_type_naming +from .user_types import register_type # Post-run hook types (ITL-523) from .hooks import ( @@ -111,7 +111,6 @@ "register_extension", # Your own data types (ITL-881) "register_type", - "set_type_naming", # Post-run hook types (ITL-523) "HookConfig", "InvocationStatus", diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index 56486bfd1..aa6fefcca 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -4,9 +4,8 @@ spec ``superpowers/specs/2026-11-10-user-class-identity-v0.3.md``): - ``kind``: ``"dataclass"`` or ``"pydantic"``. -- ``name``: a declared name, or a name chosen by the naming policy (the qualified name - for ``__main__`` classes is just ``__qualname__`` by default; module classes use - ``module.qualname`` by default). +- ``name``: a declared name; otherwise ``__qualname__`` for a class defined in + ``__main__`` (a notebook or script) and ``module.qualname`` for a module class. - ``fields``: a mapping from each stored field's name to its *type token*. It is serialized with sorted keys, so field order never enters the identity. @@ -14,7 +13,7 @@ Arrow extension name. ``identity_string`` is the only place that defines its format. This module also holds the process-wide state that belongs to classes rather than to a -data context: declared names, the naming policy and the table of classes seen under each +data context: declared names and the table of classes seen under each ``(kind, name)``, which the read path uses to find a class for a stored column. """ @@ -38,12 +37,6 @@ KIND_DATACLASS = "dataclass" KIND_PYDANTIC = "pydantic" -NamingStyle = Literal["name", "qualified"] - -_NAMING_STYLES: frozenset[str] = frozenset({"name", "qualified"}) -_DEFAULT_NAMING: dict[str, NamingStyle] = {"main": "name", "module": "qualified"} -_naming: dict[str, NamingStyle] = dict(_DEFAULT_NAMING) - #: Modules whose classes are "defined in a notebook or script". ``__mp_main__`` is the #: name a ``multiprocessing`` spawn child gives the main module. MAIN_MODULES: frozenset[str] = frozenset({"__main__", "__mp_main__"}) @@ -51,8 +44,8 @@ #: Prefix reserved for Orcapod's own types (``orcapod.path``, ``orcapod.uuid``, ...). RESERVED_PREFIX = "orcapod." -#: Number of hex digits of the SHA-256 fingerprint in an identity string. -FINGERPRINT_HEX_DIGITS = 16 +#: Number of hex digits of the SHA-256 fingerprint in an identity string (the full digest). +FINGERPRINT_HEX_DIGITS = 64 _NAME_FORBIDDEN = re.compile(r"[@\s]") @@ -120,47 +113,38 @@ def validate_name(name: str) -> None: ) -def get_naming() -> dict[str, NamingStyle]: - """Return the current naming policy as ``{"main": ..., "module": ...}``.""" - return dict(_naming) - +def is_main_class(cls: type) -> bool: + """Return True if ``cls`` was defined in ``__main__`` (or ``__mp_main__``).""" + return cls.__module__ in MAIN_MODULES -def set_naming( - main: NamingStyle | None = None, module: NamingStyle | None = None -) -> None: - """Set how classes that declare no name are named. - Applies to classes registered after the call. +def default_name(cls: type, *, qualified: bool | None = None) -> str: + """Return the name a class gets without a declared name. Args: - main: ``"name"`` (default) or ``"qualified"`` for classes defined in - ``__main__`` (a notebook or a script run directly). - module: ``"qualified"`` (default) or ``"name"`` for classes defined in an - importable module. + cls: A dataclass or pydantic model class. + qualified: ``None`` applies the location rule: ``__qualname__`` for a class + defined in ``__main__`` (or ``__mp_main__``), ``module.qualname`` for a + module class. ``False`` always gives ``__qualname__``; ``True`` always gives + ``module.qualname``. - Raises: - ValueError: If a value is not ``"name"`` or ``"qualified"``. + Returns: + The name component. """ - for key, value in (("main", main), ("module", module)): - if value is None: - continue - if value not in _NAMING_STYLES: - raise ValueError( - f"Invalid naming style for {key}: {value!r}. Use 'name' or 'qualified'." - ) - _naming[key] = typing.cast(NamingStyle, value) - + if qualified is None: + qualified = not is_main_class(cls) + if not qualified: + return cls.__qualname__ + module = cls.__module__ + if not is_main_class(cls): + from orcapod.utils.canonical_type_names import canonical_module_name -def is_main_class(cls: type) -> bool: - """Return True if ``cls`` was defined in ``__main__`` (or ``__mp_main__``).""" - return cls.__module__ in MAIN_MODULES + module = canonical_module_name(cls) or module + return f"{module}.{cls.__qualname__}" def resolve_name(cls: type) -> str: - """Return the name component for ``cls``. - - The first rule that applies wins: a declared name; the naming policy for - ``__main__`` classes; the naming policy for module classes. + """Return the name component for ``cls``: its declared name or ``default_name``. Args: cls: A dataclass or pydantic model class. @@ -171,28 +155,27 @@ def resolve_name(cls: type) -> str: declared = _declared.get(cls) if declared is not None: return declared - style = _naming["main" if is_main_class(cls) else "module"] - if style == "name": - return cls.__qualname__ - module = cls.__module__ - if not is_main_class(cls): - from orcapod.utils.canonical_type_names import canonical_module_name + return default_name(cls) - module = canonical_module_name(cls) or module - return f"{module}.{cls.__qualname__}" - -def declare(cls: type, name: str | None) -> None: +def declare(cls: type, name: str | None, *, qualified: bool | None = None) -> None: """Record the name a user class is identified by. + Registration never modifies the class: the name is kept in a process-wide table + keyed by the class object. Declaring the same class again with the same name is a + no-op; any other name raises. + Args: cls: A dataclass or pydantic model class. - name: The declared name, or ``None`` to use the naming policy. + name: The declared name, or ``None`` for ``default_name(cls, qualified=...)``. + qualified: Only without ``name``: ``False`` for the class name alone, + ``True`` for ``module.qualname``, ``None`` for the location rule. Raises: TypeError: If ``cls`` is not a dataclass or pydantic model class. - ValueError: If the class is local, the name is invalid, or the class was - already registered in this process under a different name. + ValueError: If both ``name`` and ``qualified`` are given, the class is local, + the name is invalid, or the class is already registered (declared or used) + under a different name. """ kind = class_kind(cls) if kind is None: @@ -200,20 +183,24 @@ def declare(cls: type, name: str | None) -> None: f"Only dataclasses and pydantic models can be registered as types, " f"got {cls!r}." ) + if name is not None and qualified is not None: + raise ValueError( + "Pass either name=... or qualified=..., not both: qualified only chooses " + "between the class name and its module path when no name is given." + ) reject_local_class(cls) if name is not None: validate_name(name) - effective = name if name is not None else resolve_name(cls) - built_as = _built.get(cls) - if built_as is not None and built_as != effective: + effective = name if name is not None else default_name(cls, qualified=qualified) + previous = _built.get(cls) or _declared.get(cls) + if previous is not None and previous != effective: raise ValueError( f"Cannot register {cls.__module__}.{cls.__qualname__} as {effective!r}: it " - f"is already registered as {built_as!r} in this process, because it was " - f"used before this declaration. Declare the name before the class is first " - f"used (for example with the @op.register_type decorator)." + f"is already registered as {previous!r} in this process, and a class has " + f"one name. Declare the name once, before the class is first used (for " + f"example with the @op.register_type decorator)." ) - if name is not None: - _declared[cls] = name + _declared[cls] = effective remember(kind, effective, cls) diff --git a/src/orcapod/user_types.py b/src/orcapod/user_types.py index d9b4cceaf..ab8069aa2 100644 --- a/src/orcapod/user_types.py +++ b/src/orcapod/user_types.py @@ -5,8 +5,9 @@ that agree on all three are the same type: results stored with one are found again with the other. See ``docs/concepts/your-own-data-types.md``. -``register_type`` declares the name a class is identified by. ``set_type_naming`` sets -how classes that declare no name are named. +``register_type`` declares the name a class is identified by. Without it, a class +defined in a notebook or script (``__main__``) is named by its class name and a class +defined in a module by its module path and class name. """ from __future__ import annotations @@ -18,43 +19,57 @@ T = TypeVar("T", bound=type) -__all__ = ["register_type", "set_type_naming"] +__all__ = ["register_type"] @overload -def register_type(cls: T, /, *, name: str | None = None) -> T: ... +def register_type( + cls: T, /, *, name: str | None = None, qualified: bool | None = None +) -> T: ... @overload -def register_type(*, name: str | None = None) -> Callable[[T], T]: ... +def register_type( + *, name: str | None = None, qualified: bool | None = None +) -> Callable[[T], T]: ... def register_type( - cls: T | None = None, /, *, name: str | None = None + cls: T | None = None, + /, + *, + name: str | None = None, + qualified: bool | None = None, ) -> T | Callable[[T], T]: """Declare the name a dataclass or pydantic model is identified by. - Use it as a decorator, with or without a name, or call it on a class you do not - own. Without a name, the class is named by the naming setting (see - ``set_type_naming``): a class defined in a notebook or script by its class name, - a class defined in a module by its module path and class name. + Use it as a decorator, with or without arguments, or call it on a class you do + not own. The class is never modified: the name is recorded in Orcapod's registry + and the same class object is returned. + + Without arguments the class keeps its default name: its class name if it is + defined in a notebook or script, its module path and class name if it is defined + in a module. ``qualified=False`` names a module class by its class name alone, + without retyping it. - Declare the name before the class is first used in a source or pod. A class - defined again with the same name and fields (a re-run notebook cell) is the same - type; with different fields it is a different type. + Declare the name before the class is first used in a source or pod. Declaring the + same class again with the same name does nothing; a different name raises. Args: cls: The class, when called directly. name: The declared name, e.g. ``"Point"`` or ``"geo.Point"``. It must not be empty, contain ``@`` or whitespace, or start with ``orcapod.``. + qualified: Only without ``name``. ``False``: the class name (``"Point"``). + ``True``: the module path and class name (``"pkg.models.Point"``). Returns: The class unchanged, or a decorator that returns it unchanged. Raises: TypeError: If the class is not a dataclass or pydantic model. - ValueError: If the name is invalid, the class is defined inside a function, or - the class was already used under a different name. + ValueError: If both ``name`` and ``qualified`` are given, the name is invalid, + the class is defined inside a function, or the class is already + registered under a different name. Example: >>> import dataclasses @@ -67,31 +82,9 @@ def register_type( """ def decorate(target: T) -> T: - _uci.declare(target, name) + _uci.declare(target, name, qualified=qualified) return target if cls is None: return decorate return decorate(cls) - - -def set_type_naming( - *, main: _uci.NamingStyle | None = None, module: _uci.NamingStyle | None = None -) -> None: - """Set how dataclasses and pydantic models that declare no name are named. - - Applies to classes first used after the call, so call it before defining or using - your classes. - - Args: - main: Classes defined in a notebook or a script run directly (``__main__``): - ``"name"`` (default) uses the class name (``"Point"``); ``"qualified"`` - uses ``"__main__.Point"``. - module: Classes defined in an importable module: ``"qualified"`` (default) - uses the module path and class name (``"pkg.models.Point"``); ``"name"`` - uses the class name only. - - Raises: - ValueError: If a value is not ``"name"`` or ``"qualified"``. - """ - _uci.set_naming(main=main, module=module) diff --git a/tests/test_hashing/golden/data/datagram.json b/tests/test_hashing/golden/data/datagram.json index 4a93fe769..641fb51cb 100644 --- a/tests/test_hashing/golden/data/datagram.json +++ b/tests/test_hashing/golden/data/datagram.json @@ -89,65 +89,65 @@ "_tag_source_id::b58c50c5ff232cdb71eb0272547485074c9784600f5a34218434f7c3de419e8e", "_tag_record_id::b58c50c5ff232cdb71eb0272547485074c9784600f5a34218434f7c3de419e8e" ], - "dataclass.content_hash": "semantic_v0.1:30806932620b0715e691bb1eaa74f58e587f5e6eeef08833cff14c9d305411f4", - "dataclass.pipeline_hash": "semantic_v0.1:32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635", - "dataclass.row[{\"k\":\"r1\"}]._tag_record_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635": "26db93a5eab45b0485c2699148b0600d", - "dataclass.row[{\"k\":\"r1\"}]._tag_source_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635": "golden_dataclass", - "dataclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000135da7ac200e567d06cc849537f7726196f576b8498fc7e46bf0849740f67c692", - "dataclass.row[{\"k\":\"r2\"}]._tag_record_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635": "71d5c6ce2f7b5b5b9dfdc7a61a7df366", - "dataclass.row[{\"k\":\"r2\"}]._tag_source_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635": "golden_dataclass", - "dataclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:00000106c5a4a7b5a8d235e66b3aad694a92cdc826c5b416f3148ba93e22767407f564", + "dataclass.content_hash": "semantic_v0.1:c5a9e1b35bfc3292f482d92c6da779430f90266b4969471c3c9dda92babb38e4", + "dataclass.pipeline_hash": "semantic_v0.1:7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e", + "dataclass.row[{\"k\":\"r1\"}]._tag_record_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e": "26db93a5eab45b0485c2699148b0600d", + "dataclass.row[{\"k\":\"r1\"}]._tag_source_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e": "golden_dataclass", + "dataclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001bf6f92863abf8af4b0aac7124cdef0ddd5389e4d8cf951ac5ec452517f8a073b", + "dataclass.row[{\"k\":\"r2\"}]._tag_record_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e": "71d5c6ce2f7b5b5b9dfdc7a61a7df366", + "dataclass.row[{\"k\":\"r2\"}]._tag_source_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e": "golden_dataclass", + "dataclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b0d45b9c6372b65c8a2be29561177153c1711e2ed1aab8323d83cfc084d52564", "dataclass.system_tag_columns": [ - "_tag_source_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635", - "_tag_record_id::32fb95c409827fcc6d4c6adea25bb26ec7a68fb56139f8c2315c9a8be40d7635" + "_tag_source_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e", + "_tag_record_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e" ], - "dataclass_declared.content_hash": "semantic_v0.1:7e97c35e68612f04276d8de55d9a5232c377aec9ff1ae10e5fa2dd3bc5507f16", - "dataclass_declared.pipeline_hash": "semantic_v0.1:6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c", - "dataclass_declared.row[{\"k\":\"r1\"}]._tag_record_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c": "570e538241085842a7994cad88c5a26b", - "dataclass_declared.row[{\"k\":\"r1\"}]._tag_source_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c": "golden_dataclass_declared", - "dataclass_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001c4cb62165dfd97cc7f0c1949354ba602436ffa8e2f0517fd922e87588cde5782", - "dataclass_declared.row[{\"k\":\"r2\"}]._tag_record_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c": "0e35aec1a9605d428b2741d9f13012d5", - "dataclass_declared.row[{\"k\":\"r2\"}]._tag_source_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c": "golden_dataclass_declared", - "dataclass_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b2cf415f684673e3fa593084372b3b8b0bb165bc82619fde3ba44e7e78ef2001", + "dataclass_declared.content_hash": "semantic_v0.1:e6245cec578022a262899f3887df76aa88814cd15febaf5d0e1da03c21fef5d7", + "dataclass_declared.pipeline_hash": "semantic_v0.1:7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf", + "dataclass_declared.row[{\"k\":\"r1\"}]._tag_record_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf": "570e538241085842a7994cad88c5a26b", + "dataclass_declared.row[{\"k\":\"r1\"}]._tag_source_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf": "golden_dataclass_declared", + "dataclass_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000100c60f00122ee9d0689aa5f61af45feeaeaa37c636cd91f711ccaf5eeab6ff7d", + "dataclass_declared.row[{\"k\":\"r2\"}]._tag_record_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf": "0e35aec1a9605d428b2741d9f13012d5", + "dataclass_declared.row[{\"k\":\"r2\"}]._tag_source_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf": "golden_dataclass_declared", + "dataclass_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:0000016146a4d84d42a55151513f89988bc964b7c813f2384484a2ddaa0e8249e348d5", "dataclass_declared.system_tag_columns": [ - "_tag_source_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c", - "_tag_record_id::6d8c984d8b961f08f37441d936724b86e1b441a42daf7b106570630327b99b0c" + "_tag_source_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf", + "_tag_record_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf" ], - "dataclass_nested.content_hash": "semantic_v0.1:8887fd4cfaf912c84ccbf49786814c789b8ea972774056c132dadf0106016a22", - "dataclass_nested.pipeline_hash": "semantic_v0.1:839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665", - "dataclass_nested.row[{\"k\":\"r1\"}]._tag_record_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665": "3b11b0ab021d59f28c8efaf02ad69aee", - "dataclass_nested.row[{\"k\":\"r1\"}]._tag_source_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665": "golden_dataclass_nested", - "dataclass_nested.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000014be5231f5cdac2ab183a8616ac8e7390385a21c5ee2efdcc951f4feb81c05ddd", - "dataclass_nested.row[{\"k\":\"r2\"}]._tag_record_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665": "178f8da0e9ed5520827e648f5a890193", - "dataclass_nested.row[{\"k\":\"r2\"}]._tag_source_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665": "golden_dataclass_nested", - "dataclass_nested.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:00000111562233d9e99e74ad15355aafd40b2e377e4a0799db72330416a082490552d9", + "dataclass_nested.content_hash": "semantic_v0.1:6980d157081d777cdab23f84ed44901566e0f0b07977b980a4cc2c4af8fc327e", + "dataclass_nested.pipeline_hash": "semantic_v0.1:ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6", + "dataclass_nested.row[{\"k\":\"r1\"}]._tag_record_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6": "3b11b0ab021d59f28c8efaf02ad69aee", + "dataclass_nested.row[{\"k\":\"r1\"}]._tag_source_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6": "golden_dataclass_nested", + "dataclass_nested.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000013e4d70f87128e09feac88ca6a1473bb286b74369f99e120b7eef26465e0b83e2", + "dataclass_nested.row[{\"k\":\"r2\"}]._tag_record_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6": "178f8da0e9ed5520827e648f5a890193", + "dataclass_nested.row[{\"k\":\"r2\"}]._tag_source_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6": "golden_dataclass_nested", + "dataclass_nested.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001a4947a9cee130966d37ef41907b9c4704a87bc4e9521356953b3e6629ce00e2b", "dataclass_nested.system_tag_columns": [ - "_tag_source_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665", - "_tag_record_id::839d02c4c871eea07cd2b8fbe59963241153d2a5ea2eabb57562fa46961c9665" + "_tag_source_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6", + "_tag_record_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6" ], - "dataclass_twin_ab.content_hash": "semantic_v0.1:313370fb33d9bea1af19c4fb2c8206a97a3e26186f686930e51bd1514bb63616", - "dataclass_twin_ab.pipeline_hash": "semantic_v0.1:95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e", - "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "f35be6f60b275f2c81fbc49366c2db67", - "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "golden_dataclass_twin_ab", - "dataclass_twin_ab.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001a4f0170a2d724ce2395c6ab6595053a9482d58e05eac5b3311d9e6b56e4f80f4", - "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "7307d67233695c9ab4952175e9735bc9", - "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "golden_dataclass_twin_ab", - "dataclass_twin_ab.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001721c4768261afb271b69fef8680f8df75e5ab8d3494bc8da048ec1322a18fd25", + "dataclass_twin_ab.content_hash": "semantic_v0.1:bfe47ccef93a7e2eb6f9becbe4325d4467422070c4a325c73a94a64e3194250a", + "dataclass_twin_ab.pipeline_hash": "semantic_v0.1:9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc", + "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "f35be6f60b275f2c81fbc49366c2db67", + "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "golden_dataclass_twin_ab", + "dataclass_twin_ab.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000125e527402024a34d9d53489982ae2634a022bc6f9a0b8249606df28a66aa118c", + "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "7307d67233695c9ab4952175e9735bc9", + "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "golden_dataclass_twin_ab", + "dataclass_twin_ab.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f0b425f53eb41258940eaf8c37d0c18b1f3be4ecbde6969cba18ee8929013a63", "dataclass_twin_ab.system_tag_columns": [ - "_tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e", - "_tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e" + "_tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc", + "_tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc" ], - "dataclass_twin_ba.content_hash": "semantic_v0.1:9dd09b862ed11da8904ae2df1d7224394adea497e46773b2aa5778448583aeda", - "dataclass_twin_ba.pipeline_hash": "semantic_v0.1:95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e", - "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "65d4fb8402a75d059b3fbfa686a22420", - "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "golden_dataclass_twin_ba", - "dataclass_twin_ba.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001a4f0170a2d724ce2395c6ab6595053a9482d58e05eac5b3311d9e6b56e4f80f4", - "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "3d1c61df9063598ca8178d55c4a47ff9", - "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e": "golden_dataclass_twin_ba", - "dataclass_twin_ba.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001721c4768261afb271b69fef8680f8df75e5ab8d3494bc8da048ec1322a18fd25", + "dataclass_twin_ba.content_hash": "semantic_v0.1:124abac0b10fc20bec6865c930effbc767f96a2ae8387c4974b3bddedb72caa3", + "dataclass_twin_ba.pipeline_hash": "semantic_v0.1:9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc", + "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "65d4fb8402a75d059b3fbfa686a22420", + "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "golden_dataclass_twin_ba", + "dataclass_twin_ba.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000125e527402024a34d9d53489982ae2634a022bc6f9a0b8249606df28a66aa118c", + "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "3d1c61df9063598ca8178d55c4a47ff9", + "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "golden_dataclass_twin_ba", + "dataclass_twin_ba.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f0b425f53eb41258940eaf8c37d0c18b1f3be4ecbde6969cba18ee8929013a63", "dataclass_twin_ba.system_tag_columns": [ - "_tag_source_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e", - "_tag_record_id::95a4adeb53a615cd4f439952de84c10081597823834f46ae700b2a6f0177c77e" + "_tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc", + "_tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc" ], "dataframe.content_hash": "semantic_v0.1:f18422d163082bd23e621e586c3d484ab3a25b6c1ad959b08daa4d63ef461e68", "dataframe.pipeline_hash": "semantic_v0.1:38f74931045e006f752450c2d26dbf9096c5bff65fbe952591dec83b04296a14", @@ -221,29 +221,29 @@ "_tag_source_id::bf2528f33e2c72825ecca9151c2302d77df0e6e7544e955e56101468f9b2bd09", "_tag_record_id::bf2528f33e2c72825ecca9151c2302d77df0e6e7544e955e56101468f9b2bd09" ], - "pydantic.content_hash": "semantic_v0.1:cdcdbc5ee86513eb5cb5a5ac65597283624cba0fdcaff90160ba2944e1bf1ca2", - "pydantic.pipeline_hash": "semantic_v0.1:1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad", - "pydantic.row[{\"k\":\"r1\"}]._tag_record_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad": "cb9bbf1a138d5c848e86f99bab7c0acb", - "pydantic.row[{\"k\":\"r1\"}]._tag_source_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad": "golden_pydantic", - "pydantic.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000017138b5a4d4d4ebc1cfafd17ad95a997c673a0ec614a1fb453b0b0adca1a698c9", - "pydantic.row[{\"k\":\"r2\"}]._tag_record_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad": "10c1dbaba95a5c5fa7c14661231224eb", - "pydantic.row[{\"k\":\"r2\"}]._tag_source_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad": "golden_pydantic", - "pydantic.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b32cd5f30c4ee4b7a38bce31ac7d94419a9212d33ad07e5d08757ce8e94e9118", + "pydantic.content_hash": "semantic_v0.1:69cacd7109204aa8adf23d7e4b52c392d08b1ea58410c6b3edd2443627eb698b", + "pydantic.pipeline_hash": "semantic_v0.1:0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb", + "pydantic.row[{\"k\":\"r1\"}]._tag_record_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb": "cb9bbf1a138d5c848e86f99bab7c0acb", + "pydantic.row[{\"k\":\"r1\"}]._tag_source_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb": "golden_pydantic", + "pydantic.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000141edb65540d977e17ea78d6ac46147f1a4e27de391d8e3ae733375ec0f68b0cc", + "pydantic.row[{\"k\":\"r2\"}]._tag_record_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb": "10c1dbaba95a5c5fa7c14661231224eb", + "pydantic.row[{\"k\":\"r2\"}]._tag_source_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb": "golden_pydantic", + "pydantic.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001aa87be5aef99410d9d42fe7a2feaa5c7fbe85086f5f22d99a3989183b5c691d9", "pydantic.system_tag_columns": [ - "_tag_source_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad", - "_tag_record_id::1a84b3bbf2e3fba5441f0488550fac320cf4f04057035d608e3d462552db68ad" + "_tag_source_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb", + "_tag_record_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb" ], - "pydantic_declared.content_hash": "semantic_v0.1:ec413e1781f85a762bb5fe2a55794d766e3390954c7606b9d6a1e3e47f71b1af", - "pydantic_declared.pipeline_hash": "semantic_v0.1:922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13", - "pydantic_declared.row[{\"k\":\"r1\"}]._tag_record_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13": "fec6f63bdc695be9aeddfdd100112d42", - "pydantic_declared.row[{\"k\":\"r1\"}]._tag_source_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13": "golden_pydantic_declared", - "pydantic_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001b2d14e386eb560940d8d9f54e03045b5f20f2e8cf560e22fd724fad309d233b5", - "pydantic_declared.row[{\"k\":\"r2\"}]._tag_record_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13": "89983c8f835d51c3aaf2bbca0d7f3d55", - "pydantic_declared.row[{\"k\":\"r2\"}]._tag_source_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13": "golden_pydantic_declared", - "pydantic_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f215ba71a9109b04f48d2627489608d2d83b599f0a4dd97ddc8862f298a9a0a0", + "pydantic_declared.content_hash": "semantic_v0.1:7fd221eb0acc8f9728479fd94e3cfde8f46d18909139d0fb37fdf6925736e084", + "pydantic_declared.pipeline_hash": "semantic_v0.1:fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007", + "pydantic_declared.row[{\"k\":\"r1\"}]._tag_record_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007": "fec6f63bdc695be9aeddfdd100112d42", + "pydantic_declared.row[{\"k\":\"r1\"}]._tag_source_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007": "golden_pydantic_declared", + "pydantic_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001215e94d60e5fa74938aaab850d2ae94c7b8fd2e2f685af5cb476d0c85a035fef", + "pydantic_declared.row[{\"k\":\"r2\"}]._tag_record_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007": "89983c8f835d51c3aaf2bbca0d7f3d55", + "pydantic_declared.row[{\"k\":\"r2\"}]._tag_source_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007": "golden_pydantic_declared", + "pydantic_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b4ff9032628c25bdce2bb7eb4408d46f20bbb8b69add7f19f2599c682976cf02", "pydantic_declared.system_tag_columns": [ - "_tag_source_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13", - "_tag_record_id::922544613bc92fc665c6522c877b5c589c28c6abd859a12205a7c5ddf8329f13" + "_tag_source_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007", + "_tag_record_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007" ], "scalars.content_hash": "semantic_v0.1:6b123e343edb881e71967e118f1e90b2dc46d20fb60e43b9b7991e3ef822a1d8", "scalars.pipeline_hash": "semantic_v0.1:1fdf145196d8e5dd7b38887cc8798325147df1f5c6f2af80952488a76f9aec07", diff --git a/tests/test_hashing/golden/data/schema.json b/tests/test_hashing/golden/data/schema.json index c2502b47d..e52408ed8 100644 --- a/tests/test_hashing/golden/data/schema.json +++ b/tests/test_hashing/golden/data/schema.json @@ -2,7 +2,7 @@ "compute_schema_hash.chars_16": "a3e81ce4fbd701a0", "compute_schema_hash.full": "a3e81ce4fbd701a0fb97cae134f236953f7f66a8e7b7f4741339b45fe9f485ba", "multi_field.empty": "semantic_v0.1:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - "multi_field.logical_mix": "semantic_v0.1:db02d466882c5127e1e6ab0e264865450102d30773579e666514eba27a72a1ca", + "multi_field.logical_mix": "semantic_v0.1:ab36d5d6db29f829c3aa875245fca70d17d25054bd44088fe18d900ae95d8e19", "multi_field.nullable_type": "semantic_v0.1:560844c33bb686eddb98de8d5b608f38d845ee5bf64e6a094f9a8326679aafe9", "multi_field.optional_field": "semantic_v0.1:bc030299c9a955ea2f18a2c746121180b5ed0d1fd6265c95ef243088e392b843", "multi_field.two_scalars": "semantic_v0.1:bc030299c9a955ea2f18a2c746121180b5ed0d1fd6265c95ef243088e392b843", @@ -11,16 +11,16 @@ "single_field.bool.schema_hash": "semantic_v0.1:d0845939265975cb588b4821498b4844a6ae18d9613e6bb9bd8c274db521eab0", "single_field.bytes.arrow_type": "large_binary", "single_field.bytes.schema_hash": "semantic_v0.1:c691ab0e3a0ce67c3f8d7b9065d60e2a14afcdce983b362c62de85c9dcf19a52", - "single_field.dataclass.arrow_type": "extension[struct]", - "single_field.dataclass.schema_hash": "semantic_v0.1:b57fab5e732720af0d9263f2e2885591790c4760d35675acaf084670ef2c4545", - "single_field.dataclass_declared.arrow_type": "extension[struct]", - "single_field.dataclass_declared.schema_hash": "semantic_v0.1:ec33ba184eb852e3b75c7c9e7541b8695f259aec588f541ac6a119d3198b1215", - "single_field.dataclass_nested.arrow_type": "extension[struct, tags: large_list>]", - "single_field.dataclass_nested.schema_hash": "semantic_v0.1:1aee0c1efd5f33da7984bfa837a50b7706ea65a5dc68ad05ba844efffda262d5", - "single_field.dataclass_twin_ab.arrow_type": "extension[struct]", - "single_field.dataclass_twin_ab.schema_hash": "semantic_v0.1:38f6e2b2c6dc3d14de182f2938128a353f056222e184caa202413e3b42edde1e", - "single_field.dataclass_twin_ba.arrow_type": "extension[struct]", - "single_field.dataclass_twin_ba.schema_hash": "semantic_v0.1:38f6e2b2c6dc3d14de182f2938128a353f056222e184caa202413e3b42edde1e", + "single_field.dataclass.arrow_type": "extension[struct]", + "single_field.dataclass.schema_hash": "semantic_v0.1:85e2c0a314e1cf2427c86f632cae5db5f8f04b681160b2b0408edec4c670d081", + "single_field.dataclass_declared.arrow_type": "extension[struct]", + "single_field.dataclass_declared.schema_hash": "semantic_v0.1:7c2cd6ddbdece7ff6754374e3a1a3d414e35c46ebcb0abf1600a06f4e9becec9", + "single_field.dataclass_nested.arrow_type": "extension[struct, tags: large_list>]", + "single_field.dataclass_nested.schema_hash": "semantic_v0.1:60298722254093098ecebefb67f409caf5abb8478e361c34c3e3223a4d2d8fc3", + "single_field.dataclass_twin_ab.arrow_type": "extension[struct]", + "single_field.dataclass_twin_ab.schema_hash": "semantic_v0.1:3258fb49892b288c4f9155d6405bda39c5d9e5d0c366f20bfe718beb2ff314ee", + "single_field.dataclass_twin_ba.arrow_type": "extension[struct]", + "single_field.dataclass_twin_ba.schema_hash": "semantic_v0.1:3258fb49892b288c4f9155d6405bda39c5d9e5d0c366f20bfe718beb2ff314ee", "single_field.dataframe.arrow_type": "extension[large_binary]", "single_field.dataframe.schema_hash": "semantic_v0.1:2544430b3979b1e0ad8cecb9ab45c93b91dda546d5a02a59b5587c73aa04ecd8", "single_field.dict_str_int.arrow_type": "large_list>", @@ -29,8 +29,8 @@ "single_field.float.schema_hash": "semantic_v0.1:1d165501d30136c6e1309c58db930165e0cb51aae8469323283a95297a880191", "single_field.int.arrow_type": "int64", "single_field.int.schema_hash": "semantic_v0.1:2faa1297b96c1248edfafb2422fded3963672852acbe2973155351859eab83d2", - "single_field.list_dataclass.arrow_type": "extension[large_list>]", - "single_field.list_dataclass.schema_hash": "semantic_v0.1:d078161a5775aafe2afde86369e2cc57dcdf00ed010292015a60147b8eee1c47", + "single_field.list_dataclass.arrow_type": "extension[large_list>]", + "single_field.list_dataclass.schema_hash": "semantic_v0.1:29d0eaf9ac1816d31044a9cacc2115d49848602457a8ca84c556b2203638767d", "single_field.list_int.arrow_type": "large_list", "single_field.list_int.schema_hash": "semantic_v0.1:906c18997d1191d374db82ea5cc763d3d04dc3ceccb4f3fde82cf6d53d0f8dc7", "single_field.list_list_int.arrow_type": "large_list>", @@ -43,10 +43,10 @@ "single_field.optional_int.schema_hash": "semantic_v0.1:16d9f93d8da8f8c5a582352f7022b1191ab1e4593722bbd37fdc1238d34cdc13", "single_field.path.arrow_type": "extension[large_string]", "single_field.path.schema_hash": "semantic_v0.1:84e095c4212af644cc68357daa75a3b9094b953c7fb9030fb4af86f8310b30c3", - "single_field.pydantic.arrow_type": "extension[struct]", - "single_field.pydantic.schema_hash": "semantic_v0.1:259566a29cf776cf97fd5c6a0201a8026b3d41644a24c2cb1d3b7544507be4ad", - "single_field.pydantic_declared.arrow_type": "extension[struct]", - "single_field.pydantic_declared.schema_hash": "semantic_v0.1:d05e37ea7d328efbd2833d604e9cff59e1b1ca5f9b09133bb37e9ba524c94896", + "single_field.pydantic.arrow_type": "extension[struct]", + "single_field.pydantic.schema_hash": "semantic_v0.1:05c0f155ab1bc6cffbcc73d225a9b7413cebca8521e5f4f5a821fcdf60b3e07a", + "single_field.pydantic_declared.arrow_type": "extension[struct]", + "single_field.pydantic_declared.schema_hash": "semantic_v0.1:ab6ecbf3038e3d789a57e5032fad7432e8accedde4a5c2c0a52cecab7ecf53bb", "single_field.set_uuid.arrow_type": "extension[large_list]", "single_field.set_uuid.schema_hash": "semantic_v0.1:73501f29f5a260ab092f67ea9d88c8c3d5d8d5c90bba8d9aea88b3b5400c6272", "single_field.str.arrow_type": "large_string", diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/semantic.json index ea3156065..8388fde1c 100644 --- a/tests/test_hashing/golden/data/semantic.json +++ b/tests/test_hashing/golden/data/semantic.json @@ -1,24 +1,24 @@ { "type.bool": "semantic_v0.1:cd8f971312b872f0e9a3ef40aa58d6bb537258e72a9a142e0cf712f0a35020e3", "type.bytes": "semantic_v0.1:f65cea8d1f29ca5b96d67caf1ced2f630b509e32574a7b78b9071958818a1c59", - "type.dataclass": "semantic_v0.1:814071362ffe7481bde7e4173907a59d5bc6df940f81c9838764d0f83973abbe", - "type.dataclass_declared": "semantic_v0.1:fee6ecce526f9b6791a66e6bb638a4ba0934e79beff9fe247bf71807224fc786", - "type.dataclass_nested": "semantic_v0.1:214904a85c3d6015ee0b2e83dceba5852d3196dbb295b0f4550198a707c23ad1", - "type.dataclass_twin_ab": "semantic_v0.1:921bd905da2fce576b9a8d0274b9716e5cba49edefd123b955c8ebcffe24835c", - "type.dataclass_twin_ba": "semantic_v0.1:921bd905da2fce576b9a8d0274b9716e5cba49edefd123b955c8ebcffe24835c", + "type.dataclass": "semantic_v0.1:a3986cabd66cbb48db7e99acc9b24ac00b7683707f8b3276e90b6b4f5a08405e", + "type.dataclass_declared": "semantic_v0.1:1358372ae18c5582131bd9db1da741d3fa41630e6d49095d1a1c8c5856b1feae", + "type.dataclass_nested": "semantic_v0.1:83aca3a4d53a66d3fb444f38b90b2296f3b516dd889c756a2550beb21c4d8cb4", + "type.dataclass_twin_ab": "semantic_v0.1:31e10e6217c02aadbec8d61e73caf4b27453757745b1a81ce282ad22b5bda539", + "type.dataclass_twin_ba": "semantic_v0.1:31e10e6217c02aadbec8d61e73caf4b27453757745b1a81ce282ad22b5bda539", "type.dataframe": "semantic_v0.1:51e3c34453121e01086e882959cd12746b3236ff7b259a40a42d43493bf26dff", "type.dict_str_int": "semantic_v0.1:f0d36b085fa65f3368d1d98687ffcb6c80099a418419e45a77d434b37e4393dd", "type.float": "semantic_v0.1:0059c08fc7dfe19dc8d095b7e95d56e8d377bbe49ac30376a00105eca2e7c75f", "type.int": "semantic_v0.1:24be397c58c33686e96b4f85a9f443fd2483564ec7dee1c14b96da3e60ec16ac", - "type.list_dataclass": "semantic_v0.1:9568f2ffe6c7dd3c492522b02c06d59b6f57a6888e0db1d59db41dcac1bcf363", + "type.list_dataclass": "semantic_v0.1:a223a6db21a8c2edd5e052457b483251e104e92fd9d8523ab12850a9cffc9f11", "type.list_int": "semantic_v0.1:5c376e29b0cfebd85277c71c07c9996afaf06fab35f81681b82f705c9feafa3e", "type.list_list_int": "semantic_v0.1:ba62756844df6c82b48a739fadd6bb568a3507896dc5dba192dd3e0285635c48", "type.list_path": "semantic_v0.1:e2029c1032fad111f798363e1f718ecde715573c690c3cc78f33824a246a82ce", "type.ndarray": "semantic_v0.1:b6f37b2016cf3ee3062c3e715bab25fa02b68e92931bce243684300ba1dda5b6", "type.optional_int": "semantic_v0.1:91bc2bd8f4f083d6cbf6157350e693dd52b0bc06c57b7719db8581712098cd73", "type.path": "semantic_v0.1:02a12969616ecd278958ffca715be2865fe4a32c27523059df37245c5048a880", - "type.pydantic": "semantic_v0.1:76a4fd9c0acc945dfba19bb621c91b9eb9f6361df50e6f17a38a3ae7b0b947bc", - "type.pydantic_declared": "semantic_v0.1:a18f664d19b35c92b7a12ac983973d9aedb5e8c86f23a168207f3eee3225cc24", + "type.pydantic": "semantic_v0.1:0a50adf8aebe578185ea609609b24c84aebe09ad403f0d1241e2604af85bdbc8", + "type.pydantic_declared": "semantic_v0.1:7e15bd38029dd3d24767d4a5a53443fe624f39292c0eddaf98477c1a23be5511", "type.set_uuid": "semantic_v0.1:0ac1b6672ab001f6ee64f406e6d1af3a21024d6f78ef0c7414014cdfad96eaf6", "type.str": "semantic_v0.1:309d607de6633b959ae5118171f079b2830f4b1340064fac065652e305b038ad", "type.tuple_int_str": "semantic_v0.1:65ac71aa155fbfa28072adba04ed55d8d99d89e9c87a0d5ed45ccd15fcafc099", diff --git a/tests/test_user_types/test_identity_record.py b/tests/test_user_types/test_identity_record.py index 26f98ea0e..90722d343 100644 --- a/tests/test_user_types/test_identity_record.py +++ b/tests/test_user_types/test_identity_record.py @@ -9,7 +9,6 @@ import dataclasses import datetime import typing -from collections.abc import Iterator from pathlib import Path from typing import Literal, Optional @@ -24,13 +23,6 @@ def converter(): return get_default_context().type_converter -@pytest.fixture(autouse=True) -def _restore_naming() -> Iterator[None]: - before = uci.get_naming() - yield - uci.set_naming(**before) - - def _make_class(module: str, qualname: str = "Point") -> type: cls = dataclasses.make_dataclass(qualname, [("x", int)]) cls.__module__ = module @@ -41,7 +33,9 @@ def _make_class(module: str, qualname: str = "Point") -> type: class TestIdentityString: def test_documented_example(self) -> None: record = uci.identity_record("dataclass", "Point", {"x": "int", "y": "float"}) - assert uci.identity_string(record) == "Point@9117beaaf7ebe3ab" + assert uci.identity_string(record) == ( + "Point@9117beaaf7ebe3ab0bd25de9448c92eb64aa070a58611517bd1e470a544584d8" + ) def test_canonical_json_is_sorted_and_compact(self) -> None: record = uci.identity_record("pydantic", "P", {"b": "int", "a": "str"}) @@ -125,17 +119,12 @@ def test_main_class_defaults_to_name(self, module: str) -> None: def test_nested_main_class_uses_qualname(self) -> None: assert uci.resolve_name(_make_class("__main__", "Outer.Point")) == "Outer.Point" - def test_main_qualified_setting(self) -> None: - uci.set_naming(main="qualified") - assert uci.resolve_name(_make_class("__main__")) == "__main__.Point" - - def test_module_name_setting(self) -> None: - uci.set_naming(module="name") - assert uci.resolve_name(_make_class("pkg.models")) == "Point" + def test_default_name_qualified_false(self) -> None: + assert uci.default_name(_make_class("pkg.models"), qualified=False) == "Point" - def test_invalid_setting_rejected(self) -> None: - with pytest.raises(ValueError, match="main"): - uci.set_naming(main="short") # pyright: ignore[reportArgumentType] + def test_default_name_qualified_true_for_main(self) -> None: + cls = _make_class("__main__") + assert uci.default_name(cls, qualified=True) == "__main__.Point" def test_declared_name_wins(self) -> None: cls = _make_class("pkg.models") diff --git a/tests/test_user_types/test_register_type.py b/tests/test_user_types/test_register_type.py index 204c78ac3..ab9daf1d2 100644 --- a/tests/test_user_types/test_register_type.py +++ b/tests/test_user_types/test_register_type.py @@ -1,4 +1,4 @@ -"""Public API for user class identity: ``op.register_type`` and ``op.set_type_naming`` (ITL-881). +"""Public API for user class identity: ``op.register_type`` (ITL-881). Identity is observed through the public interface: the Arrow extension name of a ``DictSource`` column (``as_table``), the source's ``pipeline_hash`` and values read back @@ -9,7 +9,6 @@ from __future__ import annotations import dataclasses -from collections.abc import Iterator from typing import Any import pydantic @@ -17,9 +16,6 @@ import orcapod as op from orcapod.errors import RecursiveTypeError -from orcapod.logical_types import ( - user_class_identity as uci, -) # get_naming: restore the policy from orcapod.sources import DictSource @@ -113,23 +109,45 @@ def test_declaring_after_use_under_another_name_raises() -> None: op.register_type(UsedFirst, name="itl881.UsedFirst") +@op.register_type(qualified=False) @dataclasses.dataclass -class RenamedByPolicy: +class ShortNamed: x: int -def test_set_type_naming_module_name() -> None: - before = uci.get_naming() - op.set_type_naming(module="name") - try: - assert _name_part(RenamedByPolicy(1)) == "RenamedByPolicy" - finally: - op.set_type_naming(**before) +def test_qualified_false_names_a_module_class_by_its_class_name() -> None: + assert _name_part(ShortNamed(1)) == "ShortNamed" -def test_set_type_naming_rejects_unknown_style() -> None: - with pytest.raises(ValueError, match="'name' or 'qualified'"): - op.set_type_naming(main="short") # pyright: ignore[reportArgumentType] +def test_name_and_qualified_together_raise() -> None: + with pytest.raises(ValueError, match="either name"): + op.register_type(Undeclared, name="itl881.X", qualified=False) + + +@dataclasses.dataclass +class DeclaredTwice: + x: int + + +def test_redeclaring_with_the_same_name_is_a_no_op_and_another_name_raises() -> None: + op.register_type(DeclaredTwice, name="itl881.Twice") + assert op.register_type(DeclaredTwice, name="itl881.Twice") is DeclaredTwice + with pytest.raises(ValueError, match="already registered as 'itl881.Twice'"): + op.register_type(DeclaredTwice, name="itl881.Other") + + +@dataclasses.dataclass +class NotMutated: + x: int + + +def test_registration_does_not_modify_the_class() -> None: + before = dict(vars(NotMutated)) + params = NotMutated.__dataclass_params__ # pyright: ignore[reportAttributeAccessIssue] + op.register_type(NotMutated, name="itl881.NotMutated") + _type_name(NotMutated(1)) + assert dict(vars(NotMutated)) == before + assert NotMutated.__dataclass_params__ is params # pyright: ignore[reportAttributeAccessIssue] # --------------------------------------------------------------------------- @@ -352,10 +370,3 @@ def test_subclass_has_its_own_identity_and_keeps_its_fields() -> None: assert _type_name(Base(1)) != _type_name(Child(1, 2)) _, data = next(iter(_source(Child(1, 2)).iter_data())) assert data.as_dict()["p"] == Child(1, 2) - - -@pytest.fixture(autouse=True) -def _no_policy_leak() -> Iterator[None]: - before = uci.get_naming() - yield - assert uci.get_naming() == before From ac55871691636aae436ef7a23d137bfc4a81e37d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:49:07 -0800 Subject: [PATCH 09/31] docs: your own data types page, identity section, CHANGELOG and spec update (ITL-881) Adds docs/concepts/your-own-data-types.md (runnable samples: cell re-run, field order, changed fields, declared name), an identity section in identity.md, links and nav. The spec records the developer decisions (full hash, no naming switch, no class mutation), the measured golden diff and an appendix on the alternative not pursued. CHANGELOG lists the intended hash changes and what existing stores do under v0.3. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 44 ++++ DESIGN_ISSUES.md | 29 +++ docs/concepts/function-pods.md | 2 + docs/concepts/identity.md | 20 ++ docs/concepts/your-own-data-types.md | 214 ++++++++++++++++++ mkdocs.yml | 1 + ...-11-10-itl-881-user-class-identity-impl.md | 5 + .../2026-11-10-user-class-identity-v0.3.md | 190 ++++++++++------ 8 files changed, 437 insertions(+), 68 deletions(-) create mode 100644 docs/concepts/your-own-data-types.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a71b50c0..a3af187f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ ### New +#### `op.register_type` — declare the name of your own data types (ITL-881) + +`@op.register_type(name="Point")` declares the name a dataclass or pydantic model is +identified by, wherever the class is defined. `@op.register_type(qualified=False)` names a +module class by its class name alone. Registration never modifies the class; declaring the +same class under a second name raises `ValueError`. See +[Your Own Data Types](docs/concepts/your-own-data-types.md). + +Re-running a notebook cell that defines a dataclass or pydantic model, or reloading its +module, no longer breaks later pods with `ValueError: … already bound`: a class with the +same name and fields is the same type, and its stored results are found again. + #### `orcapod.File` — content-identified file type (ITL-450) `orcapod.File` is a new first-class type for materialized files. It subclasses @@ -115,6 +127,38 @@ replaced by `_`. Correcting stored values is tracked in ITL-853. ### Breaking Changes +#### Dataclasses and pydantic models are identified by name, kind and fields (ITL-881) + +Orcapod 0.3 identifies a dataclass or pydantic model by its name, its kind (dataclass or +pydantic) and its fields' names and types, in any order. The name is the class name for a +class defined in a notebook or script (`__main__`), the module path and class name for a +class defined in a module, or the name declared with `op.register_type`. + +- **Hashes change for every pipeline that uses such a class.** The class's logical type + name and Arrow extension name become `Name@`, and the + stored struct lists fields in name order. Every schema hash, pipeline hash, table path, + system-tag column name and data hash that involves a dataclass or pydantic value moves. + The first run under 0.3 computes those pod and operator results again, into new tables. + Tables written by earlier versions are left untouched and are not read by 0.3 pipelines; + reading such a column directly raises `TypeIdentityError`. There is no migration tool. + Results that involve no dataclass or pydantic value are unaffected. +- **Changing a class's fields makes a new type.** Its pods get new tables; the results of + the old definition stay where they are. Before, a changed class shared the old tables and + a run returned 0 rows with only log warnings. +- **A stored shape that disagrees with the registered class raises.** `DeltaTableDatabase` + re-raises the new `orcapod.errors.TypeIdentityError` instead of logging the read failure + and returning no records. Reading a column whose class cannot be found raises + `ImportError` naming the column, the type and the `op.register_type` call to declare it. +- **A subclass is its own type.** A dataclass or pydantic subclass was stored as its + registered parent, silently dropping its extra fields; it now has its own identity. +- **Recursive classes** raise `RecursiveTypeError` (a `TypeError`) naming the field path, + for example `Node.next -> Node`. +- Pinned golden hash entries changed (approved): `semantic.json` `type.dataclass`, + `type.pydantic`; `schema.json` `multi_field.logical_mix`, + `single_field.{dataclass,pydantic}.{arrow_type,schema_hash}`; `datagram.json` + `{dataclass,pydantic}.{content_hash,pipeline_hash,system_tag_columns}` and the per-row + `data_hash`, `_tag_record_id::…` and `_tag_source_id::…` entries of both cases. + #### Operator outputs keep their inputs' Arrow types, normalized to large variants (ITL-832, ITL-813) `MergeJoin` no longer infers Arrow types from Python values, and it no longer fails on a diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index 5b167d8de..a748e7c57 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2208,3 +2208,32 @@ subprocesses with different `PYTHONHASHSEED` values and were verified identical 3.11, 3.12 and 3.13. The generator's header says when regeneration is legitimate: a changed value needs a migration assessment and the developer's approval. Not pinned yet: the `StarfixArrowHasher` gaps tracked in ITL-848. + +--- + +## `src/orcapod/logical_types/` — identity of user classes (dataclasses, pydantic models) + +### UCI1 — A user class was identified by `module.qualname` only, without its fields +**Status:** resolved +**Severity:** high +**Issue:** ITL-881 + +A dataclass or pydantic model was identified by `f"{module}.{qualname}"`. The same class +defined in a notebook (`__main__`) and in a module were different types; two classes with +the same name and different fields shared pipeline hashes and tables, so a changed class +read the old table, failed, and the run returned 0 rows with only log warnings; re-running +the notebook cell that defines a class raised `ValueError: … already bound` until a kernel +restart; a subclass was stored as its parent, dropping its extra fields. + +**Fix:** a user class is identified by `{kind, name, fields}` (spec +`superpowers/specs/2026-11-10-user-class-identity-v0.3.md`). The identity string +`Name@` is the logical type name and Arrow extension +name, with the record in the extension metadata; stored structs list fields in name order. +`op.register_type` declares names without modifying the class; `__main__` classes default +to their class name. The registry rebinds an unchanged identity to a re-defined class and +matches user classes by exact class only. The read path finds the class by kind and name, +checks the identity, and raises `ImportError` or `TypeIdentityError` naming the column; +`DeltaTableDatabase` re-raises `TypeIdentityError`. Resolves the entry PR #288 proposes for +the same issue. Not covered: computed fields (deferred by the developer), local classes, +Enum/NamedTuple/TypedDict. + diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index b8618052c..28928b3d9 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -276,3 +276,5 @@ primary user-facing APIs. - [Operators](operators.md) handle structural transforms (the complement of function pods) - Function pods participate in the [identity chain](identity.md) -- each pod's hash includes the function's identity and its upstream pipeline hashes +- Function pods can take and return dataclasses and pydantic models; see + [Your Own Data Types](your-own-data-types.md) for when two classes count as the same type diff --git a/docs/concepts/identity.md b/docs/concepts/identity.md index 6bf7a2b5a..d845c146d 100644 --- a/docs/concepts/identity.md +++ b/docs/concepts/identity.md @@ -171,6 +171,24 @@ print(step.pod(source).pipeline_hash() == original) When you change what a function computes, bump its major version (or rename it) so that stale results are not reused. +## Your own classes in the identity + +A dataclass or pydantic model enters both chains through its **identity**: its name, its +kind (dataclass or pydantic) and the names and types of its fields, in any order. The +identity is part of every schema hash, so it reaches the pipeline hash of each source, +pod and node that uses the class, and with it the database table paths. Each stored value +is hashed together with it, so it also reaches the content hash. + +Two consequences: + +- Two classes with the same name, kind and fields share results, wherever each is + defined. A class re-defined in a notebook cell with unchanged fields keeps its cache. +- A class whose fields change is a new type: its pods get new pipeline hashes and new + tables, and the results stored with the old definition stay where they are. + +How a class gets its name, and how to keep results when moving a class from a notebook +into a module, is described in [Your Own Data Types](your-own-data-types.md). + ## How it connects to other concepts - [Sources](sources.md) form the base case of the Merkle chain (schema-only for pipeline hash, @@ -179,3 +197,5 @@ stale results are not reused. - [Operators](operators.md) contribute their own identity to the chain - [Function Pods](function-pods.md) run inside a `PipelineJob` use content hashes for memoization and pipeline hashes for DB scoping +- [Your Own Data Types](your-own-data-types.md) explains the identity of dataclasses and + pydantic models diff --git a/docs/concepts/your-own-data-types.md b/docs/concepts/your-own-data-types.md new file mode 100644 index 000000000..a56ca5d8b --- /dev/null +++ b/docs/concepts/your-own-data-types.md @@ -0,0 +1,214 @@ +# Your Own Data Types + +Function pods can take and return your own classes: Python +[dataclasses](https://docs.python.org/3/library/dataclasses.html) and +[pydantic](https://docs.pydantic.dev/) models. Orcapod stores their values in Arrow +columns and caches results computed from them like any other value. + +Caching needs to know when two classes are **the same type**. This page explains how +Orcapod decides, and what that means for notebooks, scripts and modules. + +## What makes two classes the same type + +Orcapod identifies a dataclass or pydantic model by three things: + +| Component | What it is | +|---|---| +| **Name** | The class name for a class defined in a notebook or a script, the module path and class name for a class defined in a module, or a name you declare with `op.register_type` | +| **Kind** | Dataclass or pydantic model. A dataclass and a pydantic model are never the same type, even with the same name and fields | +| **Fields** | The name and type of every stored field, in any order | + +Two classes that agree on all three are the same type: results stored with one are found +again with the other. Anything else about the class does not count: field order, +defaults, methods, properties, pydantic computed fields and validators, docstrings. + +Changing a field's name or type makes a **new type**. Results computed with the old +definition stay in the database, and the new type computes its results into new tables, +so a stored value is never read back with fields it was not written with. + +## Defining and redefining a class in a notebook + +Define a class, use it in a pod and run a job: + +```python +from dataclasses import dataclass + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import DictSource + + +@dataclass +class Point: + x: float + y: float + + +calls = [] + + +@function_pod(output_keys="norm") +def norm(p: Point) -> float: + calls.append(p) # only here to show when the function really runs + return (p.x**2 + p.y**2) ** 0.5 + + +db = InMemoryArrowDatabase() + + +def run(point_class): + rows = [ + {"k": "a", "p": point_class(x=3.0, y=4.0)}, + {"k": "b", "p": point_class(x=6.0, y=8.0)}, + ] + job = PipelineJob(name="points", store=db) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="pts") + norm.pod(source, label="norm") + job.run() + return job.nodes["norm"] + + +print(run(Point).as_table().to_pandas()) +# k norm +# 0 a 5.0 +# 1 b 10.0 +print("function calls:", len(calls)) +# function calls: 2 +``` + +Running the cell that defines `Point` again creates a new class object with the same +name and fields. It is the same type, so the next run finds the stored results. The +same holds when the fields are listed in another order: + +```python +@dataclass +class Point: + y: float + x: float + + +run(Point) +print("function calls:", len(calls)) +# function calls: 2 +``` + +Adding, removing or retyping a field makes a new type. Re-run the pod cell too, so that `norm` takes +the new `Point`; the run then computes into new tables: + +```python +@dataclass +class Point: + x: float + y: float + label: str = "" + + +@function_pod(output_keys="norm") +def norm(p: Point) -> float: + calls.append(p) + return (p.x**2 + p.y**2) ** 0.5 + + +run(Point) +print("function calls:", len(calls)) +# function calls: 4 +``` + +## How a class is named + +The name is part of the identity, so it decides whether results survive moving code +around: + +- A class defined in a **notebook or a script** run directly (`__main__`) is named by its + class name: `Point`. +- A class defined in a **module** is named by its module path and class name: + `pkg.models.Point`. + +So results computed with `Point` in a notebook are **not** found by +`pkg.models.Point` after you move the class into a module, unless you give the module +class the same name. Declare it with `op.register_type`: + +```python +import orcapod as op + + +@op.register_type(name="Point") +@dataclass +class ModulePoint: + x: float + y: float + label: str = "" + + +run(ModulePoint) +print("function calls:", len(calls)) +# function calls: 4 +``` + +`op.register_type` accepts: + +- `name="..."`: the name to use, the same wherever the class is defined. It must not be + empty, contain `@` or spaces, or start with `orcapod.`. +- `qualified=False`: name a module class by its class name alone, without retyping it. +- no arguments: keep the default name. + +It never changes your class; it records the name in Orcapod's registry and returns the +class. Declare the name before the class is first used in a source or a pod. Declaring +the same class again with the same name does nothing; declaring it under a different +name raises `ValueError`. + +You can see a class's identity on any column that holds it. The part before `@` is the +name; the rest is a hash of the name, the kind and the fields: + +```python +source = DictSource(data=[{"k": "a", "p": ModulePoint(1.0, 2.0)}], tag_columns=["k"]) +print(source.as_table().schema.field("p").type.extension_name) +# Point@... +``` + +The first time a class defined in a notebook or a script is used, Orcapod prints a +one-time warning saying that it is identified by its class name and fields. + +## Classes inside classes + +A field whose type is another dataclass or pydantic model carries that class's own +identity: its name (however that class is named or registered), its kind and its fields. +Renaming the inner class, registering it under another name, or changing its fields +therefore makes the outer class a new type too. The order in which you declare the inner +and outer classes does not matter, as long as both are declared before first use. + +A class that refers back to itself, directly (`next: Node | None`) or through other +classes, cannot be stored: Arrow has no recursive struct type. Orcapod raises +`RecursiveTypeError` naming the path, for example `Node.next -> Node`. + +## Reading stored values back + +Reading stored values needs a class with the stored name, kind and fields. Orcapod +looks for one among the classes registered or used in the current process, then in the +script or notebook (`__main__`), then by importing the name as a module path. + +- If no class with that name exists, reading raises `ImportError` naming the column, the + type and its fields, and showing the `op.register_type` call to declare it. +- If a class with that name exists but its fields differ, reading raises + `TypeIdentityError` (from `orcapod.errors`) naming the column and both sets of fields. + +Neither case is reported as missing data, and nothing stored is changed. + +## Rules and limits + +- Classes defined inside a function cannot be registered: define them at module level or + at the top of the notebook. +- Each subclass is its own type with its own fields; it is never stored as its parent. +- Two different classes registered under the same name with the same fields are the same + type. Orcapod warns once when that happens across two modules, because stored values are + then read back as the most recently registered class. +- Results stored by Orcapod versions before 0.3 identified classes by module and class + name only. Version 0.3 does not read those columns; pipelines compute their results again + into new tables and leave the old ones untouched. + +## How it connects to other concepts + +- [Function Pods](function-pods.md): the pods that take and return these classes. +- [Identity & Hashing](identity.md): how a type's identity enters pipeline and content + hashes. diff --git a/mkdocs.yml b/mkdocs.yml index dd10922a2..7e7eec44e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,6 +38,7 @@ nav: - Streams: concepts/streams.md - Operators: concepts/operators.md - Function Pods: concepts/function-pods.md + - Your Own Data Types: concepts/your-own-data-types.md - Identity & Hashing: concepts/identity.md - File Hash Caching: concepts/file-hash-caching.md - API Reference: diff --git a/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md index caf332c3e..ca4ae5671 100644 --- a/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md +++ b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md @@ -19,6 +19,11 @@ path re-raises it. **Tech Stack:** Python 3.11–3.13, pyarrow, polars, pydantic v2, deltalake, IPython (dev, for the notebook scenarios), pytest via `uv run`. +**Changes after the plan was written (developer decisions, November 10, 2026):** +`set_type_naming` was dropped (fixed location rule plus `register_type(qualified=False)`), +the identity string uses the full 64-hex digest, and stored structs list fields in name +order. The spec is authoritative where this plan differs. + --- ## File map diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md index 96252c9ed..140d0f4b8 100644 --- a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -1,6 +1,8 @@ # Identity of a user class in v0.3 (ITL-881) -**Status:** implementation spec, approved by the coordinator November 10, 2026 (P1 decided by the developer; P2 names and the identity-string format provisional). +**Status:** implemented on this branch (PR #292). Approved by the coordinator November 10, +2026; P1, the identity-string format (fold, full-length hash) and the dropped naming switch +decided by the developer the same day. **Issue:** ITL-881 (v0.3, a public release; tag target November 16, 2026). **Builds on:** the findings in PR #288 (`superpowers/specs/2026-11-10-user-class-identity.md`, experiments E01–E08). "W1"–"W15" are the workshop scenarios from §3.4 of that document. @@ -19,7 +21,10 @@ Each decision is marked **CONFIRMED** (the developer decided it, November 10, 20 | D-doc | A page documenting identity | CONFIRMED: developer answer to S4 | | P1 | A field typed by another user class carries that class's identity as that class was registered | CONFIRMED: developer decision, November 10, 2026 | | D-cycle | A self-referential or cyclic class is rejected with a clear `TypeError` | CONFIRMED by the coordinator from this proposal: Arrow cannot store a recursive struct | -| P2 | Public API names (`register_type`, `set_type_naming`) and the identity-string format | PENDING with the developer; implemented provisionally, each a single-point change | +| D-format | Identity string = name + `@` + the full 64-hex SHA-256 of the record (the "fold") | CONFIRMED: developer, November 10, 2026; alternative B (fields only in metadata) not pursued, appendix A | +| D-naming | No process-wide naming switch: fixed location rule, per-class `register_type(name=...)` or `register_type(qualified=False)` | CONFIRMED: developer dropped `set_type_naming`; a project-wide choice would live in the data context, not in v0.3 | +| D-mutation | `register_type` never modifies the class; same name again is a no-op; another name raises | CONFIRMED: developer | +| P2 | Public API name `register_type` and argument `qualified` | `register_type` accepted by the developer; `qualified=False` proposed and implemented provisionally | | D-hash | The 25 golden entries in §e move; nothing else | CONFIRMED by the coordinator: follows from decision 2; the PR lists every moved entry | | D-legacy | Reading a pre-0.3 user-class column directly raises a clear error | CONFIRMED by the coordinator: a consequence of the identity check; nothing extra is built | | D-notice | One-time notice when a `__main__` class is first registered (S3) | CONFIRMED in scope by the coordinator; last item before the cuts | @@ -37,14 +42,17 @@ A user class's identity is an **identity record** with three components: The name is a string chosen once per class, by the first rule that applies: 1. **Declared name.** The user declared one (§b). Used verbatim. -2. **`__main__` class** (`cls.__module__` is `"__main__"` or `"__mp_main__"`, the name a +2. **Per-class choice without a name:** `register_type(qualified=False)` gives + `cls.__qualname__`; `register_type(qualified=True)` gives `module.qualname`. +3. **`__main__` class** (`cls.__module__` is `"__main__"` or `"__mp_main__"`, the name a `multiprocessing` spawn child gives the main module, W14): `cls.__qualname__` - (`"Point"`, or `"Outer.Point"` for a nested class), under the default setting - `main="name"`. Setting `main="qualified"` gives `"__main__.Point"` (today's string). -3. **Module class:** `f"{module}.{qualname}"` (`"pkg.models.Point"`), under the default - setting `module="qualified"`. Setting `module="name"` gives `"Point"`. The module is + (`"Point"`, or `"Outer.Point"` for a nested class). +4. **Module class:** `f"{module}.{qualname}"` (`"pkg.models.Point"`). The module is passed through `canonical_module_name`, which leaves user modules unchanged. +Rules 3 and 4 are fixed (D-naming): no process-wide switch changes them, so a class's +identity never depends on whether or when some setting was called. + A name must be non-empty, must not contain `@` or whitespace, and must not start with `orcapod.` (reserved for built-in types such as `orcapod.path`). A class whose `__qualname__` contains `` stays rejected in v0.3, even with a declared name @@ -66,7 +74,10 @@ pydantic logical types by exact class only. `fields` maps each **stored field's name** to its **type token**. It is a JSON object serialized with sorted keys, so field order never enters. The stored fields are exactly -the ones stored today: dataclass fields with `init=True`, pydantic `model_fields`. +the ones stored today: dataclass fields with `init=True`, pydantic `model_fields`. The +Arrow storage struct also lists the fields in name order, so two classes with the same +identity always have the same storage type (found by the W11 test: with class order the +same identity had two structs, and the Delta read failed into 0 rows). Type token of an annotation (from `typing.get_type_hints`, which strips `Annotated` extras): @@ -77,8 +88,9 @@ extras): | other class without a logical type (stdlib) | `canonical_type_name(cls)` | `datetime.datetime`, `decimal.Decimal` | | type with a built-in logical type | its logical type name | `orcapod.path`, `orcapod.uuid`, `orcapod.file` | | generic container | `origin[arg, …]`, origin by its canonical name, args in order | `list[int]`, `dict[str, float]`, `tuple[int, str]` (`typing.List[int]` gives `list[int]`) | -| `Optional` / union | member tokens sorted, joined by ` \| ` | `int \| None` for `Optional[int]`, `int \| None` and `None \| int` | -| nested dataclass / pydantic model (P1) | that class's identity string (below), as that class is registered | `Inner@3c1d…`, `geo.Inner@…` | +| `Optional` / union | member tokens sorted, `None` last, joined by ` \| ` | `int \| None` for `Optional[int]`, `int \| None` and `None \| int` | +| `Literal` | `Literal[...]` with the members' `repr`s sorted | `Literal['a', 'b']` | +| nested dataclass / pydantic model (P1) | that class's identity string (below), as that class is registered | `Inner@3c1d…` (full digest), `geo.Inner@…` | **Nested classes (P1, CONFIRMED).** The token of a field typed by another user class is that class's own registered identity string: its name component however *that* class was @@ -89,17 +101,18 @@ declaring it under another name, or changing its fields changes the outer identi **Registration order does not matter.** `register_type` only records the declared name and checks it (§b); the logical type, and so the identity, is built at first use, and the -outer class's build registers the inner class first (implicitly, by the setting, if it -was never declared). Declaring every class before first use therefore gives the same +outer class's build registers the inner class first (implicitly, by the location rule, if +it was never declared). Declaring every class before first use therefore gives the same identities in any order: outer decorated before inner, inner declared later in the same cell, inner registered implicitly by the outer's first use. Declaring a class after it was used under a different name raises (§c). An identical notebook re-run of the inner class rebinds it (§c) and leaves the outer identity unchanged. **Cycles (D-cycle).** A class that refers to itself, directly (`next: Node | None`) or -through other classes, is rejected at first use with a `TypeError` naming the cycle -(`Node.next -> Node`). Today's converter already raises `TypeError: Circular type -dependency detected…`; v0.3 keeps the rejection and names the path. Encoding the +through other classes, is rejected at first use with `RecursiveTypeError` (a `TypeError` subclass) naming the +path (`Node.next -> Node`, `CycleA.b -> CycleB.a -> CycleA`). Today's converter already +raises `TypeError: Circular type dependency detected…`; v0.3 keeps the rejection and names +the path. Encoding the back-reference by name would give a well-defined identity, but Arrow cannot store a recursive struct, so there is nothing to identify. @@ -119,15 +132,25 @@ named field *types* and because the Arrow type cannot tell `str` from `orcapod.p ``` identity string = f"{name}@{fingerprint}" -fingerprint = sha256(canonical_json(record)).hexdigest()[:16] +fingerprint = sha256(canonical_json(record)).hexdigest() # full 64 hex digits canonical_json = json.dumps(record, sort_keys=True, separators=(",", ":")) ``` -`Point(x: int, y: float)` as a dataclass gives `Point@9117beaaf7ebe3ab`; with `y: int`, -`Point@c0a75f0c39b86873`; as a pydantic model with `y: float`, `Point@703bff8af8d6002c`. -SHA-256 of a documented JSON string, not the Orcapod semantic hasher, so the identity can -be reproduced by hand and does not depend on a hasher version. 16 hex digits (64 bits) -only have to separate classes with the same name and kind. +`Point(x: int, y: float)` as a dataclass gives +`Point@9117beaaf7ebe3ab0bd25de9448c92eb64aa070a58611517bd1e470a544584d8`; with `y: int`, +`Point@c0a75f0c…`; as a pydantic model with `y: float`, `Point@703bff8a…`. SHA-256 of a +documented JSON string, not the Orcapod semantic hasher, so the identity can be reproduced +by hand and does not depend on a hasher version. The full digest is used (developer): +there is no truncation rule to explain, and the name is not meant for people to type. +The length lives in one constant (`FINGERPRINT_HEX_DIGITS`) and the format in one function +(`identity_string`). + +Where the long name shows: the Arrow field metadata of stored columns, `extension<...>` in +Arrow type reprs, and Polars extension dtypes. It does not appear in table paths (those +use pipeline hashes) or in Orcapod's error messages, which describe a class as +`Point (dataclass) {x: int, y: float}`. A prefix-matching lookup for abbreviated names was +considered and not built: no API takes a logical type name, and the read path compares +full strings from stored metadata. ### Where each component enters @@ -140,7 +163,7 @@ second mechanism. | Logical type name | identity string | `DataclassLogicalType.logical_type_name` / pydantic equivalent | | Arrow extension name (`ARROW:extension:name`) | identity string | same string; one Arrow/Polars extension registration per identity | | Arrow extension metadata | category plus the full record: `{"category":"orcapod.dataclass","identity":{…}}`, canonical JSON | read path uses it to find the class and to print the stored shape | -| Arrow storage type | struct of the stored fields | unchanged (derived from the annotations) | +| Arrow storage type | struct of the stored fields, in name order | derived from the annotations | | Schema hash, source `pipeline_hash`, system-tag column names | identity string | `TypeObjectHandler` hashes a registered class as `type:` (unchanged code) | | Pod signature hash, node `pipeline_hash`, DB table paths | identity string | via the schema/signature hashes | | Per-row data hash, `content_hash` | identity string + record + values | the extension column is hashed with its extension name and metadata (unchanged code) | @@ -156,45 +179,45 @@ extension names, so Arrow's and Polars' global extension registries (keyed by na both without conflict; that is why the fields must be in the extension *name*, not only in its metadata. -## b. Public API — PENDING (P2: names), behaviour proposed +## b. Public API — `register_type` (D-naming, D-mutation) ```python import orcapod as op -@op.register_type # name by the setting: __main__ → "Point", module → "pkg.models.Point" +@op.register_type # default name: __main__ → "Point", module → "pkg.models.Point" @dataclass class Point: ... -@op.register_type(name="Point") # declared name, same everywhere it is defined +@op.register_type(name="Point") # declared name, the same wherever the class is defined @dataclass class Point: ... -op.register_type(Point, name="Point") # call form, for classes you do not own +@op.register_type(qualified=False) # a module class named by its class name: "Point" +@dataclass +class Point: ... -op.set_type_naming(main="name", module="qualified") # process-wide defaults (shown) +op.register_type(Point, name="Point") # call form, for classes you do not own ``` -- `register_type` records the name for the class object (a process-wide weak-keyed table, - so the class is not modified), checks the kind, the name and that the class is not - local, and returns the class unchanged. The logical type is built at first use, as - today, so declarations may come in any order (§a). Classes that are never declared are - registered on first use with the name from the setting. -- Provisional (P2): the public names live in one place (`orcapod/user_types.py`, - re-exported from `orcapod`) and the identity-string format in one function - (`identity_string(record)`), so a rename or a format change is a single-point edit. -- `set_type_naming` changes the default for classes registered after the call. Values: - `"name"` or `"qualified"`. Defaults: `main="name"`, `module="qualified"`. No TOML key - in v0.3 (scope, §g). -- Registering a class object under a second, different name after it is registered (for - example `register_type(Point, name="Geo")` after `Point` was used) raises `ValueError` - naming both names: one class object has one identity per process. +- `register_type` records the name for the class object in a process-wide weak-keyed + table. It never modifies the class (no attributes, no dataclass parameters, no pydantic + config) and returns the same object. It checks the kind, the name, `name` and + `qualified` not both given, and that the class is not local. The logical type is built + at first use, so declarations may come in any order (§a). Classes never declared are + registered on first use under the location rule. +- Registering the same class again with the same name is a no-op. A different name raises + `ValueError` naming both, including a declaration after the class was already used + under its default name: one class object has one name per process, and a registration + is never changed later. +- The public name lives in one place (`orcapod/user_types.py`, re-exported from + `orcapod`). **Name collisions (same kind, same name, two different class objects):** | Case | What happens | What the user sees | |---|---|---| | Same fields, same `module.qualname` (cell re-run, `%autoreload`, W1/W3/W4) | Same identity: the identity is rebound to the new class object (§c) | nothing | -| Same fields, different `module.qualname` (two modules declare `"Point"`, or `module="name"`) | Same identity by definition: they are the same type; values read back are built as the most recently registered class | one `UserWarning` per rebinding naming both classes. Warranted: values come back as a different class object than may be expected | +| Same fields, different `module.qualname` (two modules declare `"Point"`, or use `qualified=False`) | Same identity by definition: they are the same type; values read back are built as the most recently registered class | one `UserWarning` per rebinding naming both classes. Warranted: values come back as a different class object than may be expected | | Different fields (W2 with changed fields, or two modules) | Different identities: two types that happen to share a display name; no clash | nothing at registration. Reading picks the class whose identity matches (§d) | ## c. Rebinding rule (S1) — CONFIRMED (developer answer to S1) @@ -207,18 +230,21 @@ op.set_type_naming(main="name", module="qualified") # process-wide defaults (s - **Different identity under the same name → a different type.** Nothing is rebound; the new class gets its own logical type and extension name (W2: the next run computes into new tables). +- A pod defined before the class cell was re-run still accepts the re-defined class: + schema compatibility treats two user classes with the same identity string as the same + type (`user_class_identity.equivalent_hints`, used by `check_schema_compatibility`). - **Still raises:** 1. One class object registered under two names (§b), including a declaration after the class was already used under its default name. - 5. A self-referential or cyclic class (D-cycle). - 2. An invalid declared name (§a). + 2. An invalid declared name, or `name` together with `qualified` (§a, §b). 3. A local class (``), as today. 4. A shape disagreement on read (§d, S2). + 5. A self-referential or cyclic class (D-cycle). ## d. Read path — CONFIRMED behaviour (S2), lookup mechanism proposed -A stored column carries the extension name `Point@9117beaaf7ebe3ab` and the record in its -metadata. To build values, the reader resolves a class: +A stored column carries the extension name `Point@9117beaa…` (full digest) and the record +in its metadata. To build values, the reader resolves a class: 1. **Registered identity:** the extension name is registered in the reading context → use it. 2. **Declared or registered by name:** classes recorded under `(kind, name)` in this @@ -237,26 +263,28 @@ Outcomes: stored fields, and how to fix it: "define the class and register it with `op.register_type(Point, name='Point')` before reading". - **Candidates found, none with a matching identity (S2):** `TypeIdentityError` - (new, a `ValueError` subclass in `orcapod.errors`) naming the column where the read - path knows it, the class name and kind, the stored fields and the registered fields, - for example "column 'p': stored Point (dataclass) has fields {x: int, y: int}; the - registered Point has {x: int, y: float}". + (new, a `ValueError` subclass in `orcapod.errors`) naming the column, the class name and + kind, the stored fields and the registered fields, for example "Column 'q': Stored + values of Point (dataclass) {x: int, y: int} do not match the registered class: Point + (dataclass) {x: int, y: float}. …". `ExtensionAwareDatabase` reads add the column name + (`register_discovered_logical_types`); the `ImportError` gets it too. - **Same identity, different Arrow storage** (possible only if two data contexts map one annotation to different Arrow types): `TypeIdentityError` naming the column, the class and both struct types. The existing check in the synthesized `__arrow_ext_deserialize__` raises it. - These errors propagate. `DeltaTableDatabase` today catches every read error and logs - "Error reading Delta table"; it re-raises `TypeIdentityError`, so a run can no longer - report success with 0 rows. Other read errors keep their current handling (out of + "Error reading Delta table"; it re-raises `TypeIdentityError` in all five read paths + (`get_all_records`, the record lookups, the schema read), so a run can no longer report + success with 0 rows. Other read errors keep their current handling (out of scope). The misleading "data may have been deleted externally" message is untouched unless the path is reached for this reason (§g lists it as out of scope). -- **Pre-0.3 column** (D-legacy, PENDING): an extension name without `@` and metadata +- **Pre-0.3 column** (D-legacy): an extension name without `@` and metadata without `identity`, read directly (pipelines never reach these tables, §e): raise `TypeIdentityError` saying the column was written by Orcapod before 0.3 under `pkg.models.Point`, that v0.3 identifies the class by name and fields, and that the pipeline recomputes into new tables. Not converted, not dropped. -## e. Intended hash changes — PENDING (developer approval of the list) +## e. Intended hash changes — CONFIRMED list, measured Every pinned value that involves a dataclass or pydantic type moves, because the identity string changes for every user class (fields are now in it, for everyone). Measured on @@ -264,8 +292,13 @@ string changes for every user class (fields are now in it, for everyone). Measur golden fixtures `GoldenPoint` and `GoldenModel` moves exactly these 25 entries, and the `unchanged` control moves 0. Every identity-string change moves the same set (E08's `__main__` and module-less variants list identical entries), so the v0.3 string moves -these and only these. The implementation re-runs the golden test and the PR lists the -measured diff; any entry outside this list is a bug. +these and only these. + +**Measured after implementation** (golden files regenerated, compared with `origin/main` +c5ab1e05): 17 values change and 8 keys are replaced, exactly the entries below; 71 keys +are added (the 8 replacements and the new fixtures); `arrow.json` and `pipeline.json` are +byte-identical to `main`. Before regenerating, the golden test failed on exactly these 25 +entries plus the unpinned additions. | File | Entries | |---|---| @@ -275,9 +308,11 @@ measured diff; any entry outside this list is a bug. | `datagram.json` (8 replaced) | `{dataclass,pydantic}.row[{"k":"r1"/"r2"}]._tag_record_id::` and `._tag_source_id::`: the keys embed the moved pipeline hash, so these 8 keys disappear and 8 keys with the new hash appear | `arrow.json` and `pipeline.json` have no user-class entries and do not move. Golden -additions (new keys only): a declared-name dataclass, a nested dataclass (P1), an -`Optional` field, a container field, and permuted-field-order twins that must pin the -same value. +additions (new keys only): a declared-name dataclass and a pydantic model with the same +name and fields (distinct values), a nested dataclass with a container field (P1), +`list[GoldenPoint]`, an `Optional` field, and field-order twins (`GoldenTwinAB`, +`GoldenTwinBA`, both `golden.Twin`) that pin equal type, schema, data and pipeline hashes +(only their `source_id`-derived entries differ). **For the CHANGELOG (stated plainly):** "Orcapod 0.3 identifies a dataclass or pydantic model by its name, its kind and its fields. Every pipeline whose data includes such a @@ -306,22 +341,24 @@ unaffected. ## g. Scope cuts, in order, if time runs short -1. TOML/config-file setting for the naming defaults (`set_type_naming` stays). -2. The one-time `__main__` notice (S3, D-notice). -3. The collision `UserWarning` for same identity from a different `module.qualname` - (behaviour stays: rebind). -4. The `__main__` lookup on read (§d step 3); a declaration is then required to read - `__main__`-named data in another process. -5. `set_type_naming` itself (defaults fixed: `main="name"`, `module="qualified"`). +None were needed: everything below is implemented. The order that applied: + +1. The one-time `__main__` notice (S3, D-notice). +2. The collision `UserWarning` for same identity from a different `module.qualname`. +3. The `__main__` lookup on read (§d step 3). -Not cut: the identity record and string, rebinding, `register_type`, the S2 raise, the +(`set_type_naming` was dropped by decision, not cut.) Not cut: the identity record and string, rebinding, `register_type`, the S2 raise, the read lookup by declared name, the docs page and the CHANGELOG entry. Out of scope for v0.3: computed fields; local classes with a declared name; Enum, NamedTuple and TypedDict support (D6 of the findings); the `DeltaTableDatabase` row-count and shutdown-hang findings (§9 of the findings). -## Workshop scenarios after v0.3 (predicted, become the Step 2 tests) +## Workshop scenarios after v0.3 (tested: `tests/test_user_types/test_workshop_scenarios.py`) + +On `origin/main` 15 of the 19 scenario tests fail; all pass on this branch. The four that +pass on `main` pin behaviour v0.3 keeps (W5, W7, W12 with the same class, and a module +class without a declared name being a different type from the notebook class). | # | Scenario | v0.3 | |---|---|---| @@ -336,3 +373,20 @@ and shutdown-hang findings (§9 of the findings). | W12 | Read notebook results from a script | works if the script defines a matching `Point` (top level or declared); otherwise the `ImportError` of §d | | W14 | spawn child (`__mp_main__`) | same identity as `__main__` | | W15 | Local class | `ValueError`, as today | + +## Appendix A. Alternative B (fields only in the extension metadata) — not pursued + +The developer asked whether the fields must be folded into the name, or whether the name +could stay `Point` with kind and fields in `ARROW:extension:metadata` and a type hash +composed from name, kind and fields. The developer then confirmed the fold (November 10, +2026) and the investigation stopped. Evidence gathered before it stopped: + +- Arrow's and Polars' extension registries are keyed by name only. Orcapod's registry + swallows `ArrowKeyError` on a second registration under the same name + (`registry.py`), so a second shape under one name would be read through the first + shape's `__arrow_ext_deserialize__`, which raises on the storage mismatch. B would need + a parametric extension type that accepts every shape and dispatches on the metadata; + not built or tested. +- Whether every Polars round trip keeps extension metadata as it keeps the name was not + verified. +- B would change the semantic hasher's type handler; the fold needs no hashing change. From 532a574e3386c9726605d4c4379d386af0b52d5d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:56:57 -0800 Subject: [PATCH 10/31] test(docs): register each doc block's source in linecache (ITL-881) inspect.getsource on a function pod defined in a doc block read the runner's own file (linecache fell back to the __main__ loader), so on Python 3.13 the new Your Own Data Types page failed with a TokenError. Other pages passed only because the wrong lines happened to tokenize. Co-Authored-By: Claude Opus 5.5 --- tests/test_docs/_doc_samples.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_docs/_doc_samples.py b/tests/test_docs/_doc_samples.py index 334fcc93e..e9044d6de 100644 --- a/tests/test_docs/_doc_samples.py +++ b/tests/test_docs/_doc_samples.py @@ -27,6 +27,7 @@ import contextlib import io import json +import linecache import re import sys import textwrap @@ -216,8 +217,18 @@ def run_blocks(blocks: list[Block], name: str = "") -> list[dict[str, Any]] results.append(result) continue captured = io.StringIO() + filename = f"{name}:{block.lineno}" + # Register the block's source so ``inspect.getsource`` (used to hash function + # pods) reads the block itself. Without it, ``linecache`` falls back to the + # ``__main__`` module's loader, i.e. this runner's own file (seen on 3.13). + linecache.cache[filename] = ( + len(block.source), + None, + block.source.splitlines(keepends=True), + filename, + ) try: - code = compile(block.source, f"{name}:{block.lineno}", "exec") + code = compile(block.source, filename, "exec") with contextlib.redirect_stdout(captured): exec(code, namespace) # noqa: S102 - running our own documentation except BaseException: # noqa: BLE001 - report SystemExit etc. as failures too From fabed37dd123c98b2e57ce0f85743f278023b98b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 21:58:17 -0800 Subject: [PATCH 11/31] refactor(types): rename register_type's qualified argument to qualified_name (ITL-881) Developer decision, November 10, 2026. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- docs/concepts/your-own-data-types.md | 2 +- .../logical_types/user_class_identity.py | 24 ++++++++++--------- src/orcapod/user_types.py | 14 +++++------ ...-11-10-itl-881-user-class-identity-impl.md | 2 +- .../2026-11-10-user-class-identity-v0.3.md | 16 ++++++------- tests/test_user_types/test_identity_record.py | 10 ++++---- tests/test_user_types/test_register_type.py | 8 +++---- 8 files changed, 41 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3af187f5..6ea2baed3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ #### `op.register_type` — declare the name of your own data types (ITL-881) `@op.register_type(name="Point")` declares the name a dataclass or pydantic model is -identified by, wherever the class is defined. `@op.register_type(qualified=False)` names a +identified by, wherever the class is defined. `@op.register_type(qualified_name=False)` names a module class by its class name alone. Registration never modifies the class; declaring the same class under a second name raises `ValueError`. See [Your Own Data Types](docs/concepts/your-own-data-types.md). diff --git a/docs/concepts/your-own-data-types.md b/docs/concepts/your-own-data-types.md index a56ca5d8b..a74c8ee3d 100644 --- a/docs/concepts/your-own-data-types.md +++ b/docs/concepts/your-own-data-types.md @@ -150,7 +150,7 @@ print("function calls:", len(calls)) - `name="..."`: the name to use, the same wherever the class is defined. It must not be empty, contain `@` or spaces, or start with `orcapod.`. -- `qualified=False`: name a module class by its class name alone, without retyping it. +- `qualified_name=False`: name a module class by its class name alone, without retyping it. - no arguments: keep the default name. It never changes your class; it records the name in Orcapod's registry and returns the diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index aa6fefcca..52670b5ad 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -118,7 +118,7 @@ def is_main_class(cls: type) -> bool: return cls.__module__ in MAIN_MODULES -def default_name(cls: type, *, qualified: bool | None = None) -> str: +def default_name(cls: type, *, qualified_name: bool | None = None) -> str: """Return the name a class gets without a declared name. Args: @@ -131,9 +131,9 @@ def default_name(cls: type, *, qualified: bool | None = None) -> str: Returns: The name component. """ - if qualified is None: - qualified = not is_main_class(cls) - if not qualified: + if qualified_name is None: + qualified_name = not is_main_class(cls) + if not qualified_name: return cls.__qualname__ module = cls.__module__ if not is_main_class(cls): @@ -158,7 +158,7 @@ def resolve_name(cls: type) -> str: return default_name(cls) -def declare(cls: type, name: str | None, *, qualified: bool | None = None) -> None: +def declare(cls: type, name: str | None, *, qualified_name: bool | None = None) -> None: """Record the name a user class is identified by. Registration never modifies the class: the name is kept in a process-wide table @@ -167,13 +167,13 @@ def declare(cls: type, name: str | None, *, qualified: bool | None = None) -> No Args: cls: A dataclass or pydantic model class. - name: The declared name, or ``None`` for ``default_name(cls, qualified=...)``. - qualified: Only without ``name``: ``False`` for the class name alone, + name: The declared name, or ``None`` for ``default_name(cls, qualified_name=...)``. + qualified_name: Only without ``name``: ``False`` for the class name alone, ``True`` for ``module.qualname``, ``None`` for the location rule. Raises: TypeError: If ``cls`` is not a dataclass or pydantic model class. - ValueError: If both ``name`` and ``qualified`` are given, the class is local, + ValueError: If both ``name`` and ``qualified_name`` are given, the class is local, the name is invalid, or the class is already registered (declared or used) under a different name. """ @@ -183,15 +183,17 @@ def declare(cls: type, name: str | None, *, qualified: bool | None = None) -> No f"Only dataclasses and pydantic models can be registered as types, " f"got {cls!r}." ) - if name is not None and qualified is not None: + if name is not None and qualified_name is not None: raise ValueError( - "Pass either name=... or qualified=..., not both: qualified only chooses " + "Pass either name=... or qualified_name=..., not both: qualified_name only chooses " "between the class name and its module path when no name is given." ) reject_local_class(cls) if name is not None: validate_name(name) - effective = name if name is not None else default_name(cls, qualified=qualified) + effective = ( + name if name is not None else default_name(cls, qualified_name=qualified_name) + ) previous = _built.get(cls) or _declared.get(cls) if previous is not None and previous != effective: raise ValueError( diff --git a/src/orcapod/user_types.py b/src/orcapod/user_types.py index ab8069aa2..1db0d9baa 100644 --- a/src/orcapod/user_types.py +++ b/src/orcapod/user_types.py @@ -24,13 +24,13 @@ @overload def register_type( - cls: T, /, *, name: str | None = None, qualified: bool | None = None + cls: T, /, *, name: str | None = None, qualified_name: bool | None = None ) -> T: ... @overload def register_type( - *, name: str | None = None, qualified: bool | None = None + *, name: str | None = None, qualified_name: bool | None = None ) -> Callable[[T], T]: ... @@ -39,7 +39,7 @@ def register_type( /, *, name: str | None = None, - qualified: bool | None = None, + qualified_name: bool | None = None, ) -> T | Callable[[T], T]: """Declare the name a dataclass or pydantic model is identified by. @@ -49,7 +49,7 @@ def register_type( Without arguments the class keeps its default name: its class name if it is defined in a notebook or script, its module path and class name if it is defined - in a module. ``qualified=False`` names a module class by its class name alone, + in a module. ``qualified_name=False`` names a module class by its class name alone, without retyping it. Declare the name before the class is first used in a source or pod. Declaring the @@ -59,7 +59,7 @@ def register_type( cls: The class, when called directly. name: The declared name, e.g. ``"Point"`` or ``"geo.Point"``. It must not be empty, contain ``@`` or whitespace, or start with ``orcapod.``. - qualified: Only without ``name``. ``False``: the class name (``"Point"``). + qualified_name: Only without ``name``. ``False``: the class name (``"Point"``). ``True``: the module path and class name (``"pkg.models.Point"``). Returns: @@ -67,7 +67,7 @@ def register_type( Raises: TypeError: If the class is not a dataclass or pydantic model. - ValueError: If both ``name`` and ``qualified`` are given, the name is invalid, + ValueError: If both ``name`` and ``qualified_name`` are given, the name is invalid, the class is defined inside a function, or the class is already registered under a different name. @@ -82,7 +82,7 @@ def register_type( """ def decorate(target: T) -> T: - _uci.declare(target, name, qualified=qualified) + _uci.declare(target, name, qualified_name=qualified_name) return target if cls is None: diff --git a/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md index ca4ae5671..54b3aaa6c 100644 --- a/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md +++ b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md @@ -20,7 +20,7 @@ path re-raises it. the notebook scenarios), pytest via `uv run`. **Changes after the plan was written (developer decisions, November 10, 2026):** -`set_type_naming` was dropped (fixed location rule plus `register_type(qualified=False)`), +`set_type_naming` was dropped (fixed location rule plus `register_type(qualified_name=False)`), the identity string uses the full 64-hex digest, and stored structs list fields in name order. The spec is authoritative where this plan differs. diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md index 140d0f4b8..15b053f8d 100644 --- a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -22,9 +22,9 @@ Each decision is marked **CONFIRMED** (the developer decided it, November 10, 20 | P1 | A field typed by another user class carries that class's identity as that class was registered | CONFIRMED: developer decision, November 10, 2026 | | D-cycle | A self-referential or cyclic class is rejected with a clear `TypeError` | CONFIRMED by the coordinator from this proposal: Arrow cannot store a recursive struct | | D-format | Identity string = name + `@` + the full 64-hex SHA-256 of the record (the "fold") | CONFIRMED: developer, November 10, 2026; alternative B (fields only in metadata) not pursued, appendix A | -| D-naming | No process-wide naming switch: fixed location rule, per-class `register_type(name=...)` or `register_type(qualified=False)` | CONFIRMED: developer dropped `set_type_naming`; a project-wide choice would live in the data context, not in v0.3 | +| D-naming | No process-wide naming switch: fixed location rule, per-class `register_type(name=...)` or `register_type(qualified_name=False)` | CONFIRMED: developer dropped `set_type_naming`; a project-wide choice would live in the data context, not in v0.3 | | D-mutation | `register_type` never modifies the class; same name again is a no-op; another name raises | CONFIRMED: developer | -| P2 | Public API name `register_type` and argument `qualified` | `register_type` accepted by the developer; `qualified=False` proposed and implemented provisionally | +| P2 | Public API name `register_type` and argument `qualified_name` | CONFIRMED: developer, November 10, 2026 | | D-hash | The 25 golden entries in §e move; nothing else | CONFIRMED by the coordinator: follows from decision 2; the PR lists every moved entry | | D-legacy | Reading a pre-0.3 user-class column directly raises a clear error | CONFIRMED by the coordinator: a consequence of the identity check; nothing extra is built | | D-notice | One-time notice when a `__main__` class is first registered (S3) | CONFIRMED in scope by the coordinator; last item before the cuts | @@ -42,8 +42,8 @@ A user class's identity is an **identity record** with three components: The name is a string chosen once per class, by the first rule that applies: 1. **Declared name.** The user declared one (§b). Used verbatim. -2. **Per-class choice without a name:** `register_type(qualified=False)` gives - `cls.__qualname__`; `register_type(qualified=True)` gives `module.qualname`. +2. **Per-class choice without a name:** `register_type(qualified_name=False)` gives + `cls.__qualname__`; `register_type(qualified_name=True)` gives `module.qualname`. 3. **`__main__` class** (`cls.__module__` is `"__main__"` or `"__mp_main__"`, the name a `multiprocessing` spawn child gives the main module, W14): `cls.__qualname__` (`"Point"`, or `"Outer.Point"` for a nested class). @@ -192,7 +192,7 @@ class Point: ... @dataclass class Point: ... -@op.register_type(qualified=False) # a module class named by its class name: "Point" +@op.register_type(qualified_name=False) # a module class named by its class name: "Point" @dataclass class Point: ... @@ -202,7 +202,7 @@ op.register_type(Point, name="Point") # call form, for classes you do not own - `register_type` records the name for the class object in a process-wide weak-keyed table. It never modifies the class (no attributes, no dataclass parameters, no pydantic config) and returns the same object. It checks the kind, the name, `name` and - `qualified` not both given, and that the class is not local. The logical type is built + `qualified_name` not both given, and that the class is not local. The logical type is built at first use, so declarations may come in any order (§a). Classes never declared are registered on first use under the location rule. - Registering the same class again with the same name is a no-op. A different name raises @@ -217,7 +217,7 @@ op.register_type(Point, name="Point") # call form, for classes you do not own | Case | What happens | What the user sees | |---|---|---| | Same fields, same `module.qualname` (cell re-run, `%autoreload`, W1/W3/W4) | Same identity: the identity is rebound to the new class object (§c) | nothing | -| Same fields, different `module.qualname` (two modules declare `"Point"`, or use `qualified=False`) | Same identity by definition: they are the same type; values read back are built as the most recently registered class | one `UserWarning` per rebinding naming both classes. Warranted: values come back as a different class object than may be expected | +| Same fields, different `module.qualname` (two modules declare `"Point"`, or use `qualified_name=False`) | Same identity by definition: they are the same type; values read back are built as the most recently registered class | one `UserWarning` per rebinding naming both classes. Warranted: values come back as a different class object than may be expected | | Different fields (W2 with changed fields, or two modules) | Different identities: two types that happen to share a display name; no clash | nothing at registration. Reading picks the class whose identity matches (§d) | ## c. Rebinding rule (S1) — CONFIRMED (developer answer to S1) @@ -236,7 +236,7 @@ op.register_type(Point, name="Point") # call form, for classes you do not own - **Still raises:** 1. One class object registered under two names (§b), including a declaration after the class was already used under its default name. - 2. An invalid declared name, or `name` together with `qualified` (§a, §b). + 2. An invalid declared name, or `name` together with `qualified_name` (§a, §b). 3. A local class (``), as today. 4. A shape disagreement on read (§d, S2). 5. A self-referential or cyclic class (D-cycle). diff --git a/tests/test_user_types/test_identity_record.py b/tests/test_user_types/test_identity_record.py index 90722d343..e9fc1e423 100644 --- a/tests/test_user_types/test_identity_record.py +++ b/tests/test_user_types/test_identity_record.py @@ -119,12 +119,14 @@ def test_main_class_defaults_to_name(self, module: str) -> None: def test_nested_main_class_uses_qualname(self) -> None: assert uci.resolve_name(_make_class("__main__", "Outer.Point")) == "Outer.Point" - def test_default_name_qualified_false(self) -> None: - assert uci.default_name(_make_class("pkg.models"), qualified=False) == "Point" + def test_default_name_qualified_name_false(self) -> None: + assert ( + uci.default_name(_make_class("pkg.models"), qualified_name=False) == "Point" + ) - def test_default_name_qualified_true_for_main(self) -> None: + def test_default_name_qualified_name_true_for_main(self) -> None: cls = _make_class("__main__") - assert uci.default_name(cls, qualified=True) == "__main__.Point" + assert uci.default_name(cls, qualified_name=True) == "__main__.Point" def test_declared_name_wins(self) -> None: cls = _make_class("pkg.models") diff --git a/tests/test_user_types/test_register_type.py b/tests/test_user_types/test_register_type.py index ab9daf1d2..1046ad229 100644 --- a/tests/test_user_types/test_register_type.py +++ b/tests/test_user_types/test_register_type.py @@ -109,19 +109,19 @@ def test_declaring_after_use_under_another_name_raises() -> None: op.register_type(UsedFirst, name="itl881.UsedFirst") -@op.register_type(qualified=False) +@op.register_type(qualified_name=False) @dataclasses.dataclass class ShortNamed: x: int -def test_qualified_false_names_a_module_class_by_its_class_name() -> None: +def test_qualified_name_false_names_a_module_class_by_its_class_name() -> None: assert _name_part(ShortNamed(1)) == "ShortNamed" -def test_name_and_qualified_together_raise() -> None: +def test_name_and_qualified_name_together_raise() -> None: with pytest.raises(ValueError, match="either name"): - op.register_type(Undeclared, name="itl881.X", qualified=False) + op.register_type(Undeclared, name="itl881.X", qualified_name=False) @dataclasses.dataclass From b0ab6b6194a4f02cee43db0fa33dddcd1e744309 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:05:36 -0800 Subject: [PATCH 12/31] fix(types): review fixes for user class identity reads (ITL-881) - Accept large/non-large storage differences at any depth when reading an extension column: Delta compaction rewrites a struct field large_string as string, which made every stored class with a str field unreadable. - Test read candidates without registering them, so a failed read never fixes a class's name; the error says when a class with the stored fields exists under another name. - With qualified_name=True a __main__ class is __main__. in a spawn child too; declared names are NFC-normalized. - Remove the now unused _canonical_storage helper. - Spec: canonical JSON uses ensure_ascii=False; logical type names (e.g. a future orcapod.ndarray) are part of field tokens; metadata bytes are already hashed. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/logical_types/registry.py | 31 ++---- .../logical_types/user_class_identity.py | 34 +++++- .../logical_types/user_class_logical_type.py | 104 ++++++++++++++---- .../2026-11-10-user-class-identity-v0.3.md | 30 ++++- tests/test_logical_types/test_registry.py | 40 +------ tests/test_user_types/test_identity_record.py | 12 +- tests/test_user_types/test_read_path.py | 35 ++++++ .../test_workshop_scenarios.py | 53 +++++++++ 8 files changed, 241 insertions(+), 98 deletions(-) diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index e3729f647..9aeb6e4b5 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -36,27 +36,6 @@ def _sanitize(name: str) -> str: return re.sub(r"[^A-Za-z0-9]", "_", name) -def _canonical_storage(t: pa.DataType) -> pa.DataType: - """Return the canonical large-variant for string/binary families; identity elsewhere. - - Maps: - ``string`` / ``string_view`` → ``large_string`` - ``binary`` / ``binary_view`` → ``large_binary`` - ``large_string`` / ``large_binary`` → unchanged (already canonical) - anything else → unchanged - - Used in ``__arrow_ext_deserialize__`` to accept physically-equivalent but - layout-different string/binary variants that Delta Lake or other compaction - tools may write (e.g. delta-rs ``optimize.compact()`` normalises - ``large_string`` columns to plain ``string`` in the rewritten Parquet files). - """ - if pa.types.is_string(t) or pa.types.is_string_view(t): - return pa.large_string() - if pa.types.is_binary(t) or pa.types.is_binary_view(t): - return pa.large_binary() - return t - - def make_arrow_extension_type( extension_name: str, storage_type: pa.DataType, @@ -123,9 +102,13 @@ def _deserialize( # # Cross-family mismatches (string where large_binary was registered, or # vice-versa) are genuine semantic errors and always raise. - if _canonical_storage(_storage) == _storage: - # _storage is the canonical large form — accept any same-family variant. - storage_ok = _canonical_storage(storage_type) == _storage + from orcapod.utils.arrow_utils import normalize_to_large_types + + if normalize_to_large_types(_storage) == _storage: + # _storage is the canonical large form — accept any same-family variant, + # at any nesting depth (a struct field rewritten from large_string to + # string by Delta compaction, ITL-881). + storage_ok = normalize_to_large_types(storage_type) == _storage else: # Non-canonical _storage — require exact match. storage_ok = storage_type == _storage diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index 52670b5ad..4f96014e2 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -25,6 +25,7 @@ import re import sys import typing +import unicodedata import warnings import weakref from typing import TYPE_CHECKING, Any, Literal @@ -53,8 +54,11 @@ _declared: weakref.WeakKeyDictionary[type, str] = weakref.WeakKeyDictionary() # The name each class object was built under; a class has one name per process. _built: weakref.WeakKeyDictionary[type, str] = weakref.WeakKeyDictionary() -# The identity string each class object was built with. +# The identity string and record each class object was built with. _identity_of: weakref.WeakKeyDictionary[type, str] = weakref.WeakKeyDictionary() +_record_of: weakref.WeakKeyDictionary[type, dict[str, Any]] = ( + weakref.WeakKeyDictionary() +) # Classes seen under each (kind, name), most recent last. _known: dict[tuple[str, str], list[weakref.ref[type]]] = {} # Names that already produced the one-time ``__main__`` notice. @@ -135,11 +139,12 @@ def default_name(cls: type, *, qualified_name: bool | None = None) -> str: qualified_name = not is_main_class(cls) if not qualified_name: return cls.__qualname__ - module = cls.__module__ - if not is_main_class(cls): - from orcapod.utils.canonical_type_names import canonical_module_name + if is_main_class(cls): + # A spawn child names the main module "__mp_main__"; both are "__main__". + return f"__main__.{cls.__qualname__}" + from orcapod.utils.canonical_type_names import canonical_module_name - module = canonical_module_name(cls) or module + module = canonical_module_name(cls) or cls.__module__ return f"{module}.{cls.__qualname__}" @@ -191,6 +196,8 @@ def declare(cls: type, name: str | None, *, qualified_name: bool | None = None) reject_local_class(cls) if name is not None: validate_name(name) + # One spelling per name: the same name typed in two Unicode forms is one name. + name = unicodedata.normalize("NFC", name) effective = ( name if name is not None else default_name(cls, qualified_name=qualified_name) ) @@ -228,7 +235,19 @@ def built_name(cls: type) -> str | None: return _built.get(cls) -def mark_built(cls: type, name: str, identity: str) -> None: +def identity_of(cls: type) -> str | None: + """Return the identity string ``cls`` was built with, or ``None`` if not built.""" + return _identity_of.get(cls) + + +def built_record(cls: type) -> dict[str, Any] | None: + """Return the identity record ``cls`` was built with, or ``None`` if not built.""" + return _record_of.get(cls) + + +def mark_built( + cls: type, name: str, identity: str, record: dict[str, Any] | None = None +) -> None: """Record that ``cls`` was built under ``name`` with ``identity``. A class has one name per process. @@ -237,9 +256,12 @@ def mark_built(cls: type, name: str, identity: str) -> None: cls: The class. name: Its name component. identity: Its identity string. + record: Its identity record. """ _built[cls] = name _identity_of[cls] = identity + if record is not None: + _record_of[cls] = record def equivalent_hints(a: Any, b: Any) -> bool: diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index 745755bdf..76b969f31 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -222,6 +222,39 @@ def create_for_python_type( """ uci.reject_local_class(python_type) name = uci.built_name(python_type) or uci.resolve_name(python_type) + record, storage_type, field_annotations = self._describe( + python_type, name, converter + ) + identity = uci.identity_string(record) + uci.mark_built(python_type, name, identity, record) + uci.remember(self.kind, name, python_type) + uci.notice_main_class(python_type, name) + logger.debug("%s: built %r for %r", type(self).__name__, identity, python_type) + return self.logical_type_class( + identity, python_type, storage_type, field_annotations, record + ) + + def _describe( + self, python_type: type, name: str, converter: TypeConverterProtocol + ) -> tuple[dict[str, Any], pa.StructType, list[tuple[str, Any]]]: + """Return the identity record, storage struct and field annotations of a class. + + Does not register ``python_type`` itself (its field types are registered, as + their Arrow types are needed), so the read path can test a candidate class + without fixing its name. + + Args: + python_type: The class. + name: The name component to use. + converter: The active converter for field-type resolution. + + Returns: + ``(record, storage_type, field_annotations)``. + + Raises: + ValueError: If the type hints cannot be resolved. + RecursiveTypeError: If the class refers back to itself, naming the path. + """ hints = _type_hints(python_type) arrow_fields = [] @@ -247,14 +280,7 @@ def create_for_python_type( tokens[field_name] = uci.type_token(annotation, converter) record = uci.identity_record(self.kind, name, tokens) - identity = uci.identity_string(record) - uci.mark_built(python_type, name, identity) - uci.remember(self.kind, name, python_type) - uci.notice_main_class(python_type, name) - logger.debug("%s: built %r for %r", type(self).__name__, identity, python_type) - return self.logical_type_class( - identity, python_type, pa.struct(arrow_fields), field_annotations, record - ) + return record, pa.struct(arrow_fields), field_annotations # ------------------------------------------------------------------ # Read path @@ -311,31 +337,63 @@ def reconstruct_from_arrow( f"op.register_type({name.rsplit('.', 1)[-1]}, name={name!r})." ) - found: list[UserClassLogicalType] = [] + mismatches: list[str] = [] + renamable = False # a candidate has the stored fields under another name for cls in candidates: - _ = converter.register_python_class(cls) # raises for unsupported classes - lt = converter.get_logical_type_for_python_type(cls) - if not isinstance(lt, UserClassLogicalType): + built = uci.identity_of(cls) + if built is not None: + if built != arrow_extension_name: + built_record = uci.built_record(cls) + mismatches.append(_describe_candidate(cls, built_record)) + continue + lt = converter.get_logical_type_for_python_type(cls) + if isinstance(lt, UserClassLogicalType): + _check_storage(lt, storage_type, record) + return lt + continue + # Not used yet in this process: test it without registering it, so a + # non-matching candidate keeps its freedom to be declared under a name. + try: + candidate_record, _, _ = self._describe( + cls, uci.resolve_name(cls), converter + ) + except (TypeError, ValueError) as exc: + mismatches.append( + f"{cls.__module__}.{cls.__qualname__} (cannot be stored: {exc})" + ) + continue + if uci.identity_string(candidate_record) != arrow_extension_name: + mismatches.append(_describe_candidate(cls, candidate_record)) + renamable = renamable or candidate_record["fields"] == record["fields"] continue - if lt.logical_type_name == arrow_extension_name: + _ = converter.register_python_class(cls) + lt = converter.get_logical_type_for_python_type(cls) + if isinstance(lt, UserClassLogicalType): _check_storage(lt, storage_type, record) return lt - found.append(lt) - registered = "; ".join( - uci.describe_record(lt.identity_record) - if lt.identity_record is not None - else lt.logical_type_name - for lt in found + hint = ( + f" A class with the stored fields exists under another name: register it " + f"as {name!r} before it is first used, e.g. " + f"op.register_type(, name={name!r})." + if renamable + else " A class with the same name but different fields is a different " + "type; restore the stored fields, or use the class with the new fields " + "to compute new results." ) raise TypeIdentityError( - f"Stored values of {uci.describe_record(record)} do not match the " - f"registered class: {registered}. A class with the same name but different " - f"fields is a different type; restore the stored fields, or use the class " - f"with the new fields to compute new results." + f"Stored values of {uci.describe_record(record)} do not match the classes " + f"found: {'; '.join(mismatches)}.{hint}" ) +def _describe_candidate(cls: type, record: dict[str, Any] | None) -> str: + where = f"{cls.__module__}.{cls.__qualname__}" + if record is None: + return where + return f"{where} as {uci.describe_record(record)}" + + def _type_hints(python_type: type) -> dict[str, Any]: try: return typing.get_type_hints(python_type) diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md index 15b053f8d..663f6b524 100644 --- a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -54,7 +54,10 @@ Rules 3 and 4 are fixed (D-naming): no process-wide switch changes them, so a cl identity never depends on whether or when some setting was called. A name must be non-empty, must not contain `@` or whitespace, and must not start with -`orcapod.` (reserved for built-in types such as `orcapod.path`). A class whose +`orcapod.` (reserved for built-in types such as `orcapod.path`). A declared name is +NFC-normalized, so one name typed in two Unicode forms is one name (Python already +normalizes class and field identifiers). With `qualified_name=True`, a `__main__` class is +named `__main__.` in a `multiprocessing` spawn child too. A class whose `__qualname__` contains `` stays rejected in v0.3, even with a declared name (scope, §g). @@ -124,6 +127,13 @@ classes (only the resulting fields count). Computed fields are deferred (D-compu later decision to include them adds a key that is absent when empty, so it would move only classes that have computed fields. +A type with a logical type contributes its logical type *name*, so those names are part of +user-class identities: adding or renaming a built-in logical type (for example a new +`orcapod.ndarray` from PR #289, where today an `np.ndarray` field gives the token +`numpy.ndarray`) moves the identity of every class with such a field. Built-in logical +type names are already a stability commitment (`orcapod.path`); after #289 lands this +branch adds a golden fixture with an ndarray field. + The token is built from the Python annotation, not the Arrow type, because the developer named field *types* and because the Arrow type cannot tell `str` from `orcapod.path` (both `large_string`). @@ -133,7 +143,8 @@ named field *types* and because the Arrow type cannot tell `str` from `orcapod.p ``` identity string = f"{name}@{fingerprint}" fingerprint = sha256(canonical_json(record)).hexdigest() # full 64 hex digits -canonical_json = json.dumps(record, sort_keys=True, separators=(",", ":")) +canonical_json = json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + # hashed as UTF-8 ``` `Point(x: int, y: float)` as a dataclass gives @@ -271,7 +282,16 @@ Outcomes: - **Same identity, different Arrow storage** (possible only if two data contexts map one annotation to different Arrow types): `TypeIdentityError` naming the column, the class and both struct types. The existing check in the synthesized - `__arrow_ext_deserialize__` raises it. + `__arrow_ext_deserialize__` raises it. Storage that differs only in large versus + non-large layouts, at any depth, is accepted: Delta compaction rewrites a struct field + `large_string` as `string` (found in review; on `main` this read failed into 0 rows). +- **Candidates are tested without registering them.** A class found by name that is not + yet used in the process is described without fixing its name, so a failed read never + stops the user from then declaring the class under the stored name. When a candidate + has the stored fields under another name, the error says to declare it. +- **Extension metadata is compared byte for byte**, like the extension name. It is part + of the per-row data hash already (`ARROW:extension:*` keys are hashed), so any change to + it is a hash change and needs the same approval; it is not a separate format risk. - These errors propagate. `DeltaTableDatabase` today catches every read error and logs "Error reading Delta table"; it re-raises `TypeIdentityError` in all five read paths (`get_all_records`, the record lookups, the schema read), so a run can no longer report @@ -332,7 +352,9 @@ unaffected. `builtin_handlers.py` or `v0.1.json` (the identity reaches hashing through the logical type name), adds a new docs page instead of editing `function-pods.md` beyond a link, and edits the golden files only by regeneration. -- **PR #289** (`orcapod.ndarray`) adds `array_format.py` and tests; no file in common. +- **PR #289** (`orcapod.ndarray`) adds `array_format.py` and tests; no file in common, but + a field typed `np.ndarray` contributes its logical type name to the identity (§a), so + after #289 lands the ndarray token is re-checked and pinned with a golden fixture. - **Preferred order:** #286 and #289 merge first. This branch then merges `origin/main`, re-runs E08 against the new main to confirm the moved set, and regenerates the golden files; the PR's golden diff must then show only the §e entries plus the additions. If diff --git a/tests/test_logical_types/test_registry.py b/tests/test_logical_types/test_registry.py index 6876b13c6..8df15212f 100644 --- a/tests/test_logical_types/test_registry.py +++ b/tests/test_logical_types/test_registry.py @@ -20,7 +20,6 @@ LogicalTypeRegistry, make_arrow_extension_type, make_polars_extension_type, - _canonical_storage, ) @@ -678,43 +677,6 @@ def test_registry_does_not_expose_ensure_methods(): assert not hasattr(registry, "ensure_extension_type") -# --------------------------------------------------------------------------- -# _canonical_storage helper -# --------------------------------------------------------------------------- - - -class TestCanonicalStorage: - """Unit tests for the _canonical_storage helper.""" - - def test_string_maps_to_large_string(self): - assert _canonical_storage(pa.string()) == pa.large_string() - - def test_utf8_maps_to_large_string(self): - # pa.utf8() is an alias for pa.string() - assert _canonical_storage(pa.utf8()) == pa.large_string() - - def test_string_view_maps_to_large_string(self): - assert _canonical_storage(pa.string_view()) == pa.large_string() - - def test_large_string_is_identity(self): - assert _canonical_storage(pa.large_string()) == pa.large_string() - - def test_binary_maps_to_large_binary(self): - assert _canonical_storage(pa.binary()) == pa.large_binary() - - def test_binary_view_maps_to_large_binary(self): - assert _canonical_storage(pa.binary_view()) == pa.large_binary() - - def test_large_binary_is_identity(self): - assert _canonical_storage(pa.large_binary()) == pa.large_binary() - - def test_int64_is_identity(self): - assert _canonical_storage(pa.int64()) == pa.int64() - - def test_fixed_binary_is_identity(self): - assert _canonical_storage(pa.binary(16)) == pa.binary(16) - - # --------------------------------------------------------------------------- # __arrow_ext_deserialize__ storage-family tolerance (ITL-602) # --------------------------------------------------------------------------- @@ -801,7 +763,7 @@ def test_non_canonical_storage_accepts_exact_match(self): """Extension registered with plain string (non-canonical) still reads its own data. Regression test for the Copilot review finding: the previous check - _canonical_storage(storage_type) != _storage would map string → large_string and + normalize_to_large_types(storage_type) != _storage would map string → large_string and then compare large_string != string, incorrectly raising for a type that should read back cleanly. """ diff --git a/tests/test_user_types/test_identity_record.py b/tests/test_user_types/test_identity_record.py index e9fc1e423..5aa94891e 100644 --- a/tests/test_user_types/test_identity_record.py +++ b/tests/test_user_types/test_identity_record.py @@ -124,10 +124,18 @@ def test_default_name_qualified_name_false(self) -> None: uci.default_name(_make_class("pkg.models"), qualified_name=False) == "Point" ) - def test_default_name_qualified_name_true_for_main(self) -> None: - cls = _make_class("__main__") + @pytest.mark.parametrize("module", ["__main__", "__mp_main__"]) + def test_default_name_qualified_name_true_for_main(self, module: str) -> None: + cls = _make_class(module) assert uci.default_name(cls, qualified_name=True) == "__main__.Point" + def test_declared_names_are_nfc_normalized(self) -> None: + composed, decomposed = "Caf\u00e9", "Cafe\u0301" + first, second = _make_class("pkg.a"), _make_class("pkg.b") + uci.declare(first, composed) + uci.declare(second, decomposed) + assert uci.resolve_name(first) == uci.resolve_name(second) == composed + def test_declared_name_wins(self) -> None: cls = _make_class("pkg.models") uci.declare(cls, "geo.Point") diff --git a/tests/test_user_types/test_read_path.py b/tests/test_user_types/test_read_path.py index 7c085b1f9..b8f3aa885 100644 --- a/tests/test_user_types/test_read_path.py +++ b/tests/test_user_types/test_read_path.py @@ -50,3 +50,38 @@ def test_storage_mismatch_raises_instead_of_reading_nothing(tmp_path: Path) -> N with pytest.raises(TypeIdentityError, match="expected storage_type"): DeltaTableDatabase(tmp_path).get_all_records(("points",)) + + +@op.register_type(name="itl881.Labelled") +@dataclasses.dataclass +class Labelled: + label: str + n: int + + +def _run_labelled(db_root: Path) -> tuple[int, list[Labelled]]: + from orcapod import PipelineJob, function_pod + + calls: list[Labelled] = [] + + @function_pod(output_keys="q") + def echo(p: Labelled) -> Labelled: + calls.append(p) + return p + + rows: list[Any] = [{"k": f"r{i}", "p": Labelled(f"x{i}", i)} for i in (1, 2)] + job = PipelineJob(name="compact", store=DeltaTableDatabase(db_root)) + with job: + echo.pod(DictSource(data=rows, tag_columns=["k"], source_id="s"), label="echo") + job.run() + values = [data.as_dict()["q"] for _, data in job.nodes["echo"].iter_data()] + return len(calls), sorted(values, key=lambda v: v.n) + + +def test_stored_values_read_after_delta_compaction(tmp_path: Path) -> None: + """Delta compaction rewrites large_string struct fields as string; still readable.""" + expected = [Labelled("x1", 1), Labelled("x2", 2)] + assert _run_labelled(tmp_path) == (2, expected) + for log in tmp_path.rglob("_delta_log"): + deltalake.DeltaTable(str(log.parent)).optimize.compact() + assert _run_labelled(tmp_path) == (0, expected) diff --git a/tests/test_user_types/test_workshop_scenarios.py b/tests/test_user_types/test_workshop_scenarios.py index 98231c023..1937888ee 100644 --- a/tests/test_user_types/test_workshop_scenarios.py +++ b/tests/test_user_types/test_workshop_scenarios.py @@ -363,3 +363,56 @@ def test_read_with_different_fields_raises_naming_both_shapes(workdir: Path) -> assert out["error"].startswith("TypeIdentityError: Column 'q'"), out assert "{x: int, y: int}" in out["message"] assert "{x: int, y: float}" in out["message"] + + +def test_failed_read_leaves_the_class_free_to_be_declared(workdir: Path) -> None: + """A read that rejects a candidate class must not fix that class's name.""" + _store_from_notebook(workdir) + write_files( + workdir, + { + "pkg/__init__.py": "", + "pkg/models.py": """ + from dataclasses import dataclass + + @dataclass + class Point: + x: int + y: int + """, + }, + ) + code = """ +import os +from pathlib import Path +import orcapod as op +from child_harness import error_text, report +from orcapod.contexts import get_default_context +from orcapod.databases import DeltaTableDatabase, ExtensionAwareDatabase +import wsjob +from pkg.models import Point + +root = Path(os.environ["WS_DB"]) +path = tuple(next(p for p in wsjob.table_paths(root) if "/_result/" in p).split("/")) +converter = get_default_context().type_converter + +def read(): + db = ExtensionAwareDatabase(DeltaTableDatabase(root), converter=converter) + table = db.get_all_records(path) + return [repr(r["q"]) for r in converter.arrow_table_to_python_dicts(table.select(["q"]))] + +out = {} +try: + read() +except Exception as exc: + out["first"] = error_text(exc) + out["message"] = str(exc) +op.register_type(Point, name="Point") +out["second"] = sorted(read()) +report(out) +""" + out = _script(workdir, code) + assert out["first"].startswith("TypeIdentityError"), out + assert "pkg.models.Point" in out["message"] + assert "op.register_type(, name='Point')" in out["message"] + assert out["second"] == ["Point(x=1, y=1)", "Point(x=2, y=2)"] From 1aae9d1b85191357d927366b07c53b4124978f0a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:05:47 -0800 Subject: [PATCH 13/31] docs(changelog): Delta compaction fix for stored user-class values (ITL-881) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ea2baed3..da68435cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,14 @@ info. Hashes of pipelines that already worked are unchanged. ### Fixed +#### Stored dataclass and pydantic values stay readable after Delta compaction (ITL-881) + +Delta compaction (`optimize.compact()`) rewrites a `large_string` struct field as +`string`. Reading a stored dataclass or pydantic column with a `str` or `bytes` field then +failed, and a cached run returned 0 rows. Extension columns now accept large and non-large +layouts of the same type at any nesting depth, as they already did at the top level +(ITL-602). + #### Materialized rows keep their own source info (ITL-845) When rows were collected and turned back into a stream, every row got the source-info From b4dadf3642cbedd907b73fbd4ffc9a3346f5dd00 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:29:07 -0800 Subject: [PATCH 14/31] test(types): pin IPC read of non-large nested storage as a known limitation (ITL-881, ITL-894) Strict xfail: over Arrow IPC a stored column whose storage differs from the registered type only by layout reads as invalid arrays. Parquet and Delta reads are correct. Fix tracked in ITL-894; noted in DESIGN_ISSUES. Co-Authored-By: Claude Opus 5.5 --- DESIGN_ISSUES.md | 10 +++++++ tests/test_user_types/test_read_path.py | 35 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index a748e7c57..242a1d35d 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2237,3 +2237,13 @@ checks the identity, and raises `ImportError` or `TypeIdentityError` naming the the same issue. Not covered: computed fields (deferred by the developer), local classes, Enum/NamedTuple/TypedDict. +**Known limitation (ITL-894):** reading a stored column whose storage differs from the +registered one only by layout (`string` vs `large_string`) is accepted at any depth, so +Delta-compacted tables stay readable. `__arrow_ext_deserialize__` then returns the +registered large storage type, which is correct for Parquet and Delta reads (the decoder +converts the data) but not over Arrow IPC or the C stream interface, where Arrow reads +32-bit offsets as 64-bit and the arrays are invalid. Top-level columns had this since +ITL-602; nested struct fields since this fix. Orcapod's own writers and Polars round trips +never produce non-large storage. Pinned by a strict xfail in +`tests/test_user_types/test_read_path.py`. + diff --git a/tests/test_user_types/test_read_path.py b/tests/test_user_types/test_read_path.py index b8f3aa885..04cb8c138 100644 --- a/tests/test_user_types/test_read_path.py +++ b/tests/test_user_types/test_read_path.py @@ -85,3 +85,38 @@ def test_stored_values_read_after_delta_compaction(tmp_path: Path) -> None: for log in tmp_path.rglob("_delta_log"): deltalake.DeltaTable(str(log.parent)).optimize.compact() assert _run_labelled(tmp_path) == (0, expected) + + +@pytest.mark.xfail( + strict=True, + raises=pa.ArrowInvalid, + reason=( + "ITL-894: _deserialize accepts a non-large storage variant but returns the " + "large storage type, so IPC reinterprets 32-bit offsets as 64-bit. Parquet " + "and Delta reads convert the data and are correct." + ), +) +def test_ipc_read_of_non_large_nested_storage_is_valid() -> None: + import io + + narrow = pa.struct([pa.field("label", pa.string()), pa.field("n", pa.int64())]) + value: Any = Labelled("x1", 1) + ext = ( + DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) + .as_table() + .schema.field("p") + .type + ) + metadata: dict[bytes | str, bytes | str] = { + b"ARROW:extension:name": ext.extension_name.encode(), + b"ARROW:extension:metadata": ext.__arrow_ext_serialize__(), + } + table = pa.table( + {"p": pa.array([{"label": "hello", "n": 1}], type=narrow)}, + schema=pa.schema([pa.field("p", narrow).with_metadata(metadata)]), + ) + sink = io.BytesIO() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + read = pa.ipc.open_stream(sink.getvalue()).read_all() + read.validate(full=True) From 5707fb7c5a93bdf1d96148753b6e1c18530c5939 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:30:04 -0800 Subject: [PATCH 15/31] docs(types): docstrings for user-class identity helpers (ITL-881) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/logical_types/user_class_identity.py | 3 +++ src/orcapod/logical_types/user_class_logical_type.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index 4f96014e2..ee005cda6 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -355,6 +355,7 @@ def candidate_classes(kind: str, name: str) -> list[type]: def _walk_attributes(root: Any, parts: list[str]) -> Any: + """Follow ``parts`` as attributes from ``root``; return ``None`` if one is missing.""" obj = root for part in parts: obj = getattr(obj, part, None) @@ -364,6 +365,7 @@ def _walk_attributes(root: Any, parts: list[str]) -> Any: def _import_dotted(name: str) -> Any: + """Import the object a dotted name refers to, or return ``None`` if there is none.""" from orcapod.logical_types.type_utils import _walk_fqcn try: @@ -471,6 +473,7 @@ def type_token(annotation: Any, converter: TypeConverterProtocol) -> str: def _class_token(cls: Any) -> str: + """Return the type token of a class: its canonical name, builtins written bare.""" name = canonical_type_name(cls) return name.removeprefix("builtins.") diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index 76b969f31..6d4954720 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -388,6 +388,7 @@ def reconstruct_from_arrow( def _describe_candidate(cls: type, record: dict[str, Any] | None) -> str: + """Render a candidate class for a read error: its path and, if known, its identity.""" where = f"{cls.__module__}.{cls.__qualname__}" if record is None: return where @@ -395,6 +396,7 @@ def _describe_candidate(cls: type, record: dict[str, Any] | None) -> str: def _type_hints(python_type: type) -> dict[str, Any]: + """Return the resolved type hints of a class, as ``ValueError`` on failure.""" try: return typing.get_type_hints(python_type) except Exception as exc: @@ -404,6 +406,7 @@ def _type_hints(python_type: type) -> dict[str, Any]: def _check_storage( lt: UserClassLogicalType, storage_type: pa.DataType, record: dict[str, Any] ) -> None: + """Raise ``TypeIdentityError`` if stored and registered storage differ beyond layout.""" from orcapod.utils.arrow_utils import normalize_to_large_types if normalize_to_large_types(lt.storage_type) != normalize_to_large_types( From 39951f01eb2d24d0fbec21abb75864cd49a7f62e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:33:44 -0800 Subject: [PATCH 16/31] docs(changelog): note the IPC limitation tracked as ITL-894 (ITL-881) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d63f80c35..51b5be066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,9 @@ Delta compaction (`optimize.compact()`) rewrites a `large_string` struct field a `string`. Reading a stored dataclass or pydantic column with a `str` or `bytes` field then failed, and a cached run returned 0 rows. Extension columns now accept large and non-large layouts of the same type at any nesting depth, as they already did at the top level -(ITL-602). +(ITL-602). Known limitation: such a non-large column read over Arrow IPC or the C stream +interface (not Parquet or Delta) gives invalid arrays, at any depth; Orcapod's own writers +never produce one (ITL-894). ## [0.2.0] — 2026-11-10 From b930f6293c2a6499e6856be3dd1ed21cd03131bc Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 10 Nov 2026 22:43:23 -0800 Subject: [PATCH 17/31] test(types): docstring and value checks for the ITL-894 xfail (ITL-881) CodeRabbit review: an XPASS should mean a correct read, so the test also checks the decoded value, the extension name and its metadata. Co-Authored-By: Claude Opus 5.5 --- tests/test_user_types/test_read_path.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_user_types/test_read_path.py b/tests/test_user_types/test_read_path.py index 04cb8c138..a4d3cf91d 100644 --- a/tests/test_user_types/test_read_path.py +++ b/tests/test_user_types/test_read_path.py @@ -97,6 +97,7 @@ def test_stored_values_read_after_delta_compaction(tmp_path: Path) -> None: ), ) def test_ipc_read_of_non_large_nested_storage_is_valid() -> None: + """A user-class column with non-large nested storage reads correctly over IPC.""" import io narrow = pa.struct([pa.field("label", pa.string()), pa.field("n", pa.int64())]) @@ -120,3 +121,7 @@ def test_ipc_read_of_non_large_nested_storage_is_valid() -> None: writer.write_table(table) read = pa.ipc.open_stream(sink.getvalue()).read_all() read.validate(full=True) + read_type = read.schema.field("p").type + assert read_type.extension_name == ext.extension_name + assert read_type.__arrow_ext_serialize__() == ext.__arrow_ext_serialize__() + assert read.column("p").to_pylist() == [{"label": "hello", "n": 1}] From e3e84406be537dacd788084b2cba3096eeafd385 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 18:32:35 -0800 Subject: [PATCH 18/31] docs(plans): plan for field order as identity (ITL-881) Co-Authored-By: Claude Fable 5.2 --- ...2026-11-11-itl-881-field-order-identity.md | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 superpowers/plans/2026-11-11-itl-881-field-order-identity.md diff --git a/superpowers/plans/2026-11-11-itl-881-field-order-identity.md b/superpowers/plans/2026-11-11-itl-881-field-order-identity.md new file mode 100644 index 000000000..bfcf5577c --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-881-field-order-identity.md @@ -0,0 +1,82 @@ +# ITL-881 rebuild: field order is identity (PR #292) + +Continues `2026-11-10-itl-881-user-class-identity-impl.md`. Worker notes are at the end, so a +successor can pick this up. + +**Goal:** rebuild PR #292 on current `main` and change the identity of a user class so that +declared field order is part of it (developer, November 11, 2026, reversing "not order"). + +## State on arrival (November 11) + +- Worktree clean, nothing unpushed; head `b930f629`, conflicting with `main`. +- Merged `origin/main` as `ac719294` (pushed). Conflicts: CHANGELOG, DESIGN_ISSUES, mkdocs, + `delta_lake_databases.py`, `schema_utils.py`. Golden files merged cleanly and pass. +- Merge finding: main's registry MRO lookup (ITL-888) returned the base class's logical type + for a dataclass subclass, bringing back the subclass-stored-as-base bug. Fixed in the merge + commit; `test_subclass_has_its_own_identity_and_keeps_its_fields` caught it. +- Merge decision: main now wraps Delta read failures in `StoreReadError`. `TypeIdentityError` + still propagates unwrapped (the table is intact; the class differs). + +## Identity today (PR head) + +Record `{"kind", "name", "fields": {name: token}}`, canonical JSON with sorted keys, identity +string `Name@`. Fields are sorted by name in the record and in the stored struct, so +order never enters. + +## Change + +1. `identity_record`: `fields` becomes a list of `[name, token]` pairs in declared order + (`dataclasses.fields` order for `init=True` fields; `model_fields` order for pydantic). +2. Stored struct lists fields in declared order (as on `main` and v0.2.0). +3. `describe_fields` prints declared order. Read-path comparisons use the ordered list. +4. Docs, spec, CHANGELOG, DESIGN_ISSUES UCI1: "in any order" becomes "in declared order". +5. Golden: twins `GoldenTwinAB`/`GoldenTwinBA` must pin different values. They share the + declared name `golden.Twin`; with order in the identity that is two types under one name, + which is allowed (same as two field sets under one name). + +No label or version marker is introduced: the identity string is `Name@` and the +metadata is `{"category", "identity"}`. + +## Evidence (run on this branch, starfix 0.4.0) + +- starfix 0.4.0 hashes a plain `struct` and `struct` with the same values to the + same hash (order-insensitive). +- The Arrow extension name enters the hash: two extension types with different names over the + same storage hash differently. Through a `DictSource`, two classes differing only in name + have different row hash, content hash and pipeline hash, for a scalar column, a `list[...]` + column and a nested class. +- So with order in the identity string, value hashes separate on starfix 0.4.0. The PR stays + on `main`; `ordered_struct_fields` (starfix 0.5.0, PR #309) is not needed. +- Equality: a dataclass compares its fields as a tuple in declared order. Pydantic `__eq__` + compares type and `__dict__` (order-insensitive), but two models differing in order are + different types, and `model_dump`, `model_fields`, iteration and the JSON output all follow + declared order, so pod code observes it. + +## Tests first (must fail before the change) + +In `tests/test_user_types/`, through `DictSource` / function pods: + +- Same fields, other order: different type name, stored struct order follows the class, row + hash, content hash and pipeline hash differ (dataclass and pydantic). +- Subclass: own identity and all its fields, with base-first and subclass-first registration + run in two fresh processes; hash, pod uri and stored fields equal across the two orders + (dataclass and pydantic). +- Class redefined with an added field: new type, new tables, old results untouched. +- Re-run with unchanged order still rebinds silently. + +## Review findings still open + +1. CodeRabbit (minor, unanswered thread): testing a read candidate registers its nested + classes under their default names, so a later `register_type(Inner, name=...)` raises. + Fix: make the probe leave no registration behind when the candidate is rejected; add the + nested recovery test. +2. CodeRabbit nitpick (review body): assert that compaction rewrote the result table. +3. IPC read of non-large storage: answered, deferred to ITL-894 (strict xfail in place). + +## Verification + +Full suite on 3.11, 3.12, 3.13 one at a time; ruff; basedpyright; push; CI; CodeRabbit round. + +## Worker notes + +- (fill in as work proceeds) From b54b03111fc00423708bae2fa00f7fc97beb9362 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 18:38:12 -0800 Subject: [PATCH 19/31] feat(types)!: declared field order is part of a user class's identity (ITL-881) The identity record lists fields as [name, token] pairs in declared order and the stored struct follows that order. Golden values are regenerated in the next commit. Co-Authored-By: Claude Fable 5.2 --- .../dataclass_logical_type_factory.py | 2 +- .../logical_types/user_class_identity.py | 36 ++-- .../logical_types/user_class_logical_type.py | 35 ++-- src/orcapod/user_types.py | 2 +- .../test_delta_read_failures.py | 66 ++++++- .../test_dataclass_logical_type_factory.py | 2 +- .../test_pydantic_logical_type_factory.py | 2 +- tests/test_user_types/child_harness.py | 18 ++ tests/test_user_types/test_field_order.py | 169 ++++++++++++++++++ tests/test_user_types/test_identity_record.py | 41 +++-- .../test_workshop_scenarios.py | 161 ++++++++++++++++- 11 files changed, 479 insertions(+), 55 deletions(-) create mode 100644 tests/test_user_types/test_field_order.py diff --git a/src/orcapod/logical_types/dataclass_logical_type_factory.py b/src/orcapod/logical_types/dataclass_logical_type_factory.py index 978cad3f4..0f906f9df 100644 --- a/src/orcapod/logical_types/dataclass_logical_type_factory.py +++ b/src/orcapod/logical_types/dataclass_logical_type_factory.py @@ -26,7 +26,7 @@ class DataclassLogicalType(UserClassLogicalType): Example: >>> lt = DataclassLogicalType( - ... "Point@c0a75f0c39b86873", Point, + ... "Point@76d961e4...", Point, ... pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]), ... [("x", int), ("y", int)], ... ) diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index ee005cda6..e987e6c43 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -6,8 +6,10 @@ - ``kind``: ``"dataclass"`` or ``"pydantic"``. - ``name``: a declared name; otherwise ``__qualname__`` for a class defined in ``__main__`` (a notebook or script) and ``module.qualname`` for a module class. -- ``fields``: a mapping from each stored field's name to its *type token*. It is - serialized with sorted keys, so field order never enters the identity. +- ``fields``: the stored fields as ``[name, type token]`` pairs in declared order. + Order is part of the identity: a dataclass compares its fields as an ordered tuple, + and a pydantic model dumps and iterates in declared order, so two classes with the + same fields in another order are different types. The *identity string* ``f"{name}@{fingerprint}"`` is the class's logical type name and Arrow extension name. ``identity_string`` is the only place that defines its format. @@ -28,6 +30,7 @@ import unicodedata import warnings import weakref +from collections.abc import Iterable from typing import TYPE_CHECKING, Any, Literal from orcapod.utils.canonical_type_names import canonical_type_name @@ -379,22 +382,33 @@ def _import_dotted(name: str) -> Any: # --------------------------------------------------------------------------- -def identity_record(kind: str, name: str, fields: dict[str, str]) -> dict[str, Any]: +def identity_record( + kind: str, name: str, fields: Iterable[tuple[str, str]] +) -> dict[str, Any]: """Build the identity record of a user class. Args: kind: ``"dataclass"`` or ``"pydantic"``. name: The name component. - fields: Stored field name to type token. + fields: ``(stored field name, type token)`` pairs in declared order. Returns: - ``{"kind": kind, "name": name, "fields": fields}`` with the fields sorted. + ``{"kind": kind, "name": name, "fields": [[field, token], ...]}``. The fields + keep the given order, which is part of the identity. """ - return {"kind": kind, "name": name, "fields": dict(sorted(fields.items()))} + return { + "kind": kind, + "name": name, + "fields": [[field, token] for field, token in fields], + } def canonical_json(obj: Any) -> str: - """Serialize ``obj`` as canonical JSON (sorted keys, no whitespace).""" + """Serialize ``obj`` as canonical JSON (sorted object keys, no whitespace). + + Arrays keep their order, so the ``fields`` of an identity record stay in declared + order. + """ return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) @@ -409,15 +423,15 @@ def identity_string(record: dict[str, Any]) -> str: record: An identity record from ``identity_record``. Returns: - The identity string, e.g. ``"Point@9117beaaf7ebe3ab"``. + The identity string, e.g. ``"Point@76d961e4..."``. """ digest = hashlib.sha256(canonical_json(record).encode("utf-8")).hexdigest() return f"{record['name']}@{digest[:FINGERPRINT_HEX_DIGITS]}" -def describe_fields(fields: dict[str, str]) -> str: - """Render a field mapping for messages, e.g. ``{x: int, y: float}``.""" - return "{" + ", ".join(f"{k}: {v}" for k, v in sorted(fields.items())) + "}" +def describe_fields(fields: Iterable[Iterable[str]]) -> str: + """Render ``[name, type token]`` pairs in their order, e.g. ``{x: int, y: float}``.""" + return "{" + ", ".join(": ".join(pair) for pair in fields) + "}" def describe_record(record: dict[str, Any]) -> str: diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index 6d4954720..0fae88df8 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -1,9 +1,10 @@ """Shared logical type and factory for user classes (dataclasses and pydantic models). A user class's logical type name and Arrow extension name are its *identity string* -(``user_class_identity.identity_string``): the name, kind and field types, so two -classes with the same name and different fields are different types with different -tables (ITL-881). The Arrow extension metadata carries the full identity record, which +(``user_class_identity.identity_string``): the name, kind and the fields' names and +types in declared order, so two classes with the same name and different fields, or +the same fields in another order, are different types with different tables +(ITL-881). The Arrow extension metadata carries the full identity record, which the read path uses to find the class and to describe a mismatch. Write path (``create_for_python_type``): @@ -46,11 +47,11 @@ class UserClassLogicalType(BaseLogicalType): """Logical type binding a user class to its Arrow extension type. The logical type name and Arrow extension name are the class's identity string; - the storage type is a ``pa.struct`` of the stored fields. Subclasses set - ``category``. + the storage type is a ``pa.struct`` of the stored fields in declared order. + Subclasses set ``category``. Args: - logical_name: The identity string (e.g. ``"Point@9117beaaf7ebe3ab"``). + logical_name: The identity string (e.g. ``"Point@76d961e4..."``). python_type: The class. storage_type: The Arrow ``pa.StructType`` of the stored fields. field_annotations: ``(field_name, python_annotation)`` pairs matching the @@ -259,10 +260,10 @@ def _describe( arrow_fields = [] field_annotations: list[tuple[str, Any]] = [] - tokens: dict[str, str] = {} - # Sorted by name: field order is not part of the identity, so it must not - # change the storage type either (same identity, same struct). - for field_name in sorted(self.stored_field_names(python_type)): + tokens: list[tuple[str, str]] = [] + # Declared order: it is part of the identity, and the storage struct follows + # it, so one identity always has one struct. + for field_name in self.stored_field_names(python_type): annotation = hints.get(field_name, Any) try: arrow_type = converter.register_python_class(annotation) @@ -277,7 +278,7 @@ def _describe( arrow_type = arrow_type.storage_type arrow_fields.append(pa.field(field_name, arrow_type)) field_annotations.append((field_name, annotation)) - tokens[field_name] = uci.type_token(annotation, converter) + tokens.append((field_name, uci.type_token(annotation, converter))) record = uci.identity_record(self.kind, name, tokens) return record, pa.struct(arrow_fields), field_annotations @@ -321,7 +322,11 @@ def reconstruct_from_arrow( ) name = record.get("name") kind = record.get("kind") - if kind != self.kind or not isinstance(name, str): + if ( + kind != self.kind + or not isinstance(name, str) + or not isinstance(record.get("fields"), list) + ): raise TypeIdentityError( f"Stored column type {arrow_extension_name!r} has an invalid identity " f"record for category {self.category!r}: {record!r}." @@ -377,9 +382,9 @@ def reconstruct_from_arrow( f"as {name!r} before it is first used, e.g. " f"op.register_type(, name={name!r})." if renamable - else " A class with the same name but different fields is a different " - "type; restore the stored fields, or use the class with the new fields " - "to compute new results." + else " A class with the same name but different fields, or the same fields " + "in another order, is a different type; restore the stored fields and " + "their order, or use the class as it is now to compute new results." ) raise TypeIdentityError( f"Stored values of {uci.describe_record(record)} do not match the classes " diff --git a/src/orcapod/user_types.py b/src/orcapod/user_types.py index 1db0d9baa..41afb0022 100644 --- a/src/orcapod/user_types.py +++ b/src/orcapod/user_types.py @@ -1,7 +1,7 @@ """Registering your own data types (dataclasses and pydantic models). Orcapod identifies a dataclass or pydantic model by its **name**, its **kind** -(dataclass or pydantic) and its **fields** (names and types, in any order). Two classes +(dataclass or pydantic) and its **fields** (names and types, in declared order). Two classes that agree on all three are the same type: results stored with one are found again with the other. See ``docs/concepts/your-own-data-types.md``. diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index e40ca3514..728e490da 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -11,10 +11,13 @@ anything is written, naming the column and both types; legitimate appends (same types in another layout, new or missing columns, unchanged extension storage) still succeed. +- A readable table whose stored user-class shape differs from the registered class + raises ``TypeIdentityError``, not ``StoreReadError`` (ITL-881). """ from __future__ import annotations +import dataclasses import datetime import json from collections.abc import Callable @@ -27,7 +30,8 @@ from deltalake.exceptions import DeltaProtocolError from orcapod.databases import DeltaTableDatabase -from orcapod.errors import InputValidationError, StoreReadError +from orcapod.errors import InputValidationError, StoreReadError, TypeIdentityError +from orcapod.sources import DictSource PATH = ("results", "t") @@ -496,3 +500,63 @@ def test_nullability_only_difference_is_accepted(self, base): back = self.read_back(base) assert back.num_rows == 2 assert back.schema.field("x").type == loose + + +# --------------------------------------------------------------------------- +# A readable table whose class differs is not an unreadable table (ITL-881) +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class _StoredShape: + """A registered user class; the table below stores another shape under its name.""" + + x: int + y: int + + +@pytest.fixture +def other_shape(base: Path) -> DeltaTableDatabase: + """A readable table storing ``_StoredShape``'s type name over another struct.""" + value: Any = _StoredShape(1, 2) + source = DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) + ext = source.as_table().schema.field("p").type + wrong = pa.struct([pa.field("x", pa.float64()), pa.field("y", pa.int64())]) + field = pa.field("x", wrong).with_metadata( + { + b"ARROW:extension:name": ext.extension_name.encode(), + b"ARROW:extension:metadata": ext.__arrow_ext_serialize__(), + } + ) + table = pa.table( + { + "__record_id": pa.array([b"a"], pa.large_binary()), + "x": pa.array([{"x": 1.5, "y": 2}], type=wrong), + }, + schema=pa.schema([pa.field("__record_id", pa.large_binary()), field]), + ) + deltalake.write_deltalake(str(table_dir(base)), table) + return DeltaTableDatabase(base_path=base) + + +READS_WITHOUT_VALUE_FILTER = { + name: read + for name, read in READS.items() + if name != "get_records_with_column_value" +} + + +@pytest.mark.parametrize( + "read", + READS_WITHOUT_VALUE_FILTER.values(), + ids=list(READS_WITHOUT_VALUE_FILTER), +) +def test_class_mismatch_is_type_identity_error_not_store_read_error(other_shape, read): + """A stored shape that differs from the registered class raises ``TypeIdentityError``. + + ``StoreReadError`` means the table cannot be read. Here the table is intact and the + class differs, so the identity error is raised as is, never wrapped. + """ + with pytest.raises(TypeIdentityError) as info: + read(other_shape) + assert not isinstance(info.value, StoreReadError) diff --git a/tests/test_logical_types/test_dataclass_logical_type_factory.py b/tests/test_logical_types/test_dataclass_logical_type_factory.py index 415103d35..fccc8e723 100644 --- a/tests/test_logical_types/test_dataclass_logical_type_factory.py +++ b/tests/test_logical_types/test_dataclass_logical_type_factory.py @@ -441,7 +441,7 @@ def test_factory_reconstruct_from_arrow_invalid_fqcn(): "identity": { "kind": "dataclass", "name": "nonexistent.module.NoSuchClass", - "fields": {"x": "int"}, + "fields": [["x", "int"]], }, }, converter, diff --git a/tests/test_logical_types/test_pydantic_logical_type_factory.py b/tests/test_logical_types/test_pydantic_logical_type_factory.py index a3bce9a06..37b36d4f4 100644 --- a/tests/test_logical_types/test_pydantic_logical_type_factory.py +++ b/tests/test_logical_types/test_pydantic_logical_type_factory.py @@ -469,7 +469,7 @@ def test_factory_reconstruct_from_arrow_invalid_fqcn(): "identity": { "kind": "pydantic", "name": "nonexistent.module.NoSuchClass", - "fields": {"x": "int"}, + "fields": [["x", "int"]], }, }, converter, diff --git a/tests/test_user_types/child_harness.py b/tests/test_user_types/child_harness.py index c28b944a2..92bf3f786 100644 --- a/tests/test_user_types/child_harness.py +++ b/tests/test_user_types/child_harness.py @@ -108,6 +108,21 @@ def table_paths(root): ) +def stored_fields(root, column): + """Return the field names of each stored struct column ``column``, as written.""" + import deltalake + import pyarrow as pa + + found = [] + for rel in table_paths(root): + schema = pa.schema(deltalake.DeltaTable(str(Path(root) / rel)).schema().to_arrow()) + if column in schema.names: + stored = schema.field(column).type + stored = getattr(stored, "storage_type", stored) + found.append([field.name for field in stored]) + return sorted(found) + + def make_pod(point_cls): """Return ``(pod, calls)`` for ``shift(p: point_cls) -> point_cls``.""" calls = [] @@ -139,7 +154,10 @@ def run_job(point_cls, pod_and_calls=None, rows=None): "calls": len(calls) - before, "rows": node.as_table().num_rows, "q_class_is_point": all(type(v) is point_cls for v in values), + "values": sorted(repr(v) for v in values), + "stored_fields": stored_fields(db_root, "q"), "pod_pipeline_hash": pod.pod.pipeline_hash().to_string(), + "pod_uri": list(pod.pod.uri), "node_identity_path": list(node.node_identity_path), "table_paths": table_paths(db_root), "extension_name": getattr( diff --git a/tests/test_user_types/test_field_order.py b/tests/test_user_types/test_field_order.py new file mode 100644 index 000000000..76992a256 --- /dev/null +++ b/tests/test_user_types/test_field_order.py @@ -0,0 +1,169 @@ +"""Declared field order is part of a user class's identity (ITL-881). + +A dataclass compares its fields as a tuple in declared order, and a pydantic model +dumps, iterates and serializes in declared order, so pod code can observe the order. +Two classes with the same fields in another order are therefore different types. + +Identity is observed through the public interface: the Arrow type of a ``DictSource`` +column, the row's ``content_hash`` and the source's ``content_hash`` and +``pipeline_hash``. Names under ``itl881.order.`` are used by no other test, because +declarations are process-wide. +""" + +from __future__ import annotations + +import dataclasses +from typing import Any + +import pyarrow as pa +import pydantic +import pytest + +import orcapod as op +from orcapod import function_pod +from orcapod.sources import DictSource + + +@op.register_type(name="itl881.order.Data") +@dataclasses.dataclass +class DataXY: + x: int + y: int + + +@op.register_type(name="itl881.order.Data") +@dataclasses.dataclass +class DataYX: + y: int + x: int + + +@op.register_type(name="itl881.order.Model") +class ModelXY(pydantic.BaseModel): + x: int + y: int + + +@op.register_type(name="itl881.order.Model") +class ModelYX(pydantic.BaseModel): + y: int + x: int + + +@op.register_type(name="itl881.order.Outer") +@dataclasses.dataclass +class OuterOfXY: + inner: DataXY + + +@op.register_type(name="itl881.order.Outer") +@dataclasses.dataclass +class OuterOfYX: + inner: DataYX + + +PAIRS: dict[str, tuple[Any, Any]] = { + "dataclass": (DataXY(x=1, y=2), DataYX(x=1, y=2)), + "pydantic": (ModelXY(x=1, y=2), ModelYX(x=1, y=2)), + "list": ([DataXY(x=1, y=2)], [DataYX(x=1, y=2)]), + "nested": (OuterOfXY(DataXY(x=1, y=2)), OuterOfYX(DataYX(x=1, y=2))), +} + + +def _source(value: Any) -> DictSource: + return DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"], source_id="s") + + +def _column_type(value: Any) -> pa.DataType: + return _source(value).as_table().schema.field("p").type + + +def _type_name(value: Any) -> str: + return _column_type(value).extension_name + + +def _row_hash(value: Any) -> str: + _, data = next(iter(_source(value).iter_data())) + return data.content_hash().to_string() + + +@pytest.fixture(params=list(PAIRS)) +def pair(request: pytest.FixtureRequest) -> tuple[Any, Any]: + """Two values that differ only in the declared field order of their class.""" + return PAIRS[request.param] + + +def test_other_field_order_is_another_type(pair: tuple[Any, Any]) -> None: + first, second = pair + assert _type_name(first) != _type_name(second) + + +def test_other_field_order_has_other_hashes(pair: tuple[Any, Any]) -> None: + first, second = pair + assert _row_hash(first) != _row_hash(second) + assert _source(first).content_hash() != _source(second).content_hash() + assert _source(first).pipeline_hash() != _source(second).pipeline_hash() + + +@pytest.mark.parametrize( + ("value", "names"), + [ + (DataXY(x=1, y=2), ["x", "y"]), + (DataYX(x=1, y=2), ["y", "x"]), + (ModelXY(x=1, y=2), ["x", "y"]), + (ModelYX(x=1, y=2), ["y", "x"]), + ], +) +def test_stored_struct_lists_fields_in_declared_order( + value: Any, names: list[str] +) -> None: + storage = _column_type(value).storage_type + assert [field.name for field in storage] == names + + +def test_values_of_both_orders_read_back_as_their_own_class( + pair: tuple[Any, Any], +) -> None: + for value in pair: + _, data = next(iter(_source(value).iter_data())) + read = data.as_dict()["p"] + assert read == value + assert type(read) is type(value) + + +def test_pods_over_the_two_orders_have_different_pipeline_hashes() -> None: + @function_pod(output_keys="q") + def keep_xy(p: DataXY) -> DataXY: + return p + + @function_pod(output_keys="q") + def keep_yx(p: DataYX) -> DataYX: + return p + + xy = keep_xy.pod(_source(DataXY(x=1, y=2))) + yx = keep_yx.pod(_source(DataYX(x=1, y=2))) + assert xy.pipeline_hash() != yx.pipeline_hash() + assert ( + xy.as_table().schema.field("q").type.extension_name + != yx.as_table().schema.field("q").type.extension_name + ) + + +@op.register_type(name="itl881.order.Same") +@dataclasses.dataclass +class SameFirst: + a: int + b: str + + +@op.register_type(name="itl881.order.Same") +@dataclasses.dataclass +class SameSecond: + a: int + b: str + + +def test_same_fields_in_the_same_order_are_one_type() -> None: + with pytest.warns(UserWarning, match="same identity"): + assert _type_name(SameFirst(1, "x")) == _type_name(SameSecond(1, "x")) + assert _row_hash(SameFirst(1, "x")) == _row_hash(SameSecond(1, "x")) diff --git a/tests/test_user_types/test_identity_record.py b/tests/test_user_types/test_identity_record.py index 5aa94891e..dc85b56b8 100644 --- a/tests/test_user_types/test_identity_record.py +++ b/tests/test_user_types/test_identity_record.py @@ -32,43 +32,48 @@ def _make_class(module: str, qualname: str = "Point") -> type: class TestIdentityString: def test_documented_example(self) -> None: - record = uci.identity_record("dataclass", "Point", {"x": "int", "y": "float"}) + record = uci.identity_record( + "dataclass", "Point", [("x", "int"), ("y", "float")] + ) assert uci.identity_string(record) == ( - "Point@9117beaaf7ebe3ab0bd25de9448c92eb64aa070a58611517bd1e470a544584d8" + "Point@76d961e40619a836535e3fdebdead7ea65aa9413955a889f3d5b51a7a94b65eb" ) - def test_canonical_json_is_sorted_and_compact(self) -> None: - record = uci.identity_record("pydantic", "P", {"b": "int", "a": "str"}) + def test_canonical_json_is_compact_and_keeps_field_order(self) -> None: + record = uci.identity_record("pydantic", "P", [("b", "int"), ("a", "str")]) assert ( uci.canonical_json(record) - == '{"fields":{"a":"str","b":"int"},"kind":"pydantic","name":"P"}' + == '{"fields":[["b","int"],["a","str"]],"kind":"pydantic","name":"P"}' ) - def test_field_order_does_not_matter(self) -> None: - a = uci.identity_record("dataclass", "P", {"x": "int", "y": "str"}) - b = uci.identity_record("dataclass", "P", {"y": "str", "x": "int"}) - assert uci.identity_string(a) == uci.identity_string(b) + def test_field_order_is_part_of_the_identity(self) -> None: + a = uci.identity_record("dataclass", "P", [("x", "int"), ("y", "str")]) + b = uci.identity_record("dataclass", "P", [("y", "str"), ("x", "int")]) + assert uci.identity_string(a) != uci.identity_string(b) @pytest.mark.parametrize( "other", [ - ("pydantic", "Point", {"x": "int", "y": "float"}), - ("dataclass", "Point2", {"x": "int", "y": "float"}), - ("dataclass", "Point", {"x": "int", "y": "int"}), - ("dataclass", "Point", {"x": "int", "z": "float"}), - ("dataclass", "Point", {"x": "int"}), + ("pydantic", "Point", [("x", "int"), ("y", "float")]), + ("dataclass", "Point2", [("x", "int"), ("y", "float")]), + ("dataclass", "Point", [("x", "int"), ("y", "int")]), + ("dataclass", "Point", [("x", "int"), ("z", "float")]), + ("dataclass", "Point", [("x", "int")]), + ("dataclass", "Point", [("y", "float"), ("x", "int")]), ], ) def test_each_component_changes_the_identity( - self, other: tuple[str, str, dict[str, str]] + self, other: tuple[str, str, list[tuple[str, str]]] ) -> None: - base = uci.identity_record("dataclass", "Point", {"x": "int", "y": "float"}) + base = uci.identity_record("dataclass", "Point", [("x", "int"), ("y", "float")]) assert uci.identity_string(base) != uci.identity_string( uci.identity_record(*other) ) - def test_describe_fields(self) -> None: - assert uci.describe_fields({"y": "float", "x": "int"}) == "{x: int, y: float}" + def test_describe_fields_keeps_the_given_order(self) -> None: + assert ( + uci.describe_fields([["y", "float"], ["x", "int"]]) == "{y: float, x: int}" + ) class TestTypeTokens: diff --git a/tests/test_user_types/test_workshop_scenarios.py b/tests/test_user_types/test_workshop_scenarios.py index 1937888ee..3c24903e2 100644 --- a/tests/test_user_types/test_workshop_scenarios.py +++ b/tests/test_user_types/test_workshop_scenarios.py @@ -27,6 +27,8 @@ "setup": "import wsjob\nfrom dataclasses import dataclass\nimport orcapod as op\n", "define": "@dataclass\nclass Point:\n x: int\n y: int\n", "define_reordered": "@dataclass\nclass Point:\n y: int\n x: int\n", + "define_added": "@dataclass\nclass Point:\n x: int\n y: int\n z: int = 0\n", + "define_model_reordered": "import pydantic\nclass Point(pydantic.BaseModel):\n y: int\n x: int\n", "define_float": "@dataclass\nclass Point:\n x: int\n y: float\n", "define_model": "import pydantic\nclass Point(pydantic.BaseModel):\n x: int\n y: int\n", "import_mod": "from wsmodels import Point\n", @@ -88,6 +90,118 @@ def _script(workdir: Path, code: str, **env: str) -> Any: ) +# --------------------------------------------------------------------------- +# Subclasses, whatever the registration order (audit ITL-942, finding R) +# --------------------------------------------------------------------------- + +WSFAMILY = """ +from dataclasses import dataclass + +import pydantic + + +@dataclass +class Base: + x: int + + +@dataclass +class Child(Base): + z: int = 0 + + +class BaseModel(pydantic.BaseModel): + x: int + + +class ChildModel(BaseModel): + z: int = 0 +""" + +FAMILY_SCRIPT = """ +import os + +import wsfamily +import wsjob +from child_harness import report +from orcapod.sources import DictSource + +base = getattr(wsfamily, os.environ["WS_BASE"]) +child = getattr(wsfamily, os.environ["WS_CHILD"]) + + +def use_base(): + source = DictSource(data=[{"k": "a", "p": base(x=1)}], tag_columns=["k"]) + return source.as_table().schema.field("p").type.extension_name + + +def use_child(): + return wsjob.run_job(child, rows=[{"x": 1, "z": 10}, {"x": 2, "z": 20}]) + + +out = {} +if os.environ["WS_ORDER"] == "base_first": + out["base_name"] = use_base() + out["child"] = use_child() +else: + out["child"] = use_child() + out["base_name"] = use_base() +report(out) +""" + + +@pytest.mark.parametrize( + ("base", "child"), + [("Base", "Child"), ("BaseModel", "ChildModel")], + ids=["dataclass", "pydantic"], +) +def test_subclass_identity_and_fields_do_not_depend_on_registration_order( + workdir: Path, base: str, child: str +) -> None: + """A subclass keeps its own identity and all its fields in both orders. + + Each order runs in a fresh process against a fresh store. On ``main`` the subclass + was stored as its base when the base was registered first: its extra field was + dropped, and the pod's hash and stored path depended on the order. + """ + write_files(workdir, {"wsfamily.py": WSFAMILY}) + out: dict[str, Any] = {} + for order in ("base_first", "sub_first"): + (workdir / order).mkdir() + out[order] = run_child( + ["-c", FAMILY_SCRIPT], + cwd=workdir, + env={ + "WS_DB": str(workdir / order), + "WS_ORDER": order, + "WS_BASE": base, + "WS_CHILD": child, + }, + ) + + for order, result in out.items(): + run = result["child"] + assert (run["calls"], run["rows"]) == (2, 2), order + assert run["q_class_is_point"], order + assert run["extension_name"].split("@")[0] == f"wsfamily.{child}", order + assert run["extension_name"] != result["base_name"], order + assert run["stored_fields"] == [["x", "z"]], order + assert run["values"] == [f"{child}(x=1, z=10)", f"{child}(x=2, z=20)"], order + + first, second = out["base_first"], out["sub_first"] + assert first["base_name"] == second["base_name"] + for key in ( + "extension_name", + "pod_pipeline_hash", + "pod_uri", + "node_identity_path", + "table_paths", + "stored_fields", + "values", + ): + assert first["child"][key] == second["child"][key], key + + # --------------------------------------------------------------------------- # Notebook cell re-runs in one kernel # --------------------------------------------------------------------------- @@ -115,11 +229,46 @@ def test_w1_run_all_twice(workdir: Path) -> None: assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) -def test_field_order_does_not_change_identity(workdir: Path) -> None: - log = _kernel(workdir, ["setup", "define", "run", "define_reordered", "run"]) - first, second = _runs(log) - assert second["calls"] == 0 - assert first["extension_name"] == second["extension_name"] +@pytest.mark.parametrize( + ("define", "reordered"), + [("define", "define_reordered"), ("define_model", "define_model_reordered")], + ids=["dataclass", "pydantic"], +) +def test_reordered_fields_are_a_new_type( + workdir: Path, define: str, reordered: str +) -> None: + """Declared field order is identity: the reordered class computes into new tables.""" + log = _kernel(workdir, ["setup", define, "run", reordered, "run", define, "run"]) + first, second, third = _runs(log) + assert (second["calls"], second["rows"]) == (2, 2) + assert second["q_class_is_point"] + assert first["extension_name"] != second["extension_name"] + assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] + assert first["pod_uri"] != second["pod_uri"] + assert set(first["table_paths"]) < set(second["table_paths"]) + assert first["stored_fields"] == [["x", "y"]] + assert second["stored_fields"] == [["x", "y"], ["y", "x"]] + # The first order's results are untouched and found again. + assert (third["calls"], third["rows"]) == (0, 2) + assert third["pod_pipeline_hash"] == first["pod_pipeline_hash"] + + +def test_class_redefined_with_an_added_field_is_a_new_type(workdir: Path) -> None: + """An added field (even with a default) makes a new type with its own tables.""" + log = _kernel( + workdir, ["setup", "define", "run", "define_added", "run", "define", "run"] + ) + first, second, third = _runs(log) + assert (second["calls"], second["rows"]) == (2, 2) + assert second["q_class_is_point"] + assert first["extension_name"] != second["extension_name"] + assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] + assert set(first["table_paths"]) < set(second["table_paths"]) + assert second["stored_fields"] == [["x", "y"], ["x", "y", "z"]] + assert second["values"] == ["Point(x=1, y=1, z=1)", "Point(x=2, y=2, z=2)"] + # The two-field results are untouched and found again. + assert (third["calls"], third["rows"]) == (0, 2) + assert third["values"] == first["values"] def test_w2_rerun_with_changed_fields_is_a_new_type(workdir: Path) -> None: @@ -261,8 +410,8 @@ def test_w11_notebook_to_module_with_declared_name(workdir: Path) -> None: @op.register_type(name="Point") @dataclass class Point: - y: int x: int + y: int """, }, ) From d5357eb6b40b556bf4d24db3a226fcae9e35632a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 18:40:09 -0800 Subject: [PATCH 20/31] test(hashing): golden values for field order as identity (ITL-881) Relative to main the same 25 approved entries move (17 changed, 8 replaced); arrow.json and pipeline.json are identical to main. The field-order twins now pin different values, and a dataclass with a NumPy array field is added. Co-Authored-By: Claude Fable 5.2 --- tests/test_hashing/golden/cases.py | 23 ++- tests/test_hashing/golden/data/datagram.json | 152 ++++++++++--------- tests/test_hashing/golden/data/schema.json | 36 ++--- tests/test_hashing/golden/data/semantic.json | 17 ++- 4 files changed, 131 insertions(+), 97 deletions(-) diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 3e0580e45..9626ae5ee 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -227,9 +227,21 @@ class GoldenOuter: tags: list[str] +@dataclasses.dataclass +class GoldenWithArray: + """Dataclass with a NumPy array field; the array's logical type enters its identity.""" + + label: str + values: np.ndarray + + @dataclasses.dataclass class GoldenTwinAB: - """Declared ``golden.Twin``; same fields as ``GoldenTwinBA`` in another order.""" + """Declared ``golden.Twin``; same fields as ``GoldenTwinBA`` in another order. + + Declared field order is part of the identity (ITL-881), so the twins are two types + and must pin different values. + """ a: int b: str @@ -237,7 +249,7 @@ class GoldenTwinAB: @dataclasses.dataclass class GoldenTwinBA: - """Declared ``golden.Twin``; must pin the same values as ``GoldenTwinAB``.""" + """Declared ``golden.Twin``; must pin values different from ``GoldenTwinAB``'s.""" b: str a: int @@ -253,6 +265,7 @@ def _declare_golden_names() -> None: register_type(GoldenDeclaredModel, name="golden.Declared") register_type(GoldenInner, name="golden.Inner") register_type(GoldenOuter, name="golden.Outer") + register_type(GoldenWithArray, name="golden.WithArray") register_type(GoldenTwinAB, name="golden.Twin") register_type(GoldenTwinBA, name="golden.Twin") @@ -269,6 +282,7 @@ def register_logical_classes() -> None: GoldenDeclared, GoldenDeclaredModel, GoldenOuter, + GoldenWithArray, GoldenTwinAB, GoldenTwinBA, ): @@ -354,6 +368,7 @@ def _semantic_types() -> dict[str, Any]: "dataclass_declared": GoldenDeclared, "pydantic_declared": GoldenDeclaredModel, "dataclass_nested": GoldenOuter, + "dataclass_with_array": GoldenWithArray, "dataclass_twin_ab": GoldenTwinAB, "dataclass_twin_ba": GoldenTwinBA, "list_dataclass": list[GoldenPoint], @@ -675,6 +690,10 @@ def _datagram_rows() -> dict[str, list[dict[str, Any]]]: {"k": "r1", "c": GoldenOuter(GoldenInner(1), ["x", "y"])}, {"k": "r2", "c": GoldenOuter(GoldenInner(2), [])}, ], + "dataclass_with_array": [ + {"k": "r1", "c": GoldenWithArray("a", np.arange(3, dtype=np.float32))}, + {"k": "r2", "c": GoldenWithArray("b", np.zeros((2, 2), dtype=np.int64))}, + ], "dataclass_twin_ab": [ {"k": "r1", "c": GoldenTwinAB(1, "x")}, {"k": "r2", "c": GoldenTwinAB(2, "y")}, diff --git a/tests/test_hashing/golden/data/datagram.json b/tests/test_hashing/golden/data/datagram.json index 91b28ad77..5347f9e2b 100644 --- a/tests/test_hashing/golden/data/datagram.json +++ b/tests/test_hashing/golden/data/datagram.json @@ -89,65 +89,77 @@ "_tag_source_id::b58c50c5ff232cdb71eb0272547485074c9784600f5a34218434f7c3de419e8e", "_tag_record_id::b58c50c5ff232cdb71eb0272547485074c9784600f5a34218434f7c3de419e8e" ], - "dataclass.content_hash": "semantic_v0.1:c5a9e1b35bfc3292f482d92c6da779430f90266b4969471c3c9dda92babb38e4", - "dataclass.pipeline_hash": "semantic_v0.1:7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e", - "dataclass.row[{\"k\":\"r1\"}]._tag_record_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e": "26db93a5eab45b0485c2699148b0600d", - "dataclass.row[{\"k\":\"r1\"}]._tag_source_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e": "golden_dataclass", - "dataclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001bf6f92863abf8af4b0aac7124cdef0ddd5389e4d8cf951ac5ec452517f8a073b", - "dataclass.row[{\"k\":\"r2\"}]._tag_record_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e": "71d5c6ce2f7b5b5b9dfdc7a61a7df366", - "dataclass.row[{\"k\":\"r2\"}]._tag_source_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e": "golden_dataclass", - "dataclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b0d45b9c6372b65c8a2be29561177153c1711e2ed1aab8323d83cfc084d52564", + "dataclass.content_hash": "semantic_v0.1:99b952a1ba65c4c6a7ade08a327eead140aa163cbcc86d179cf9a3fa4a9d93fd", + "dataclass.pipeline_hash": "semantic_v0.1:8f023e91993ce15bb390c5d18bcc48ec22e6451b9bbed42b06301f3e967f1fc6", + "dataclass.row[{\"k\":\"r1\"}]._tag_record_id::8f023e91993ce15bb390c5d18bcc48ec22e6451b9bbed42b06301f3e967f1fc6": "26db93a5eab45b0485c2699148b0600d", + "dataclass.row[{\"k\":\"r1\"}]._tag_source_id::8f023e91993ce15bb390c5d18bcc48ec22e6451b9bbed42b06301f3e967f1fc6": "golden_dataclass", + "dataclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000177e95189e842fe5e881334901c9330d3df9c29252bb424e43b49aad3b4ef83bf", + "dataclass.row[{\"k\":\"r2\"}]._tag_record_id::8f023e91993ce15bb390c5d18bcc48ec22e6451b9bbed42b06301f3e967f1fc6": "71d5c6ce2f7b5b5b9dfdc7a61a7df366", + "dataclass.row[{\"k\":\"r2\"}]._tag_source_id::8f023e91993ce15bb390c5d18bcc48ec22e6451b9bbed42b06301f3e967f1fc6": "golden_dataclass", + "dataclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f2b70d12c3f114f6aa7ee66ceb317321e8499ae60e3792347928044ae9acb5e0", "dataclass.system_tag_columns": [ - "_tag_source_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e", - "_tag_record_id::7ec5213aa2a100ec89a8e49a6b8983e529d90154d6b6a061c8743c776f921b3e" + "_tag_source_id::8f023e91993ce15bb390c5d18bcc48ec22e6451b9bbed42b06301f3e967f1fc6", + "_tag_record_id::8f023e91993ce15bb390c5d18bcc48ec22e6451b9bbed42b06301f3e967f1fc6" ], - "dataclass_declared.content_hash": "semantic_v0.1:e6245cec578022a262899f3887df76aa88814cd15febaf5d0e1da03c21fef5d7", - "dataclass_declared.pipeline_hash": "semantic_v0.1:7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf", - "dataclass_declared.row[{\"k\":\"r1\"}]._tag_record_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf": "570e538241085842a7994cad88c5a26b", - "dataclass_declared.row[{\"k\":\"r1\"}]._tag_source_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf": "golden_dataclass_declared", - "dataclass_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000100c60f00122ee9d0689aa5f61af45feeaeaa37c636cd91f711ccaf5eeab6ff7d", - "dataclass_declared.row[{\"k\":\"r2\"}]._tag_record_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf": "0e35aec1a9605d428b2741d9f13012d5", - "dataclass_declared.row[{\"k\":\"r2\"}]._tag_source_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf": "golden_dataclass_declared", - "dataclass_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:0000016146a4d84d42a55151513f89988bc964b7c813f2384484a2ddaa0e8249e348d5", + "dataclass_declared.content_hash": "semantic_v0.1:1ba8f2301bcf4dfe11920fffc5db3dbe5c3bfc19b51d7c8b150ceac1d8a572a8", + "dataclass_declared.pipeline_hash": "semantic_v0.1:c5816ce94b492ebf33604d342c8ec6f01d89a8875c085ee5bfa76e7abefb6635", + "dataclass_declared.row[{\"k\":\"r1\"}]._tag_record_id::c5816ce94b492ebf33604d342c8ec6f01d89a8875c085ee5bfa76e7abefb6635": "570e538241085842a7994cad88c5a26b", + "dataclass_declared.row[{\"k\":\"r1\"}]._tag_source_id::c5816ce94b492ebf33604d342c8ec6f01d89a8875c085ee5bfa76e7abefb6635": "golden_dataclass_declared", + "dataclass_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000017354a206848bc2a7650b59338dbcafa35596a2b2db65db6bc6fa26584f92fcb3", + "dataclass_declared.row[{\"k\":\"r2\"}]._tag_record_id::c5816ce94b492ebf33604d342c8ec6f01d89a8875c085ee5bfa76e7abefb6635": "0e35aec1a9605d428b2741d9f13012d5", + "dataclass_declared.row[{\"k\":\"r2\"}]._tag_source_id::c5816ce94b492ebf33604d342c8ec6f01d89a8875c085ee5bfa76e7abefb6635": "golden_dataclass_declared", + "dataclass_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001194f7f136d0fa72bd7f3a5cbf538a1eb7b5b0563cbaad2687169a7079482b5a1", "dataclass_declared.system_tag_columns": [ - "_tag_source_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf", - "_tag_record_id::7173cbcdb0a61df572dce8cd566fb2fdee7efa6ba1bebc2470f9c55178389baf" + "_tag_source_id::c5816ce94b492ebf33604d342c8ec6f01d89a8875c085ee5bfa76e7abefb6635", + "_tag_record_id::c5816ce94b492ebf33604d342c8ec6f01d89a8875c085ee5bfa76e7abefb6635" ], - "dataclass_nested.content_hash": "semantic_v0.1:6980d157081d777cdab23f84ed44901566e0f0b07977b980a4cc2c4af8fc327e", - "dataclass_nested.pipeline_hash": "semantic_v0.1:ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6", - "dataclass_nested.row[{\"k\":\"r1\"}]._tag_record_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6": "3b11b0ab021d59f28c8efaf02ad69aee", - "dataclass_nested.row[{\"k\":\"r1\"}]._tag_source_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6": "golden_dataclass_nested", - "dataclass_nested.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:0000013e4d70f87128e09feac88ca6a1473bb286b74369f99e120b7eef26465e0b83e2", - "dataclass_nested.row[{\"k\":\"r2\"}]._tag_record_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6": "178f8da0e9ed5520827e648f5a890193", - "dataclass_nested.row[{\"k\":\"r2\"}]._tag_source_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6": "golden_dataclass_nested", - "dataclass_nested.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001a4947a9cee130966d37ef41907b9c4704a87bc4e9521356953b3e6629ce00e2b", + "dataclass_nested.content_hash": "semantic_v0.1:1ac6d4e0ec3d7c8576691881c94ad0b2cf4edeef851407d6c2299e5f3168571f", + "dataclass_nested.pipeline_hash": "semantic_v0.1:69ace2c031c299994b74a158e9e378ad022462c73c3425940908ab3feb6cd8e1", + "dataclass_nested.row[{\"k\":\"r1\"}]._tag_record_id::69ace2c031c299994b74a158e9e378ad022462c73c3425940908ab3feb6cd8e1": "3b11b0ab021d59f28c8efaf02ad69aee", + "dataclass_nested.row[{\"k\":\"r1\"}]._tag_source_id::69ace2c031c299994b74a158e9e378ad022462c73c3425940908ab3feb6cd8e1": "golden_dataclass_nested", + "dataclass_nested.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001c7eb019bd01bb4976d553d4118dcc3b723341f122539e46432fb69ce84541f28", + "dataclass_nested.row[{\"k\":\"r2\"}]._tag_record_id::69ace2c031c299994b74a158e9e378ad022462c73c3425940908ab3feb6cd8e1": "178f8da0e9ed5520827e648f5a890193", + "dataclass_nested.row[{\"k\":\"r2\"}]._tag_source_id::69ace2c031c299994b74a158e9e378ad022462c73c3425940908ab3feb6cd8e1": "golden_dataclass_nested", + "dataclass_nested.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001545bfd8ceeb6dbf0b37e90453efb8dd3a87683e3cb68de16b88c1eaf95333d9f", "dataclass_nested.system_tag_columns": [ - "_tag_source_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6", - "_tag_record_id::ed1742fde2b22c4bf603b673363a85adf2c84a747caeda527e74575f96465be6" + "_tag_source_id::69ace2c031c299994b74a158e9e378ad022462c73c3425940908ab3feb6cd8e1", + "_tag_record_id::69ace2c031c299994b74a158e9e378ad022462c73c3425940908ab3feb6cd8e1" ], - "dataclass_twin_ab.content_hash": "semantic_v0.1:bfe47ccef93a7e2eb6f9becbe4325d4467422070c4a325c73a94a64e3194250a", - "dataclass_twin_ab.pipeline_hash": "semantic_v0.1:9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc", - "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "f35be6f60b275f2c81fbc49366c2db67", - "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "golden_dataclass_twin_ab", - "dataclass_twin_ab.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000125e527402024a34d9d53489982ae2634a022bc6f9a0b8249606df28a66aa118c", - "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "7307d67233695c9ab4952175e9735bc9", - "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "golden_dataclass_twin_ab", - "dataclass_twin_ab.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f0b425f53eb41258940eaf8c37d0c18b1f3be4ecbde6969cba18ee8929013a63", + "dataclass_twin_ab.content_hash": "semantic_v0.1:aa9eff19f5f95aa420458fcf019e194d80c3dd6670e7f08d2093e38a5b73f1f9", + "dataclass_twin_ab.pipeline_hash": "semantic_v0.1:56b62fdc3c36dc6bffeb568f50e1ad4ad37ba2646803ff294ac558a8ceaa4838", + "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_record_id::56b62fdc3c36dc6bffeb568f50e1ad4ad37ba2646803ff294ac558a8ceaa4838": "f35be6f60b275f2c81fbc49366c2db67", + "dataclass_twin_ab.row[{\"k\":\"r1\"}]._tag_source_id::56b62fdc3c36dc6bffeb568f50e1ad4ad37ba2646803ff294ac558a8ceaa4838": "golden_dataclass_twin_ab", + "dataclass_twin_ab.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001d4968f420084c240955170ad6b0fd5c0d532262d8d0f786b43d06d8a3bd4fb70", + "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_record_id::56b62fdc3c36dc6bffeb568f50e1ad4ad37ba2646803ff294ac558a8ceaa4838": "7307d67233695c9ab4952175e9735bc9", + "dataclass_twin_ab.row[{\"k\":\"r2\"}]._tag_source_id::56b62fdc3c36dc6bffeb568f50e1ad4ad37ba2646803ff294ac558a8ceaa4838": "golden_dataclass_twin_ab", + "dataclass_twin_ab.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f4e92b93fbbdbc75fc39a77efa1164f69fa67d3495296df3e9335bf30c7adf2a", "dataclass_twin_ab.system_tag_columns": [ - "_tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc", - "_tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc" + "_tag_source_id::56b62fdc3c36dc6bffeb568f50e1ad4ad37ba2646803ff294ac558a8ceaa4838", + "_tag_record_id::56b62fdc3c36dc6bffeb568f50e1ad4ad37ba2646803ff294ac558a8ceaa4838" ], - "dataclass_twin_ba.content_hash": "semantic_v0.1:124abac0b10fc20bec6865c930effbc767f96a2ae8387c4974b3bddedb72caa3", - "dataclass_twin_ba.pipeline_hash": "semantic_v0.1:9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc", - "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "65d4fb8402a75d059b3fbfa686a22420", - "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "golden_dataclass_twin_ba", - "dataclass_twin_ba.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000125e527402024a34d9d53489982ae2634a022bc6f9a0b8249606df28a66aa118c", - "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "3d1c61df9063598ca8178d55c4a47ff9", - "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc": "golden_dataclass_twin_ba", - "dataclass_twin_ba.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f0b425f53eb41258940eaf8c37d0c18b1f3be4ecbde6969cba18ee8929013a63", + "dataclass_twin_ba.content_hash": "semantic_v0.1:428428413512e863f13e953e63b1eece43392c7759aad41418ab56148f6f7835", + "dataclass_twin_ba.pipeline_hash": "semantic_v0.1:8cf5ef3d4175a1c3c3c9fbde49c06b9cd9aad450180438821314f57a72a4818c", + "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_record_id::8cf5ef3d4175a1c3c3c9fbde49c06b9cd9aad450180438821314f57a72a4818c": "65d4fb8402a75d059b3fbfa686a22420", + "dataclass_twin_ba.row[{\"k\":\"r1\"}]._tag_source_id::8cf5ef3d4175a1c3c3c9fbde49c06b9cd9aad450180438821314f57a72a4818c": "golden_dataclass_twin_ba", + "dataclass_twin_ba.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000152a2008b0e44868a177002f8503dbf6d6fa0cd2b91b66b441fadb1c43ced3f60", + "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_record_id::8cf5ef3d4175a1c3c3c9fbde49c06b9cd9aad450180438821314f57a72a4818c": "3d1c61df9063598ca8178d55c4a47ff9", + "dataclass_twin_ba.row[{\"k\":\"r2\"}]._tag_source_id::8cf5ef3d4175a1c3c3c9fbde49c06b9cd9aad450180438821314f57a72a4818c": "golden_dataclass_twin_ba", + "dataclass_twin_ba.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001f15c52cf727aa46859cb5319a6d76689249e4d47ae3d6ffd09845d025ae27d3e", "dataclass_twin_ba.system_tag_columns": [ - "_tag_source_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc", - "_tag_record_id::9e3acf5de3bee62961bbc669bde6a0513bb0552d09a8ce6c96efb030fe6d9acc" + "_tag_source_id::8cf5ef3d4175a1c3c3c9fbde49c06b9cd9aad450180438821314f57a72a4818c", + "_tag_record_id::8cf5ef3d4175a1c3c3c9fbde49c06b9cd9aad450180438821314f57a72a4818c" + ], + "dataclass_with_array.content_hash": "semantic_v0.1:1141f67561a7b5baa13bc2f2c4d49d8be3b0e1eaa0acc2edbde54d88ff1953f6", + "dataclass_with_array.pipeline_hash": "semantic_v0.1:b322744780334c65d8f3146f09d6a7716c7876f795bdf09b373043d9326bb86c", + "dataclass_with_array.row[{\"k\":\"r1\"}]._tag_record_id::b322744780334c65d8f3146f09d6a7716c7876f795bdf09b373043d9326bb86c": "a0149c54c0765ef5b9b23ba4b3578961", + "dataclass_with_array.row[{\"k\":\"r1\"}]._tag_source_id::b322744780334c65d8f3146f09d6a7716c7876f795bdf09b373043d9326bb86c": "golden_dataclass_with_array", + "dataclass_with_array.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001ce5bead2854a51d3017727f2d93743c3eb9691221c884792b5b4cbcf3b1a2479", + "dataclass_with_array.row[{\"k\":\"r2\"}]._tag_record_id::b322744780334c65d8f3146f09d6a7716c7876f795bdf09b373043d9326bb86c": "e1e4954e269a5c2ca6a25e15d1be8ac2", + "dataclass_with_array.row[{\"k\":\"r2\"}]._tag_source_id::b322744780334c65d8f3146f09d6a7716c7876f795bdf09b373043d9326bb86c": "golden_dataclass_with_array", + "dataclass_with_array.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001868abb5fd95100a544e21bba232070629d9a69c0d34073e9ebbf3c212ab0e953", + "dataclass_with_array.system_tag_columns": [ + "_tag_source_id::b322744780334c65d8f3146f09d6a7716c7876f795bdf09b373043d9326bb86c", + "_tag_record_id::b322744780334c65d8f3146f09d6a7716c7876f795bdf09b373043d9326bb86c" ], "dataframe.content_hash": "semantic_v0.1:f18422d163082bd23e621e586c3d484ab3a25b6c1ad959b08daa4d63ef461e68", "dataframe.pipeline_hash": "semantic_v0.1:38f74931045e006f752450c2d26dbf9096c5bff65fbe952591dec83b04296a14", @@ -221,29 +233,29 @@ "_tag_source_id::bf2528f33e2c72825ecca9151c2302d77df0e6e7544e955e56101468f9b2bd09", "_tag_record_id::bf2528f33e2c72825ecca9151c2302d77df0e6e7544e955e56101468f9b2bd09" ], - "pydantic.content_hash": "semantic_v0.1:69cacd7109204aa8adf23d7e4b52c392d08b1ea58410c6b3edd2443627eb698b", - "pydantic.pipeline_hash": "semantic_v0.1:0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb", - "pydantic.row[{\"k\":\"r1\"}]._tag_record_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb": "cb9bbf1a138d5c848e86f99bab7c0acb", - "pydantic.row[{\"k\":\"r1\"}]._tag_source_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb": "golden_pydantic", - "pydantic.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:00000141edb65540d977e17ea78d6ac46147f1a4e27de391d8e3ae733375ec0f68b0cc", - "pydantic.row[{\"k\":\"r2\"}]._tag_record_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb": "10c1dbaba95a5c5fa7c14661231224eb", - "pydantic.row[{\"k\":\"r2\"}]._tag_source_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb": "golden_pydantic", - "pydantic.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001aa87be5aef99410d9d42fe7a2feaa5c7fbe85086f5f22d99a3989183b5c691d9", + "pydantic.content_hash": "semantic_v0.1:a562e8602a08e839af035082d0f3f3a9603e0d2ae6bd93898af25769b72ae998", + "pydantic.pipeline_hash": "semantic_v0.1:e5b0028b5796f344c271e471fc8bc340a0080cb4dd20bea2fb920a468c34d3f3", + "pydantic.row[{\"k\":\"r1\"}]._tag_record_id::e5b0028b5796f344c271e471fc8bc340a0080cb4dd20bea2fb920a468c34d3f3": "cb9bbf1a138d5c848e86f99bab7c0acb", + "pydantic.row[{\"k\":\"r1\"}]._tag_source_id::e5b0028b5796f344c271e471fc8bc340a0080cb4dd20bea2fb920a468c34d3f3": "golden_pydantic", + "pydantic.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001aa5111bc17c3ababc2c4e20ef9df8b562361b92323868fefe71ad4b85f28530d", + "pydantic.row[{\"k\":\"r2\"}]._tag_record_id::e5b0028b5796f344c271e471fc8bc340a0080cb4dd20bea2fb920a468c34d3f3": "10c1dbaba95a5c5fa7c14661231224eb", + "pydantic.row[{\"k\":\"r2\"}]._tag_source_id::e5b0028b5796f344c271e471fc8bc340a0080cb4dd20bea2fb920a468c34d3f3": "golden_pydantic", + "pydantic.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:0000017b71e9443562b42e1ba151e0a7293935cb86d27f021beba44ffbbb420d2d5fa0", "pydantic.system_tag_columns": [ - "_tag_source_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb", - "_tag_record_id::0b31d2b41a234110192f6ec86c12f91796c8993e1b0cdc92670027ae511fc2cb" + "_tag_source_id::e5b0028b5796f344c271e471fc8bc340a0080cb4dd20bea2fb920a468c34d3f3", + "_tag_record_id::e5b0028b5796f344c271e471fc8bc340a0080cb4dd20bea2fb920a468c34d3f3" ], - "pydantic_declared.content_hash": "semantic_v0.1:7fd221eb0acc8f9728479fd94e3cfde8f46d18909139d0fb37fdf6925736e084", - "pydantic_declared.pipeline_hash": "semantic_v0.1:fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007", - "pydantic_declared.row[{\"k\":\"r1\"}]._tag_record_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007": "fec6f63bdc695be9aeddfdd100112d42", - "pydantic_declared.row[{\"k\":\"r1\"}]._tag_source_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007": "golden_pydantic_declared", - "pydantic_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001215e94d60e5fa74938aaab850d2ae94c7b8fd2e2f685af5cb476d0c85a035fef", - "pydantic_declared.row[{\"k\":\"r2\"}]._tag_record_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007": "89983c8f835d51c3aaf2bbca0d7f3d55", - "pydantic_declared.row[{\"k\":\"r2\"}]._tag_source_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007": "golden_pydantic_declared", - "pydantic_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001b4ff9032628c25bdce2bb7eb4408d46f20bbb8b69add7f19f2599c682976cf02", + "pydantic_declared.content_hash": "semantic_v0.1:22df490b8e6b6791fb9fd906eeacd63905fac738b7cfb4be3271a3239c4df138", + "pydantic_declared.pipeline_hash": "semantic_v0.1:82c461acecff6180eb5da6cc09350c2b1ed99f3bae2403cdd9260aa572c815df", + "pydantic_declared.row[{\"k\":\"r1\"}]._tag_record_id::82c461acecff6180eb5da6cc09350c2b1ed99f3bae2403cdd9260aa572c815df": "fec6f63bdc695be9aeddfdd100112d42", + "pydantic_declared.row[{\"k\":\"r1\"}]._tag_source_id::82c461acecff6180eb5da6cc09350c2b1ed99f3bae2403cdd9260aa572c815df": "golden_pydantic_declared", + "pydantic_declared.row[{\"k\":\"r1\"}].data_hash": "arrow_v0.1:000001be6ede08b634adfe1d71f04160080ffe3c0f68a7334d6a063d139979c0c1ad4e", + "pydantic_declared.row[{\"k\":\"r2\"}]._tag_record_id::82c461acecff6180eb5da6cc09350c2b1ed99f3bae2403cdd9260aa572c815df": "89983c8f835d51c3aaf2bbca0d7f3d55", + "pydantic_declared.row[{\"k\":\"r2\"}]._tag_source_id::82c461acecff6180eb5da6cc09350c2b1ed99f3bae2403cdd9260aa572c815df": "golden_pydantic_declared", + "pydantic_declared.row[{\"k\":\"r2\"}].data_hash": "arrow_v0.1:000001dd7d2fc15f1710889d084dd1c32212031400da29c6aa3cc51e2572257daf4e07", "pydantic_declared.system_tag_columns": [ - "_tag_source_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007", - "_tag_record_id::fcc47a35b179eaca586c8b0c9877454ab9a9bbe7380c73e94531a9d4f4d90007" + "_tag_source_id::82c461acecff6180eb5da6cc09350c2b1ed99f3bae2403cdd9260aa572c815df", + "_tag_record_id::82c461acecff6180eb5da6cc09350c2b1ed99f3bae2403cdd9260aa572c815df" ], "scalars.content_hash": "semantic_v0.1:6b123e343edb881e71967e118f1e90b2dc46d20fb60e43b9b7991e3ef822a1d8", "scalars.pipeline_hash": "semantic_v0.1:1fdf145196d8e5dd7b38887cc8798325147df1f5c6f2af80952488a76f9aec07", diff --git a/tests/test_hashing/golden/data/schema.json b/tests/test_hashing/golden/data/schema.json index 5ac20cafe..9669a3cd7 100644 --- a/tests/test_hashing/golden/data/schema.json +++ b/tests/test_hashing/golden/data/schema.json @@ -2,7 +2,7 @@ "compute_schema_hash.chars_16": "a3e81ce4fbd701a0", "compute_schema_hash.full": "a3e81ce4fbd701a0fb97cae134f236953f7f66a8e7b7f4741339b45fe9f485ba", "multi_field.empty": "semantic_v0.1:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - "multi_field.logical_mix": "semantic_v0.1:ab36d5d6db29f829c3aa875245fca70d17d25054bd44088fe18d900ae95d8e19", + "multi_field.logical_mix": "semantic_v0.1:7e60623e8883af424bd2b77ebdc685422b1fce9b830847e8a5240f7063b9dc50", "multi_field.nullable_type": "semantic_v0.1:560844c33bb686eddb98de8d5b608f38d845ee5bf64e6a094f9a8326679aafe9", "multi_field.optional_field": "semantic_v0.1:bc030299c9a955ea2f18a2c746121180b5ed0d1fd6265c95ef243088e392b843", "multi_field.two_scalars": "semantic_v0.1:bc030299c9a955ea2f18a2c746121180b5ed0d1fd6265c95ef243088e392b843", @@ -11,16 +11,18 @@ "single_field.bool.schema_hash": "semantic_v0.1:d0845939265975cb588b4821498b4844a6ae18d9613e6bb9bd8c274db521eab0", "single_field.bytes.arrow_type": "large_binary", "single_field.bytes.schema_hash": "semantic_v0.1:c691ab0e3a0ce67c3f8d7b9065d60e2a14afcdce983b362c62de85c9dcf19a52", - "single_field.dataclass.arrow_type": "extension[struct]", - "single_field.dataclass.schema_hash": "semantic_v0.1:85e2c0a314e1cf2427c86f632cae5db5f8f04b681160b2b0408edec4c670d081", - "single_field.dataclass_declared.arrow_type": "extension[struct]", - "single_field.dataclass_declared.schema_hash": "semantic_v0.1:7c2cd6ddbdece7ff6754374e3a1a3d414e35c46ebcb0abf1600a06f4e9becec9", - "single_field.dataclass_nested.arrow_type": "extension[struct, tags: large_list>]", - "single_field.dataclass_nested.schema_hash": "semantic_v0.1:60298722254093098ecebefb67f409caf5abb8478e361c34c3e3223a4d2d8fc3", - "single_field.dataclass_twin_ab.arrow_type": "extension[struct]", - "single_field.dataclass_twin_ab.schema_hash": "semantic_v0.1:3258fb49892b288c4f9155d6405bda39c5d9e5d0c366f20bfe718beb2ff314ee", - "single_field.dataclass_twin_ba.arrow_type": "extension[struct]", - "single_field.dataclass_twin_ba.schema_hash": "semantic_v0.1:3258fb49892b288c4f9155d6405bda39c5d9e5d0c366f20bfe718beb2ff314ee", + "single_field.dataclass.arrow_type": "extension[struct]", + "single_field.dataclass.schema_hash": "semantic_v0.1:e6aac61112a1ce4881eddb9ce06daf61cf365e3cedfc807e72ee806a22bc7746", + "single_field.dataclass_declared.arrow_type": "extension[struct]", + "single_field.dataclass_declared.schema_hash": "semantic_v0.1:9395538bf4de55a01310739d8f70bcb6f69c06ecd364c1c13be3c9aa55baac50", + "single_field.dataclass_nested.arrow_type": "extension[struct, tags: large_list>]", + "single_field.dataclass_nested.schema_hash": "semantic_v0.1:69194f86121318f9b45e0e60566f8acb12b6ca358622ced0b9ba3be2979c4b01", + "single_field.dataclass_twin_ab.arrow_type": "extension[struct]", + "single_field.dataclass_twin_ab.schema_hash": "semantic_v0.1:0c3b32cf3937edf38f7ecc7a81da6468913438ba76f6c7670d384f20fa26e2a2", + "single_field.dataclass_twin_ba.arrow_type": "extension[struct]", + "single_field.dataclass_twin_ba.schema_hash": "semantic_v0.1:90f647076f0ed60a06ceccd4cd9b4a403b0b5d3703d6c4cab4a16f3a9d440e36", + "single_field.dataclass_with_array.arrow_type": "extension[struct, data: large_binary>>]", + "single_field.dataclass_with_array.schema_hash": "semantic_v0.1:c8da1d88d5e0eb285113a9a24100a46ebbbe5289cd92e127d4641b0c25e9df6d", "single_field.dataframe.arrow_type": "extension[large_binary]", "single_field.dataframe.schema_hash": "semantic_v0.1:2544430b3979b1e0ad8cecb9ab45c93b91dda546d5a02a59b5587c73aa04ecd8", "single_field.dict_str_int.arrow_type": "large_list>", @@ -29,8 +31,8 @@ "single_field.float.schema_hash": "semantic_v0.1:1d165501d30136c6e1309c58db930165e0cb51aae8469323283a95297a880191", "single_field.int.arrow_type": "int64", "single_field.int.schema_hash": "semantic_v0.1:2faa1297b96c1248edfafb2422fded3963672852acbe2973155351859eab83d2", - "single_field.list_dataclass.arrow_type": "extension[large_list>]", - "single_field.list_dataclass.schema_hash": "semantic_v0.1:29d0eaf9ac1816d31044a9cacc2115d49848602457a8ca84c556b2203638767d", + "single_field.list_dataclass.arrow_type": "extension[large_list>]", + "single_field.list_dataclass.schema_hash": "semantic_v0.1:bb56fb15a95f2d69dbad9f770998afb779e59e3fde54ce247de40f2b9fc3fb92", "single_field.list_int.arrow_type": "large_list", "single_field.list_int.schema_hash": "semantic_v0.1:906c18997d1191d374db82ea5cc763d3d04dc3ceccb4f3fde82cf6d53d0f8dc7", "single_field.list_list_int.arrow_type": "large_list>", @@ -43,10 +45,10 @@ "single_field.optional_int.schema_hash": "semantic_v0.1:16d9f93d8da8f8c5a582352f7022b1191ab1e4593722bbd37fdc1238d34cdc13", "single_field.path.arrow_type": "extension[large_string]", "single_field.path.schema_hash": "semantic_v0.1:84e095c4212af644cc68357daa75a3b9094b953c7fb9030fb4af86f8310b30c3", - "single_field.pydantic.arrow_type": "extension[struct]", - "single_field.pydantic.schema_hash": "semantic_v0.1:05c0f155ab1bc6cffbcc73d225a9b7413cebca8521e5f4f5a821fcdf60b3e07a", - "single_field.pydantic_declared.arrow_type": "extension[struct]", - "single_field.pydantic_declared.schema_hash": "semantic_v0.1:ab6ecbf3038e3d789a57e5032fad7432e8accedde4a5c2c0a52cecab7ecf53bb", + "single_field.pydantic.arrow_type": "extension[struct]", + "single_field.pydantic.schema_hash": "semantic_v0.1:0ff4543bbbe9edf6894ce3da5bd2ca4256dcf963c88903a2fb151d9cf7a0e218", + "single_field.pydantic_declared.arrow_type": "extension[struct]", + "single_field.pydantic_declared.schema_hash": "semantic_v0.1:3eb3e65895d2abf3880d949d368e5644b61dc3dff9e1a5c4d9b55f336e64c52b", "single_field.set_uuid.arrow_type": "extension[large_list]", "single_field.set_uuid.schema_hash": "semantic_v0.1:73501f29f5a260ab092f67ea9d88c8c3d5d8d5c90bba8d9aea88b3b5400c6272", "single_field.str.arrow_type": "large_string", diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/semantic.json index d73d5af04..27851374f 100644 --- a/tests/test_hashing/golden/data/semantic.json +++ b/tests/test_hashing/golden/data/semantic.json @@ -11,24 +11,25 @@ "polars_expr.str_contains": "polars_expr_v1:4c4cc5f6fe5bed1fbff4f59706145e216bb2597c2710c557d5ab0c39de476feb", "type.bool": "semantic_v0.1:cd8f971312b872f0e9a3ef40aa58d6bb537258e72a9a142e0cf712f0a35020e3", "type.bytes": "semantic_v0.1:f65cea8d1f29ca5b96d67caf1ced2f630b509e32574a7b78b9071958818a1c59", - "type.dataclass": "semantic_v0.1:a3986cabd66cbb48db7e99acc9b24ac00b7683707f8b3276e90b6b4f5a08405e", - "type.dataclass_declared": "semantic_v0.1:1358372ae18c5582131bd9db1da741d3fa41630e6d49095d1a1c8c5856b1feae", - "type.dataclass_nested": "semantic_v0.1:83aca3a4d53a66d3fb444f38b90b2296f3b516dd889c756a2550beb21c4d8cb4", - "type.dataclass_twin_ab": "semantic_v0.1:31e10e6217c02aadbec8d61e73caf4b27453757745b1a81ce282ad22b5bda539", - "type.dataclass_twin_ba": "semantic_v0.1:31e10e6217c02aadbec8d61e73caf4b27453757745b1a81ce282ad22b5bda539", + "type.dataclass": "semantic_v0.1:4e727cf9e14922437a44da6b0c2336556f778a3dc3f4803ce107578aececd4be", + "type.dataclass_declared": "semantic_v0.1:a8614dae386f71c572cbbe2972c3e6bdfb3dfa7d5d079c9aa5cb3f12adc6b2bb", + "type.dataclass_nested": "semantic_v0.1:c01cbb4ad9425b1ec15b07fb92838546746ed6336533273fe61b0af4ff2f9ddd", + "type.dataclass_twin_ab": "semantic_v0.1:bf02ad8fc3b36181d615af7c8128484d6cff41ded7b02bbcd3f6fc3f8a876255", + "type.dataclass_twin_ba": "semantic_v0.1:6627ee2617e7a5b5eafa83613b75438b3e00b6550bda15f03265cf17c092586b", + "type.dataclass_with_array": "semantic_v0.1:686b184d05127b487a1edfa14d329275a6e55f5c660586013870ebebb777fb5f", "type.dataframe": "semantic_v0.1:51e3c34453121e01086e882959cd12746b3236ff7b259a40a42d43493bf26dff", "type.dict_str_int": "semantic_v0.1:f0d36b085fa65f3368d1d98687ffcb6c80099a418419e45a77d434b37e4393dd", "type.float": "semantic_v0.1:0059c08fc7dfe19dc8d095b7e95d56e8d377bbe49ac30376a00105eca2e7c75f", "type.int": "semantic_v0.1:24be397c58c33686e96b4f85a9f443fd2483564ec7dee1c14b96da3e60ec16ac", - "type.list_dataclass": "semantic_v0.1:a223a6db21a8c2edd5e052457b483251e104e92fd9d8523ab12850a9cffc9f11", + "type.list_dataclass": "semantic_v0.1:53cfa02be7f0951818912b20f0d46796eb78c0210b9de887b17107faf831ed5c", "type.list_int": "semantic_v0.1:5c376e29b0cfebd85277c71c07c9996afaf06fab35f81681b82f705c9feafa3e", "type.list_list_int": "semantic_v0.1:ba62756844df6c82b48a739fadd6bb568a3507896dc5dba192dd3e0285635c48", "type.list_path": "semantic_v0.1:e2029c1032fad111f798363e1f718ecde715573c690c3cc78f33824a246a82ce", "type.ndarray": "semantic_v0.1:5d1d023b62e109ec4c9dd9dbd017205f2dddbbbda4fb3e8d81f0c2a155747e99", "type.optional_int": "semantic_v0.1:91bc2bd8f4f083d6cbf6157350e693dd52b0bc06c57b7719db8581712098cd73", "type.path": "semantic_v0.1:02a12969616ecd278958ffca715be2865fe4a32c27523059df37245c5048a880", - "type.pydantic": "semantic_v0.1:0a50adf8aebe578185ea609609b24c84aebe09ad403f0d1241e2604af85bdbc8", - "type.pydantic_declared": "semantic_v0.1:7e15bd38029dd3d24767d4a5a53443fe624f39292c0eddaf98477c1a23be5511", + "type.pydantic": "semantic_v0.1:b5cd6dbe1f7eb1593f224674c8fae5282957181e522ce2c12481eca9100c1d98", + "type.pydantic_declared": "semantic_v0.1:9f01c4a056e3a8316747ccd7d3614d391f7f4efd562f8c7c0b66aaed1d3d570d", "type.set_uuid": "semantic_v0.1:0ac1b6672ab001f6ee64f406e6d1af3a21024d6f78ef0c7414014cdfad96eaf6", "type.str": "semantic_v0.1:309d607de6633b959ae5118171f079b2830f4b1340064fac065652e305b038ad", "type.tuple_int_str": "semantic_v0.1:65ac71aa155fbfa28072adba04ed55d8d99d89e9c87a0d5ed45ccd15fcafc099", From ac48ce21188ccbac7361bd3c5ed6e3d2d5d41363 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 18:42:36 -0800 Subject: [PATCH 21/31] fix(types): testing a read candidate no longer registers its nested classes (ITL-881) A rejected candidate left its nested classes built under their default names, so declaring them under the stored names afterwards raised until the process restarted. Candidates are now described by probe_record, which registers nothing. Also assert that Delta compaction rewrote the result table in the compaction test. Co-Authored-By: Claude Fable 5.2 --- .../dataclass_logical_type_factory.py | 5 - .../pydantic_logical_type_factory.py | 4 - .../logical_types/user_class_identity.py | 111 +++++++++++++++++- .../logical_types/user_class_logical_type.py | 34 ++---- tests/test_user_types/test_read_path.py | 14 ++- .../test_workshop_scenarios.py | 85 ++++++++++++++ 6 files changed, 216 insertions(+), 37 deletions(-) diff --git a/src/orcapod/logical_types/dataclass_logical_type_factory.py b/src/orcapod/logical_types/dataclass_logical_type_factory.py index 0f906f9df..e98a2cfb4 100644 --- a/src/orcapod/logical_types/dataclass_logical_type_factory.py +++ b/src/orcapod/logical_types/dataclass_logical_type_factory.py @@ -9,7 +9,6 @@ from __future__ import annotations -import dataclasses import orcapod.logical_types.user_class_identity as uci from orcapod.logical_types.user_class_logical_type import ( @@ -54,7 +53,3 @@ class DataclassLogicalTypeFactory(UserClassLogicalTypeFactory): kind = uci.KIND_DATACLASS category = DATACLASS_CATEGORY logical_type_class = DataclassLogicalType - - def stored_field_names(self, python_type: type) -> list[str]: - """Return the names of the ``init=True`` dataclass fields, in class order.""" - return [f.name for f in dataclasses.fields(python_type) if f.init] diff --git a/src/orcapod/logical_types/pydantic_logical_type_factory.py b/src/orcapod/logical_types/pydantic_logical_type_factory.py index baa66fc93..dbd3bb43f 100644 --- a/src/orcapod/logical_types/pydantic_logical_type_factory.py +++ b/src/orcapod/logical_types/pydantic_logical_type_factory.py @@ -49,7 +49,3 @@ class PydanticLogicalTypeFactory(UserClassLogicalTypeFactory): kind = uci.KIND_PYDANTIC category = PYDANTIC_CATEGORY logical_type_class = PydanticLogicalType - - def stored_field_names(self, python_type: type) -> list[str]: - """Return the names in ``model_fields``, in class order.""" - return list(python_type.model_fields) # pyright: ignore[reportAttributeAccessIssue] # supports_class guarantees a BaseModel subclass diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index e987e6c43..d606d3948 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -33,6 +33,7 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Any, Literal +from orcapod.errors import RecursiveTypeError from orcapod.utils.canonical_type_names import canonical_type_name if TYPE_CHECKING: @@ -94,6 +95,45 @@ def class_kind(cls: Any) -> str | None: return None +def stored_field_names(cls: type) -> list[str]: + """Return the names of the fields stored for a user class, in declared order. + + Args: + cls: A dataclass or pydantic model class. + + Returns: + The ``init=True`` fields of a dataclass, or the ``model_fields`` of a pydantic + model. Computed fields and private attributes are not stored. + + Raises: + TypeError: If ``cls`` is not a dataclass or pydantic model class. + """ + kind = class_kind(cls) + if kind == KIND_DATACLASS: + return [f.name for f in dataclasses.fields(cls) if f.init] + if kind == KIND_PYDANTIC: + return list(cls.model_fields) # pyright: ignore[reportAttributeAccessIssue] # class_kind guarantees a BaseModel subclass + raise TypeError(f"{cls!r} is not a dataclass or pydantic model class.") + + +def type_hints(cls: type) -> dict[str, Any]: + """Return the resolved type hints of a class. + + Args: + cls: A class. + + Returns: + Field name to annotation, with ``Annotated`` extras stripped. + + Raises: + ValueError: If the hints cannot be resolved. + """ + try: + return typing.get_type_hints(cls) + except Exception as exc: + raise ValueError(f"Cannot get type hints for {cls!r}: {exc}") from exc + + def validate_name(name: str) -> None: """Check that ``name`` can be used as the name of a user class. @@ -444,6 +484,45 @@ def describe_record(record: dict[str, Any]) -> str: # --------------------------------------------------------------------------- +def probe_record( + cls: type, converter: TypeConverterProtocol, _path: tuple[type, ...] = () +) -> dict[str, Any]: + """Return the identity record ``cls`` would be built with, without building it. + + Neither ``cls`` nor any user class it refers to is registered or given a name, so + a class tested as a read candidate and rejected can still be declared under a name + afterwards. A user class already built in this process keeps its built identity. + + Args: + cls: A dataclass or pydantic model class. + converter: The type converter that resolves the logical types of field types + that are not user classes. + + Returns: + The identity record. + + Raises: + TypeError: If ``cls`` is not a user class or a field type has no type token. + RecursiveTypeError: If the class refers back to itself. + ValueError: If the class is local or its type hints cannot be resolved. + """ + kind = class_kind(cls) + if kind is None: + raise TypeError(f"{cls!r} is not a dataclass or pydantic model class.") + if cls in _path: + raise RecursiveTypeError( + " -> ".join(c.__qualname__ for c in (*_path[_path.index(cls) :], cls)) + ) + reject_local_class(cls) + hints = type_hints(cls) + path = (*_path, cls) + fields = [ + (name, _type_token(hints.get(name, Any), converter, path)) + for name in stored_field_names(cls) + ] + return identity_record(kind, _built.get(cls) or resolve_name(cls), fields) + + def type_token(annotation: Any, converter: TypeConverterProtocol) -> str: """Return the type token of a field annotation. @@ -459,6 +538,29 @@ def type_token(annotation: Any, converter: TypeConverterProtocol) -> str: Returns: The type token. + Raises: + TypeError: If the annotation is not a type the token rules cover. + """ + return _type_token(annotation, converter, None) + + +def _type_token( + annotation: Any, + converter: TypeConverterProtocol, + probing: tuple[type, ...] | None, +) -> str: + """Return a type token; with ``probing``, without registering any user class. + + Args: + annotation: A field annotation. + converter: The type converter that resolves logical types. + probing: ``None`` to register the classes found, as ``type_token`` does. + Otherwise the user classes being probed, outermost first: a user class is + then described by ``probe_record`` instead of being registered. + + Returns: + The type token. + Raises: TypeError: If the annotation is not a type the token rules cover. """ @@ -469,15 +571,20 @@ def type_token(annotation: Any, converter: TypeConverterProtocol) -> str: origin = typing.get_origin(annotation) args = typing.get_args(annotation) if origin is typing.Union or origin is _types.UnionType: - members = [type_token(a, converter) for a in args] + members = [_type_token(a, converter, probing) for a in args] non_none = sorted(m for m in members if m != "None") return " | ".join(non_none + (["None"] if "None" in members else [])) if origin is Literal: return f"Literal[{', '.join(sorted(repr(a) for a in args))}]" if origin is not None: - inner = ", ".join(type_token(a, converter) for a in args) + inner = ", ".join(_type_token(a, converter, probing) for a in args) return f"{_class_token(origin)}[{inner}]" if isinstance(annotation, type): + if probing is not None and class_kind(annotation) is not None: + built = _identity_of.get(annotation) + if built is not None: + return built + return identity_string(probe_record(annotation, converter, probing)) _ = converter.register_python_class(annotation) # raises for unsupported types lt = converter.get_logical_type_for_python_type(annotation) if lt is not None: diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index 0fae88df8..dffdadd84 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -19,7 +19,6 @@ from __future__ import annotations import logging -import typing from typing import TYPE_CHECKING, Any from orcapod.errors import RecursiveTypeError, TypeIdentityError @@ -182,8 +181,7 @@ def storage_to_python( class UserClassLogicalTypeFactory: """Base factory for user-class logical types. - Subclasses set ``kind``, ``category`` and ``logical_type_class`` and implement - ``supports_class`` and ``stored_field_names``. + Subclasses set ``kind``, ``category`` and ``logical_type_class``. """ kind: str = "" @@ -194,10 +192,6 @@ def supports_class(self, python_type: type) -> bool: """Return True if this factory handles ``python_type``.""" return uci.class_kind(python_type) == self.kind - def stored_field_names(self, python_type: type) -> list[str]: - """Return the names of the fields stored for ``python_type``, in class order.""" - raise NotImplementedError - # ------------------------------------------------------------------ # Write path # ------------------------------------------------------------------ @@ -240,9 +234,8 @@ def _describe( ) -> tuple[dict[str, Any], pa.StructType, list[tuple[str, Any]]]: """Return the identity record, storage struct and field annotations of a class. - Does not register ``python_type`` itself (its field types are registered, as - their Arrow types are needed), so the read path can test a candidate class - without fixing its name. + Registers the field types (their Arrow types are needed), but not + ``python_type`` itself. Args: python_type: The class. @@ -256,14 +249,14 @@ def _describe( ValueError: If the type hints cannot be resolved. RecursiveTypeError: If the class refers back to itself, naming the path. """ - hints = _type_hints(python_type) + hints = uci.type_hints(python_type) arrow_fields = [] field_annotations: list[tuple[str, Any]] = [] tokens: list[tuple[str, str]] = [] # Declared order: it is part of the identity, and the storage struct follows # it, so one identity always has one struct. - for field_name in self.stored_field_names(python_type): + for field_name in uci.stored_field_names(python_type): annotation = hints.get(field_name, Any) try: arrow_type = converter.register_python_class(annotation) @@ -356,12 +349,11 @@ def reconstruct_from_arrow( _check_storage(lt, storage_type, record) return lt continue - # Not used yet in this process: test it without registering it, so a - # non-matching candidate keeps its freedom to be declared under a name. + # Not used yet in this process: test it without registering it or the + # classes it refers to, so a rejected candidate and its nested classes + # keep their freedom to be declared under a name. try: - candidate_record, _, _ = self._describe( - cls, uci.resolve_name(cls), converter - ) + candidate_record = uci.probe_record(cls, converter) except (TypeError, ValueError) as exc: mismatches.append( f"{cls.__module__}.{cls.__qualname__} (cannot be stored: {exc})" @@ -400,14 +392,6 @@ def _describe_candidate(cls: type, record: dict[str, Any] | None) -> str: return f"{where} as {uci.describe_record(record)}" -def _type_hints(python_type: type) -> dict[str, Any]: - """Return the resolved type hints of a class, as ``ValueError`` on failure.""" - try: - return typing.get_type_hints(python_type) - except Exception as exc: - raise ValueError(f"Cannot get type hints for {python_type!r}: {exc}") from exc - - def _check_storage( lt: UserClassLogicalType, storage_type: pa.DataType, record: dict[str, Any] ) -> None: diff --git a/tests/test_user_types/test_read_path.py b/tests/test_user_types/test_read_path.py index a4d3cf91d..7d13f80ef 100644 --- a/tests/test_user_types/test_read_path.py +++ b/tests/test_user_types/test_read_path.py @@ -82,8 +82,20 @@ def test_stored_values_read_after_delta_compaction(tmp_path: Path) -> None: """Delta compaction rewrites large_string struct fields as string; still readable.""" expected = [Labelled("x1", 1), Labelled("x2", 2)] assert _run_labelled(tmp_path) == (2, expected) + rewritten: list[str] = [] for log in tmp_path.rglob("_delta_log"): - deltalake.DeltaTable(str(log.parent)).optimize.compact() + table = deltalake.DeltaTable(str(log.parent)) + before = set(table.file_uris()) + metrics = table.optimize.compact() + if "q" in [field.name for field in table.schema().fields]: + # The result table must really be rewritten, or the cached read below + # would pass on the files the first run wrote. + assert metrics["numFilesRemoved"] == len(before) >= 2 + assert metrics["numFilesAdded"] == 1 + after = set(deltalake.DeltaTable(str(log.parent)).file_uris()) + assert len(after) == 1 and not after & before + rewritten.append(str(log.parent)) + assert len(rewritten) == 1 assert _run_labelled(tmp_path) == (0, expected) diff --git a/tests/test_user_types/test_workshop_scenarios.py b/tests/test_user_types/test_workshop_scenarios.py index 3c24903e2..a86ccf998 100644 --- a/tests/test_user_types/test_workshop_scenarios.py +++ b/tests/test_user_types/test_workshop_scenarios.py @@ -565,3 +565,88 @@ def read(): assert "pkg.models.Point" in out["message"] assert "op.register_type(, name='Point')" in out["message"] assert out["second"] == ["Point(x=1, y=1)", "Point(x=2, y=2)"] + + +NESTED_WRITER = """ +from dataclasses import dataclass + +import wsjob +from child_harness import report + + +@dataclass +class Inner: + v: int + + +@dataclass +class Outer: + inner: Inner + + +report(wsjob.run_job(Outer, rows=[{"inner": Inner(1)}, {"inner": Inner(2)}])) +""" + +NESTED_READER = """ +import os +from pathlib import Path +import orcapod as op +from child_harness import error_text, report +from orcapod.contexts import get_default_context +from orcapod.databases import DeltaTableDatabase, ExtensionAwareDatabase +import wsjob +from pkg.models import Inner, Outer + +root = Path(os.environ["WS_DB"]) +path = tuple(next(p for p in wsjob.table_paths(root) if "/_result/" in p).split("/")) +converter = get_default_context().type_converter + +def read(): + db = ExtensionAwareDatabase(DeltaTableDatabase(root), converter=converter) + table = db.get_all_records(path) + return [repr(r["q"]) for r in converter.arrow_table_to_python_dicts(table.select(["q"]))] + +out = {} +try: + read() +except Exception as exc: + out["first"] = error_text(exc) +op.register_type(Inner, name="Inner") +op.register_type(Outer, name="Outer") +out["second"] = sorted(read()) +report(out) +""" + + +def test_failed_read_leaves_nested_classes_free_to_be_declared(workdir: Path) -> None: + """Testing a read candidate must not fix the names of its nested classes. + + The stored ``Outer`` holds an ``Inner`` named ``Inner``. The reader's module classes + are not declared yet, so the first read rejects ``Outer``; declaring both classes + under their stored names afterwards must work in the same process. + """ + stored = _script(workdir, NESTED_WRITER) + assert (stored["calls"], stored["rows"]) == (2, 2) + write_files( + workdir, + { + "pkg/__init__.py": "", + "pkg/models.py": """ + from dataclasses import dataclass + + @dataclass + class Inner: + v: int + + @dataclass + class Outer: + inner: Inner + """, + }, + ) + out = _script(workdir, NESTED_READER) + assert out["first"].startswith("TypeIdentityError"), out + assert out["second"] == [ + "Outer(inner=Inner(v=1))", + "Outer(inner=Inner(v=2))", + ] From 5b26f92e791dd61c05d4ed3090f8333898391c79 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 18:45:15 -0800 Subject: [PATCH 22/31] docs(types): declared field order is identity; known boundary for defaults and methods (ITL-881) Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 23 +++++++++----- DESIGN_ISSUES.md | 8 +++-- docs/concepts/identity.md | 4 +-- docs/concepts/your-own-data-types.md | 25 ++++++++++------ ...2026-11-11-itl-881-field-order-identity.md | 13 +++++++- .../2026-11-10-user-class-identity-v0.3.md | 30 +++++++++++-------- tests/test_hashing/golden/cases.py | 2 +- tests/test_user_types/test_field_order.py | 6 ++-- 8 files changed, 74 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d11d9e3a7..d8b7a9a95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,27 +73,33 @@ stay queued for the next flush. A flush that fails only for other reasons still #### Dataclasses and pydantic models are identified by name, kind and fields (ITL-881) Orcapod 0.3 identifies a dataclass or pydantic model by its name, its kind (dataclass or -pydantic) and its fields' names and types, in any order. The name is the class name for a +pydantic) and its fields' names and types, in declared order. The name is the class name for a class defined in a notebook or script (`__main__`), the module path and class name for a class defined in a module, or the name declared with `op.register_type`. - **Hashes change for every pipeline that uses such a class.** The class's logical type - name and Arrow extension name become `Name@`, and the - stored struct lists fields in name order. Every schema hash, pipeline hash, table path, + name and Arrow extension name become `Name@`; the + stored struct lists fields in declared order, as before. Every schema hash, pipeline hash, table path, system-tag column name and data hash that involves a dataclass or pydantic value moves. The first run under 0.3 computes those pod and operator results again, into new tables. Tables written by earlier versions are left untouched and are not read by 0.3 pipelines; reading such a column directly raises `TypeIdentityError`. There is no migration tool. Results that involve no dataclass or pydantic value are unaffected. -- **Changing a class's fields makes a new type.** Its pods get new tables; the results of +- **Changing a class's fields makes a new type.** Adding, removing, renaming, retyping or + reordering fields all count: a dataclass compares its fields as an ordered tuple and a + pydantic model dumps in declared order, so two classes with the same fields in another + order are different types. Defaults, methods and pydantic computed fields are not part + of the identity. The new type's pods get new tables; the results of the old definition stay where they are. Before, a changed class shared the old tables and a run returned 0 rows with only log warnings. - **A stored shape that disagrees with the registered class raises.** `DeltaTableDatabase` - re-raises the new `orcapod.errors.TypeIdentityError` instead of logging the read failure - and returning no records. Reading a column whose class cannot be found raises + raises the new `orcapod.errors.TypeIdentityError` as is. It is not a `StoreReadError`: + the table is intact and the class differs. Reading a column whose class cannot be found raises `ImportError` naming the column, the type and the `op.register_type` call to declare it. - **A subclass is its own type.** A dataclass or pydantic subclass was stored as its - registered parent, silently dropping its extra fields; it now has its own identity. + parent when the parent was registered first: its extra fields were silently dropped from + stored data, and the pod's hash and table path depended on the registration order. It + now has its own identity and keeps all its fields, in any registration order. - **Recursive classes** raise `RecursiveTypeError` (a `TypeError`) naming the field path, for example `Node.next -> Node`. - Pinned golden hash entries changed (approved): `semantic.json` `type.dataclass`, @@ -123,7 +129,8 @@ same class under a second name raises `ValueError`. See Re-running a notebook cell that defines a dataclass or pydantic model, or reloading its module, no longer breaks later pods with `ValueError: … already bound`: a class with the -same name and fields is the same type, and its stored results are found again. +same name and the same fields in the same order is the same type, and its stored results +are found again. ### Fixed diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index b0da11942..48635176b 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2460,12 +2460,16 @@ restart; a subclass was stored as its parent, dropping its extra fields. **Fix:** a user class is identified by `{kind, name, fields}` (spec `superpowers/specs/2026-11-10-user-class-identity-v0.3.md`). The identity string `Name@` is the logical type name and Arrow extension -name, with the record in the extension metadata; stored structs list fields in name order. +name, with the record in the extension metadata. `fields` lists `[name, type]` pairs in +declared order and the stored struct follows it: field order is part of the identity +(developer, November 11, 2026), so the same fields in another order are another type. `op.register_type` declares names without modifying the class; `__main__` classes default to their class name. The registry rebinds an unchanged identity to a re-defined class and matches user classes by exact class only. The read path finds the class by kind and name, checks the identity, and raises `ImportError` or `TypeIdentityError` naming the column; -`DeltaTableDatabase` re-raises `TypeIdentityError`. Resolves the entry PR #288 proposes for +`DeltaTableDatabase` raises `TypeIdentityError` as is, never wrapped in `StoreReadError`. +Testing a read candidate registers neither the class nor its nested classes +(`probe_record`), so a rejected candidate can still be declared under a name. Resolves the entry PR #288 proposes for the same issue. Not covered: computed fields (deferred by the developer), local classes, Enum/NamedTuple/TypedDict. diff --git a/docs/concepts/identity.md b/docs/concepts/identity.md index d845c146d..f3d2f806a 100644 --- a/docs/concepts/identity.md +++ b/docs/concepts/identity.md @@ -174,7 +174,7 @@ stale results are not reused. ## Your own classes in the identity A dataclass or pydantic model enters both chains through its **identity**: its name, its -kind (dataclass or pydantic) and the names and types of its fields, in any order. The +kind (dataclass or pydantic) and the names and types of its fields, in declared order. The identity is part of every schema hash, so it reaches the pipeline hash of each source, pod and node that uses the class, and with it the database table paths. Each stored value is hashed together with it, so it also reaches the content hash. @@ -183,7 +183,7 @@ Two consequences: - Two classes with the same name, kind and fields share results, wherever each is defined. A class re-defined in a notebook cell with unchanged fields keeps its cache. -- A class whose fields change is a new type: its pods get new pipeline hashes and new +- A class whose fields change or are reordered is a new type: its pods get new pipeline hashes and new tables, and the results stored with the old definition stay where they are. How a class gets its name, and how to keep results when moving a class from a notebook diff --git a/docs/concepts/your-own-data-types.md b/docs/concepts/your-own-data-types.md index a74c8ee3d..4bac41971 100644 --- a/docs/concepts/your-own-data-types.md +++ b/docs/concepts/your-own-data-types.md @@ -16,13 +16,21 @@ Orcapod identifies a dataclass or pydantic model by three things: |---|---| | **Name** | The class name for a class defined in a notebook or a script, the module path and class name for a class defined in a module, or a name you declare with `op.register_type` | | **Kind** | Dataclass or pydantic model. A dataclass and a pydantic model are never the same type, even with the same name and fields | -| **Fields** | The name and type of every stored field, in any order | +| **Fields** | The name and type of every stored field, in the order the class declares them | Two classes that agree on all three are the same type: results stored with one are found -again with the other. Anything else about the class does not count: field order, -defaults, methods, properties, pydantic computed fields and validators, docstrings. +again with the other. -Changing a field's name or type makes a **new type**. Results computed with the old +Field order counts because your code can see it: a dataclass compares, unpacks and +prints its fields in declared order, and a pydantic model dumps and serializes in +declared order. Two classes with the same fields in another order are two types. + +Known boundary: defaults, methods, properties, pydantic computed fields and validators, +and docstrings are **not** part of the identity, although a pod can observe them. +Changing one of them keeps the type and its stored results. If such a change alters what +a pod computes, bump the pod's version, as you would for any change to its code. + +Adding, removing, renaming, retyping or reordering fields makes a **new type**. Results computed with the old definition stay in the database, and the new type computes its results into new tables, so a stored value is never read back with fields it was not written with. @@ -78,14 +86,13 @@ print("function calls:", len(calls)) ``` Running the cell that defines `Point` again creates a new class object with the same -name and fields. It is the same type, so the next run finds the stored results. The -same holds when the fields are listed in another order: +name and fields. It is the same type, so the next run finds the stored results: ```python @dataclass class Point: - y: float x: float + y: float run(Point) @@ -93,7 +100,7 @@ print("function calls:", len(calls)) # function calls: 2 ``` -Adding, removing or retyping a field makes a new type. Re-run the pod cell too, so that `norm` takes +Adding, removing, retyping or reordering fields makes a new type. Re-run the pod cell too, so that `norm` takes the new `Point`; the run then computes into new tables: ```python @@ -190,7 +197,7 @@ script or notebook (`__main__`), then by importing the name as a module path. - If no class with that name exists, reading raises `ImportError` naming the column, the type and its fields, and showing the `op.register_type` call to declare it. -- If a class with that name exists but its fields differ, reading raises +- If a class with that name exists but its fields or their order differ, reading raises `TypeIdentityError` (from `orcapod.errors`) naming the column and both sets of fields. Neither case is reported as missing data, and nothing stored is changed. diff --git a/superpowers/plans/2026-11-11-itl-881-field-order-identity.md b/superpowers/plans/2026-11-11-itl-881-field-order-identity.md index bfcf5577c..f6eb76d55 100644 --- a/superpowers/plans/2026-11-11-itl-881-field-order-identity.md +++ b/superpowers/plans/2026-11-11-itl-881-field-order-identity.md @@ -79,4 +79,15 @@ Full suite on 3.11, 3.12, 3.13 one at a time; ruff; basedpyright; push; CI; Code ## Worker notes -- (fill in as work proceeds) +- Coordinator approved the plan (November 11): field order in identity, stay on `main`, + `TypeIdentityError` unwrapped (pinned in `tests/test_databases/test_delta_read_failures.py`), + defaults/methods/computed fields outside the identity (stated in the docs). +- Done and pushed: identity record with ordered fields; storage struct in declared order; + golden regenerated (vs `main`: 17 changed + 8 replaced = the 25 approved entries, rest + added; twins pin different values; ndarray-field fixture added); `probe_record` for read + candidates (nested classes stay undeclared); compaction test asserts the rewrite; docs, + spec, CHANGELOG, DESIGN_ISSUES UCI1. +- Remaining: full suite on 3.11/3.12/3.13, ruff, basedpyright, PR body, CI, CodeRabbit round. +- Out of scope, to report: the semantic hasher expands a namedtuple with fields sorted by + name (`semantic_hasher._expand_namedtuple`), so namedtuple field order does not enter its + hash, although a namedtuple compares as an ordered tuple. diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md index 663f6b524..a04664ae8 100644 --- a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -3,6 +3,12 @@ **Status:** implemented on this branch (PR #292). Approved by the coordinator November 10, 2026; P1, the identity-string format (fold, full-length hash) and the dropped naming switch decided by the developer the same day. +**Revised November 11, 2026 (developer):** declared field **order is part of the identity**, +for dataclasses and pydantic models. This reverses the November 10 ruling "not field order". +Criterion: take the type's own equality seriously (a dataclass compares its fields as an +ordered tuple; a pydantic model dumps, iterates and serializes in declared order); a finer +identity can cost a recompute but cannot be incorrect. The sections below state the revised +rule. **Issue:** ITL-881 (v0.3, a public release; tag target November 16, 2026). **Builds on:** the findings in PR #288 (`superpowers/specs/2026-11-10-user-class-identity.md`, experiments E01–E08). "W1"–"W15" are the workshop scenarios from §3.4 of that document. @@ -13,7 +19,7 @@ Each decision is marked **CONFIRMED** (the developer decided it, November 10, 20 | ID | Decision | Status | |---|---|---| | D-name | Nominal identity with a declared name; `__main__` classes default to the name only | CONFIRMED: developer decision 1 | -| D-fields | Field names and field types are part of the identity; field order is not | CONFIRMED: developer decision 2 | +| D-fields | Field names, field types and declared field order are part of the identity | CONFIRMED: developer decision 2, revised November 11, 2026 (order is identity) | | D-computed | Computed fields are not part of the identity in v0.3 | CONFIRMED as deferred: developer decision 2 defers it; v0.3 keeps today's stored fields | | D-kind | Dataclass and pydantic are distinct kinds | CONFIRMED: developer decision 3 | | D-rebind | Same identity re-registered: rebind silently | CONFIRMED: developer answer to S1 | @@ -75,12 +81,11 @@ pydantic logical types by exact class only. ### Field component — CONFIRMED (decision 2; nested classes: P1) -`fields` maps each **stored field's name** to its **type token**. It is a JSON object -serialized with sorted keys, so field order never enters. The stored fields are exactly +`fields` lists each **stored field's name** with its **type token**, as `[name, token]` +pairs in declared order. It is a JSON array, so field order enters the identity. The stored fields are exactly the ones stored today: dataclass fields with `init=True`, pydantic `model_fields`. The -Arrow storage struct also lists the fields in name order, so two classes with the same -identity always have the same storage type (found by the W11 test: with class order the -same identity had two structs, and the Delta read failed into 0 rows). +Arrow storage struct lists the fields in the same declared order, so two classes with the +same identity always have the same storage type. Type token of an annotation (from `typing.get_type_hints`, which strips `Annotated` extras): @@ -121,7 +126,7 @@ recursive struct, so there is nothing to identify. Deliberately **not** part of the identity: defaults and default factories, methods and validators, properties and pydantic `computed_field`s, `init=False` dataclass fields, -`ClassVar`s, private attributes, field order, docstrings, pydantic field aliases and +`ClassVar`s, private attributes, docstrings, pydantic field aliases and constraints (`Field(gt=0)`, `Annotated` metadata), `frozen`/`slots`/config flags, base classes (only the resulting fields count). Computed fields are deferred (D-computed); a later decision to include them adds a key that is absent when empty, so it would move @@ -148,8 +153,9 @@ canonical_json = json.dumps(record, sort_keys=True, separators=(",", ":"), ensu ``` `Point(x: int, y: float)` as a dataclass gives -`Point@9117beaaf7ebe3ab0bd25de9448c92eb64aa070a58611517bd1e470a544584d8`; with `y: int`, -`Point@c0a75f0c…`; as a pydantic model with `y: float`, `Point@703bff8a…`. SHA-256 of a +`Point@76d961e40619a836535e3fdebdead7ea65aa9413955a889f3d5b51a7a94b65eb` (the SHA-256 of +`{"fields":[["x","int"],["y","float"]],"kind":"dataclass","name":"Point"}`); with `y: int`, +`Point@008496cf…`; as a pydantic model with `y: float`, `Point@8a0f5b42…`. SHA-256 of a documented JSON string, not the Orcapod semantic hasher, so the identity can be reproduced by hand and does not depend on a hasher version. The full digest is used (developer): there is no truncation rule to explain, and the name is not meant for people to type. @@ -174,7 +180,7 @@ second mechanism. | Logical type name | identity string | `DataclassLogicalType.logical_type_name` / pydantic equivalent | | Arrow extension name (`ARROW:extension:name`) | identity string | same string; one Arrow/Polars extension registration per identity | | Arrow extension metadata | category plus the full record: `{"category":"orcapod.dataclass","identity":{…}}`, canonical JSON | read path uses it to find the class and to print the stored shape | -| Arrow storage type | struct of the stored fields, in name order | derived from the annotations | +| Arrow storage type | struct of the stored fields, in declared order | derived from the annotations | | Schema hash, source `pipeline_hash`, system-tag column names | identity string | `TypeObjectHandler` hashes a registered class as `type:` (unchanged code) | | Pod signature hash, node `pipeline_hash`, DB table paths | identity string | via the schema/signature hashes | | Per-row data hash, `content_hash` | identity string + record + values | the extension column is hashed with its extension name and metadata (unchanged code) | @@ -331,8 +337,8 @@ entries plus the unpinned additions. additions (new keys only): a declared-name dataclass and a pydantic model with the same name and fields (distinct values), a nested dataclass with a container field (P1), `list[GoldenPoint]`, an `Optional` field, and field-order twins (`GoldenTwinAB`, -`GoldenTwinBA`, both `golden.Twin`) that pin equal type, schema, data and pipeline hashes -(only their `source_id`-derived entries differ). +`GoldenTwinBA`, both `golden.Twin`) that pin **different** type, schema, data and pipeline +hashes, and a dataclass with a NumPy array field. **For the CHANGELOG (stated plainly):** "Orcapod 0.3 identifies a dataclass or pydantic model by its name, its kind and its fields. Every pipeline whose data includes such a diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 9626ae5ee..1e479de13 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -232,7 +232,7 @@ class GoldenWithArray: """Dataclass with a NumPy array field; the array's logical type enters its identity.""" label: str - values: np.ndarray + values: np.ndarray # pyright: ignore[reportMissingTypeArgument] # the bare class is the annotation Orcapod stores @dataclasses.dataclass diff --git a/tests/test_user_types/test_field_order.py b/tests/test_user_types/test_field_order.py index 76992a256..dfd5b972b 100644 --- a/tests/test_user_types/test_field_order.py +++ b/tests/test_user_types/test_field_order.py @@ -74,8 +74,10 @@ def _source(value: Any) -> DictSource: return DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"], source_id="s") -def _column_type(value: Any) -> pa.DataType: - return _source(value).as_table().schema.field("p").type +def _column_type(value: Any) -> pa.ExtensionType: + column_type = _source(value).as_table().schema.field("p").type + assert isinstance(column_type, pa.ExtensionType) + return column_type def _type_name(value: Any) -> str: From fe41fa0b9c5447c76158eea744e3ed1b78a2cdab Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 18:46:03 -0800 Subject: [PATCH 23/31] test(types): type-safe struct field access in field-order tests (ITL-881) Co-Authored-By: Claude Fable 5.2 --- tests/test_user_types/test_field_order.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_user_types/test_field_order.py b/tests/test_user_types/test_field_order.py index dfd5b972b..c69596533 100644 --- a/tests/test_user_types/test_field_order.py +++ b/tests/test_user_types/test_field_order.py @@ -120,7 +120,8 @@ def test_stored_struct_lists_fields_in_declared_order( value: Any, names: list[str] ) -> None: storage = _column_type(value).storage_type - assert [field.name for field in storage] == names + assert isinstance(storage, pa.StructType) + assert storage.names == names def test_values_of_both_orders_read_back_as_their_own_class( From 887b31232a13e625ad2b8cf4bc29a01572c4337a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 19:01:26 -0800 Subject: [PATCH 24/31] fix(pipeline): a stored row that cannot be decoded as its class fails the run (ITL-881) Under error_policy="continue" a TypeIdentityError from a stored-result read was skipped as an ordinary row failure, so the run completed without the row. The per-row handlers of FunctionNode and FunctionPod now re-raise it, and the async orchestrator raises it on its own, as the sync orchestrator does. It stays unwrapped: the table is intact, so it is not a StoreReadError. Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/function_pod.py | 7 +- src/orcapod/core/nodes/function_node.py | 7 + src/orcapod/pipeline/async_orchestrator.py | 15 +- .../test_type_identity_failure.py | 161 ++++++++++++++++++ 4 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 tests/test_pipeline/test_type_identity_failure.py diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 3bd2c8e74..5dbecf059 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -14,7 +14,7 @@ from orcapod.core.data_function import CachedDataFunction, PythonDataFunction from orcapod.core.streams.base import StreamBase from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.errors import StoreError +from orcapod.errors import StoreError, TypeIdentityError from orcapod.utils.store_errors import keep_store_error, note_callback_failure from orcapod.protocols.core_protocols import ( ArgumentGroup, @@ -606,6 +606,11 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: with keep_store_error(exc, "Observer on_data_crash"): obs.on_data_crash(pod_label, tag, data, exc) raise + except TypeIdentityError as exc: + # A stored row that cannot be decoded as its class is never + # skipped as a row failure (ITL-881). + obs.on_data_crash(pod_label, tag, data, exc) + raise except Exception as exc: logger.debug( "Data processing failed, skipping: %s", exc, exc_info=True diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 3454c8856..89ff8cafa 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -36,6 +36,7 @@ PipelineJobRequiredError, SchemaVersionError, StoreError, + TypeIdentityError, StoreReadError, StoreWriteError, ) @@ -1317,6 +1318,12 @@ def execute( with keep_store_error(exc, "Observer on_node_end"): ctx_obs.on_node_end(node_label, node_hash) raise + except TypeIdentityError as exc: + # A stored row that cannot be decoded as its class is never + # skipped as a row failure, whatever the error policy (ITL-881). + ctx_obs.on_data_crash(node_label, tag, data, exc) + ctx_obs.on_node_end(node_label, node_hash) + raise except Exception as exc: logger.warning( "Data execution failed in %s: %s", diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 87ffed444..049440dd9 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, Literal from orcapod.channels import BroadcastChannel, Channel -from orcapod.errors import StoreError, StoreWriteError +from orcapod.errors import StoreError, StoreWriteError, TypeIdentityError from orcapod.pipeline.result import OrchestratorResult from orcapod.utils.store_errors import keep_store_error from orcapod.protocols.node_protocols import ( @@ -268,9 +268,16 @@ async def _run_async( # sync orchestrator does, rather than inside the TaskGroup's exception # group. store_error = _single_store_error(group, topo_order) - if store_error is None: - raise - raise store_error + if store_error is not None: + raise store_error + # A stored row that cannot be decoded as its class stops the run too, + # raised on its own as the sync orchestrator raises it (ITL-881). + identity_errors = [ + exc for exc in _leaves(group) if isinstance(exc, TypeIdentityError) + ] + if identity_errors: + raise identity_errors[0] + raise finally: # A failing on_run_end must not replace a failed store read or write. with keep_store_error(store_error, "Observer on_run_end"): diff --git a/tests/test_pipeline/test_type_identity_failure.py b/tests/test_pipeline/test_type_identity_failure.py new file mode 100644 index 000000000..b62c4f44c --- /dev/null +++ b/tests/test_pipeline/test_type_identity_failure.py @@ -0,0 +1,161 @@ +"""A stored row that cannot be decoded as its class fails the run (ITL-881). + +``TypeIdentityError`` means a stored table is intact but holds another shape than the +registered class. It is not a ``StoreReadError``, yet it must be as fatal: under +``error_policy="continue"`` a row whose stored result cannot be decoded must never be +skipped as an ordinary row failure, or stored results would silently look dropped. Both +orchestrators must raise it. +""" + +from __future__ import annotations + +import dataclasses +import shutil +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import deltalake +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod import function_pod +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import StoreError, TypeIdentityError +from orcapod.pipeline import PipelineJob +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +_CALLS: list[int] = [] + + +@op.register_type(name="itl881.fatal.Pair") +@dataclasses.dataclass +class Pair: + """The value the pod stores.""" + + x: int + y: int + + +@function_pod(output_keys="q") +def pair(x: int) -> Pair: + """Record the call and return ``Pair(x, x)``.""" + _CALLS.append(x) + return Pair(x, x) + + +ORCHESTRATORS: dict[str, Callable[[], object]] = { + "sync": lambda: SyncPipelineOrchestrator(error_policy="continue"), + "async": lambda: AsyncPipelineOrchestrator(error_policy="continue"), +} + + +@pytest.fixture(autouse=True) +def _reset_calls(): + """Start and end every test with no recorded calls.""" + _CALLS.clear() + yield + _CALLS.clear() + + +@pytest.fixture(params=list(ORCHESTRATORS)) +def make_orchestrator(request: pytest.FixtureRequest) -> Callable[[], object]: + """Return a factory for the parametrized orchestrator (``continue`` policy).""" + return ORCHESTRATORS[request.param] + + +def build_job(store_path: Path) -> PipelineJob: + """Build a job that stores one ``Pair`` per row of a three-row source.""" + source = ArrowTableSource( + pa.table({"k": ["a", "b", "c"], "x": [1, 2, 3]}), + tag_columns=["k"], + infer_nullable=True, + ) + job = PipelineJob(name="itl881", store=DeltaTableDatabase(base_path=store_path)) + with job: + _ = pair.pod(source, label="pair") + return job + + +def table_dirs(store_path: Path) -> tuple[Path, Path]: + """The (result table, pipeline table) directories the job wrote.""" + dirs = {p.parent for p in store_path.rglob("*.parquet")} + result = [d for d in dirs if "_result" in d.parts] + pipeline = [d for d in dirs if "_result" not in d.parts] + assert len(result) == 1 and len(pipeline) == 1, sorted(dirs) + return result[0], pipeline[0] + + +def store_another_shape(result_table: Path) -> None: + """Rewrite column ``q`` with another struct under the same extension name. + + Stands in for a table written by other means: the table stays readable, but its + stored shape no longer matches the registered ``Pair``. + """ + stored = deltalake.DeltaTable(str(result_table)).to_pyarrow_table() + index = stored.schema.get_field_index("q") + ext = stored.schema.field(index).type + assert isinstance(ext, pa.ExtensionType) + wrong = pa.struct([pa.field("x", pa.float64()), pa.field("y", pa.int64())]) + values: list[Any] = [{"x": 0.5, "y": 1}] * stored.num_rows + field = pa.field("q", wrong).with_metadata( + { + b"ARROW:extension:name": ext.extension_name.encode(), + b"ARROW:extension:metadata": ext.__arrow_ext_serialize__(), + } + ) + columns = [ + pa.array(values, type=wrong) if i == index else stored.column(i) + for i in range(stored.num_columns) + ] + fields = [field if i == index else f for i, f in enumerate(stored.schema)] + deltalake.write_deltalake( + str(result_table), + pa.Table.from_arrays(columns, schema=pa.schema(fields)), + mode="overwrite", + schema_mode="overwrite", + ) + + +@pytest.fixture +def stored(tmp_path: Path, make_orchestrator: Callable[[], object]) -> Path: + """A store in which the job has computed and stored all three rows.""" + store_path = tmp_path / "store" + build_job(store_path).run(orchestrator=make_orchestrator()) + assert len(_CALLS) == 3 + _CALLS.clear() + return store_path + + +def test_other_stored_shape_fails_the_run( + stored: Path, make_orchestrator: Callable[[], object] +) -> None: + """A result table holding another shape fails the run before any function call.""" + result_table, _ = table_dirs(stored) + store_another_shape(result_table) + + with pytest.raises(TypeIdentityError) as info: + build_job(stored).run(orchestrator=make_orchestrator()) + + assert not isinstance(info.value, StoreError) + assert _CALLS == [] + + +def test_other_stored_shape_fails_the_run_in_the_per_row_lookup( + stored: Path, make_orchestrator: Callable[[], object] +) -> None: + """The per-row lookup's identity error fails the run under ``continue``.""" + # With the pipeline table gone, the first read of the result table is the per-row + # cache lookup, which runs inside the error_policy="continue" handler. + result_table, pipeline_table = table_dirs(stored) + store_another_shape(result_table) + shutil.rmtree(pipeline_table) + + with pytest.raises(TypeIdentityError) as info: + build_job(stored).run(orchestrator=make_orchestrator()) + + assert not isinstance(info.value, StoreError) + assert _CALLS == [] From 22b4b1e30350f22c7ea702ae4685420bd3861301 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 19:03:44 -0800 Subject: [PATCH 25/31] fix(types): optional nested class None reads back; stored record checked against its name (ITL-881) - A nested user class stored as None (child: Inner | None) raised TypeError on read (also on main); storage_to_python now passes None through. - reconstruct_from_arrow rejects a stored identity record that does not hash to the extension name. - Read candidates are looked up lazily: the dotted name is imported only when no registered or __main__ class matches. Co-Authored-By: Claude Fable 5.2 --- .../logical_types/user_class_identity.py | 37 ++++---- .../logical_types/user_class_logical_type.py | 34 +++++--- .../2026-11-10-user-class-identity-v0.3.md | 2 +- .../test_dataclass_logical_type_factory.py | 85 ++++++++++++++++--- .../test_pydantic_logical_type_factory.py | 16 ++-- tests/test_user_types/test_register_type.py | 41 +++++++++ 6 files changed, 169 insertions(+), 46 deletions(-) diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index d606d3948..928150e4e 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -30,7 +30,7 @@ import unicodedata import warnings import weakref -from collections.abc import Iterable +from collections.abc import Iterable, Iterator from typing import TYPE_CHECKING, Any, Literal from orcapod.errors import RecursiveTypeError @@ -366,35 +366,42 @@ def known_classes(kind: str, name: str) -> list[type]: return [c for c in reversed(classes) if c is not None] -def candidate_classes(kind: str, name: str) -> list[type]: - """Return the classes that may be the class stored under ``(kind, name)``. +def candidate_classes(kind: str, name: str) -> Iterator[type]: + """Yield the classes that may be the class stored under ``(kind, name)``. In order: classes registered or declared under that name in this process, a class reachable from ``__main__`` by that name, and a class importable by that dotted - name. Each candidate still has to match the stored identity. + name. Each candidate still has to match the stored identity. The dotted name is + imported only when the caller asks for a candidate beyond the earlier ones, so a + read that a registered class satisfies imports nothing. Args: kind: The stored kind. name: The stored name component. - Returns: + Yields: Distinct candidate classes of the right kind. """ - found: list[type] = list(known_classes(kind, name)) + seen: list[type] = [] + + def fresh(cls: Any) -> bool: + if class_kind(cls) != kind or cls in seen: + return False + seen.append(cls) + return True + + for cls in known_classes(kind, name): + if fresh(cls): + yield cls for module_name in MAIN_MODULES: module = sys.modules.get(module_name) if module is not None: obj = _walk_attributes(module, name.split(".")) - if obj is not None: - found.append(obj) + if fresh(obj): + yield obj obj = _import_dotted(name) - if obj is not None: - found.append(obj) - result: list[type] = [] - for cls in found: - if class_kind(cls) == kind and cls not in result: - result.append(cls) - return result + if fresh(obj): + yield obj def _walk_attributes(root: Any, parts: list[str]) -> Any: diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index dffdadd84..84daa189c 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -157,11 +157,13 @@ def storage_to_python( """Build an instance from an Arrow struct dict. Args: - storage_value: A dict mapping field names to Arrow storage values. + storage_value: A dict mapping field names to Arrow storage values, or + ``None`` for a null value (an optional field of this class that + holds ``None``). converter: The active converter for per-field conversion. Returns: - An instance of ``python_type``. + An instance of ``python_type``, or ``None`` for a null value. Raises: ValueError: If ``converter`` is ``None``. @@ -171,6 +173,8 @@ def storage_to_python( f"{type(self).__name__}.storage_to_python requires a converter — " "pass a TypeConverterProtocol instance for field-level conversion." ) + if storage_value is None: + return None kwargs = { name: converter.storage_to_python(storage_value[name], annotation) for name, annotation in self._field_annotations @@ -325,19 +329,19 @@ def reconstruct_from_arrow( f"record for category {self.category!r}: {record!r}." ) - kind = self.kind - candidates = uci.candidate_classes(kind, name) - if not candidates: - raise ImportError( - f"Cannot find the {kind} {name!r} to read stored values of type " - f"{uci.describe_record(record)}. Define the class and register it " - f"under that name before reading, for example " - f"op.register_type({name.rsplit('.', 1)[-1]}, name={name!r})." + if uci.identity_string(record) != arrow_extension_name: + raise TypeIdentityError( + f"Stored column type {arrow_extension_name!r} does not match its " + f"identity record {uci.describe_record(record)}." ) + kind = self.kind + found_any = False mismatches: list[str] = [] renamable = False # a candidate has the stored fields under another name - for cls in candidates: + # Lazy: the dotted name is imported only if no registered class matches. + for cls in uci.candidate_classes(kind, name): + found_any = True built = uci.identity_of(cls) if built is not None: if built != arrow_extension_name: @@ -369,6 +373,14 @@ def reconstruct_from_arrow( _check_storage(lt, storage_type, record) return lt + if not found_any: + raise ImportError( + f"Cannot find the {kind} {name!r} to read stored values of type " + f"{uci.describe_record(record)}. Define the class and register it " + f"under that name before reading, for example " + f"op.register_type({name.rsplit('.', 1)[-1]}, name={name!r})." + ) + hint = ( f" A class with the stored fields exists under another name: register it " f"as {name!r} before it is first used, e.g. " diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md index a04664ae8..b8faf1868 100644 --- a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -40,7 +40,7 @@ Each decision is marked **CONFIRMED** (the developer decided it, November 10, 20 A user class's identity is an **identity record** with three components: ```json -{"fields": {"x": "int", "y": "float"}, "kind": "dataclass", "name": "Point"} +{"fields": [["x", "int"], ["y", "float"]], "kind": "dataclass", "name": "Point"} ``` ### Name component — CONFIRMED (decision 1) diff --git a/tests/test_logical_types/test_dataclass_logical_type_factory.py b/tests/test_logical_types/test_dataclass_logical_type_factory.py index fccc8e723..b40a6a978 100644 --- a/tests/test_logical_types/test_dataclass_logical_type_factory.py +++ b/tests/test_logical_types/test_dataclass_logical_type_factory.py @@ -432,18 +432,16 @@ def test_factory_reconstruct_from_arrow_invalid_fqcn(): factory = DataclassLogicalTypeFactory() converter = _make_full_converter() + import orcapod.logical_types.user_class_identity as uci + + record = uci.identity_record( + "dataclass", "nonexistent.module.NoSuchClass", [("x", "int")] + ) with pytest.raises(ImportError): factory.reconstruct_from_arrow( - "nonexistent.module.NoSuchClass", + uci.identity_string(record), storage, - { - "category": "orcapod.dataclass", - "identity": { - "kind": "dataclass", - "name": "nonexistent.module.NoSuchClass", - "fields": [["x", "int"]], - }, - }, + {"category": "orcapod.dataclass", "identity": record}, converter, ) @@ -553,10 +551,20 @@ def test_reconstruct_finds_class_nested_in_a_class(): DataclassLogicalTypeFactory, ) + import orcapod.logical_types.user_class_identity as uci + inner = _OuterForNestedTest.Inner + # Build the stored identity by hand, so that the class is not registered and the + # read can only find it by importing its dotted name. + name = f"{inner.__module__}.{inner.__qualname__}" + record = uci.identity_record("dataclass", name, [("x", "int"), ("y", "str")]) + assert uci.known_classes("dataclass", name) == [] storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.large_string())]) lt = DataclassLogicalTypeFactory().reconstruct_from_arrow( - _identity(inner), storage, _metadata(inner), _make_full_converter() + uci.identity_string(record), + storage, + {"category": "orcapod.dataclass", "identity": record}, + _make_full_converter(), ) assert lt.python_type is inner @@ -707,3 +715,60 @@ def test_list_of_nested_dataclass_parquet_roundtrip(tmp_path): assert reconstructed.items[0].x == 1 assert reconstructed.items[1].y == 4 assert reconstructed.label == "test" + + +def test_reconstruct_rejects_a_record_that_does_not_match_the_extension_name(): + """A stored record must hash to the stored extension name (ITL-881).""" + from orcapod.errors import TypeIdentityError + from orcapod.logical_types.dataclass_logical_type_factory import ( + DataclassLogicalTypeFactory, + ) + + # This test is about the factory's read entry point: an inconsistent stored + # column cannot be produced through the public interface. + metadata = _metadata(_Flat) + record = dict(metadata["identity"]) # pyright: ignore[reportCallIssue, reportArgumentType] # the identity is a JSON object + record["fields"] = [["name", "str"], ["count", "str"]] + storage = pa.struct( + [pa.field("name", pa.large_string()), pa.field("count", pa.int64())] + ) + with pytest.raises(TypeIdentityError, match="does not match its identity record"): + DataclassLogicalTypeFactory().reconstruct_from_arrow( + _identity(_Flat), + storage, + {"category": metadata["category"], "identity": record}, + _make_full_converter(), + ) + + +@dataclasses.dataclass +class _RegisteredElsewhere: + """Declared under a dotted name whose module fails to import.""" + + x: int + + +def test_reconstruct_uses_a_registered_class_without_importing_its_name( + tmp_path, monkeypatch +): + """A registered class that matches is used before its dotted name is imported.""" + import orcapod as op + from orcapod.logical_types.dataclass_logical_type_factory import ( + DataclassLogicalTypeFactory, + ) + + # A module that would be imported for the dotted name, and fails on import. + package = tmp_path / "itl881_broken_module" + package.mkdir() + (package / "__init__.py").write_text("raise RuntimeError('import side effect')\n") + monkeypatch.syspath_prepend(str(tmp_path)) + op.register_type(_RegisteredElsewhere, name="itl881_broken_module.Point") + + # This test is about the factory's read entry point (candidate lookup order). + lt = DataclassLogicalTypeFactory().reconstruct_from_arrow( + _identity(_RegisteredElsewhere), + pa.struct([pa.field("x", pa.int64())]), + _metadata(_RegisteredElsewhere), + _make_full_converter(), + ) + assert lt.python_type is _RegisteredElsewhere diff --git a/tests/test_logical_types/test_pydantic_logical_type_factory.py b/tests/test_logical_types/test_pydantic_logical_type_factory.py index 37b36d4f4..2bad77625 100644 --- a/tests/test_logical_types/test_pydantic_logical_type_factory.py +++ b/tests/test_logical_types/test_pydantic_logical_type_factory.py @@ -460,18 +460,16 @@ def test_factory_reconstruct_from_arrow_invalid_fqcn(): factory = PydanticLogicalTypeFactory() converter = _make_full_converter() + import orcapod.logical_types.user_class_identity as uci + + record = uci.identity_record( + "pydantic", "nonexistent.module.NoSuchClass", [("x", "int")] + ) with pytest.raises(ImportError): factory.reconstruct_from_arrow( - "nonexistent.module.NoSuchModel", + uci.identity_string(record), storage, - { - "category": "orcapod.pydantic", - "identity": { - "kind": "pydantic", - "name": "nonexistent.module.NoSuchClass", - "fields": [["x", "int"]], - }, - }, + {"category": "orcapod.pydantic", "identity": record}, converter, ) diff --git a/tests/test_user_types/test_register_type.py b/tests/test_user_types/test_register_type.py index 1046ad229..899c54cdc 100644 --- a/tests/test_user_types/test_register_type.py +++ b/tests/test_user_types/test_register_type.py @@ -370,3 +370,44 @@ def test_subclass_has_its_own_identity_and_keeps_its_fields() -> None: assert _type_name(Base(1)) != _type_name(Child(1, 2)) _, data = next(iter(_source(Child(1, 2)).iter_data())) assert data.as_dict()["p"] == Child(1, 2) + + +# --------------------------------------------------------------------------- +# Optional nested classes +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class OptionalInner: + """A class nested in ``OptionalOuter`` as an optional field.""" + + v: int + + +@dataclasses.dataclass +class OptionalOuter: + """A class whose nested class may be ``None``.""" + + child: OptionalInner | None + + +class OptionalOuterModel(pydantic.BaseModel): + """A pydantic model whose nested class may be ``None``.""" + + child: OptionalInner | None + + +@pytest.mark.parametrize( + "value", + [ + OptionalOuter(OptionalInner(1)), + OptionalOuter(None), + OptionalOuterModel(child=OptionalInner(1)), + OptionalOuterModel(child=None), + ], + ids=["dataclass-set", "dataclass-none", "pydantic-set", "pydantic-none"], +) +def test_optional_nested_class_reads_back(value: Any) -> None: + """A nested class stored as ``None`` reads back as ``None``, not a ``TypeError``.""" + _, data = next(iter(_source(value).iter_data())) + assert data.as_dict()["p"] == value From 25346abb514f11e63acc8a5fe03919bee2821e9f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 19:04:39 -0800 Subject: [PATCH 26/31] test(types): distinct value per field in workshop scenarios; assert decoded values (ITL-881) Co-Authored-By: Claude Fable 5.2 --- tests/test_user_types/child_harness.py | 3 ++- tests/test_user_types/test_workshop_scenarios.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_user_types/child_harness.py b/tests/test_user_types/child_harness.py index 92bf3f786..46d813f7b 100644 --- a/tests/test_user_types/child_harness.py +++ b/tests/test_user_types/child_harness.py @@ -140,7 +140,8 @@ def run_job(point_cls, pod_and_calls=None, rows=None): pod, calls = pod_and_calls or make_pod(point_cls) if rows is None: names = list(getattr(point_cls, "model_fields", None) or point_cls.__dataclass_fields__) - rows = [{n: i for n in names} for i in (1, 2)] + # A distinct value per field, so that swapped fields would be noticed. + rows = [{n: 10 * i + j for j, n in enumerate(names)} for i in (1, 2)] data = [{"k": f"r{i}", "p": point_cls(**row)} for i, row in enumerate(rows, 1)] before = len(calls) db_root = os.environ["WS_DB"] diff --git a/tests/test_user_types/test_workshop_scenarios.py b/tests/test_user_types/test_workshop_scenarios.py index a86ccf998..af00319d4 100644 --- a/tests/test_user_types/test_workshop_scenarios.py +++ b/tests/test_user_types/test_workshop_scenarios.py @@ -214,6 +214,8 @@ def test_w1_rerun_define_cell_same_fields_rebinds(workdir: Path) -> None: assert (second["calls"], second["rows"]) == (0, 2) assert second["q_class_is_point"] assert first["pod_pipeline_hash"] == second["pod_pipeline_hash"] + assert first["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] + assert second["values"] == first["values"] def test_w1_old_pod_cell_with_redefined_class(workdir: Path) -> None: @@ -248,6 +250,9 @@ def test_reordered_fields_are_a_new_type( assert set(first["table_paths"]) < set(second["table_paths"]) assert first["stored_fields"] == [["x", "y"]] assert second["stored_fields"] == [["x", "y"], ["y", "x"]] + assert first["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] + assert second["values"] == ["Point(y=10, x=11)", "Point(y=20, x=21)"] + assert third["values"] == first["values"] # The first order's results are untouched and found again. assert (third["calls"], third["rows"]) == (0, 2) assert third["pod_pipeline_hash"] == first["pod_pipeline_hash"] @@ -265,7 +270,7 @@ def test_class_redefined_with_an_added_field_is_a_new_type(workdir: Path) -> Non assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] assert set(first["table_paths"]) < set(second["table_paths"]) assert second["stored_fields"] == [["x", "y"], ["x", "y", "z"]] - assert second["values"] == ["Point(x=1, y=1, z=1)", "Point(x=2, y=2, z=2)"] + assert second["values"] == ["Point(x=10, y=11, z=12)", "Point(x=20, y=21, z=22)"] # The two-field results are untouched and found again. assert (third["calls"], third["rows"]) == (0, 2) assert third["values"] == first["values"] @@ -426,6 +431,8 @@ class Point: assert module["node_identity_path"] == notebook["node_identity_path"] assert module["table_paths"] == notebook["table_paths"] assert module["extension_name"] == notebook["extension_name"] + assert module["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] + assert module["values"] == notebook["values"] def test_w11_module_without_declared_name_is_a_different_type(workdir: Path) -> None: @@ -470,7 +477,7 @@ def test_w12_read_notebook_results_from_script_with_same_class(workdir: Path) -> define = "from dataclasses import dataclass\n@dataclass\nclass Point:\n x: int\n y: int\n" out = _script(workdir, READER.format(define=define)) assert out.get("error") is None, out - assert out["values"] == ["Point(x=1, y=1)", "Point(x=2, y=2)"] + assert out["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] assert out["classes"] == ["__main__.Point"] @@ -564,7 +571,7 @@ def read(): assert out["first"].startswith("TypeIdentityError"), out assert "pkg.models.Point" in out["message"] assert "op.register_type(, name='Point')" in out["message"] - assert out["second"] == ["Point(x=1, y=1)", "Point(x=2, y=2)"] + assert out["second"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] NESTED_WRITER = """ From e1bd70089ef1f95152b1e39eadd6694b7eccc0c6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 19:19:56 -0800 Subject: [PATCH 27/31] test(types): docstrings for the user-type tests; changelog for the review fixes (ITL-881) Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 10 +- ...2026-11-11-itl-881-field-order-identity.md | 6 ++ .../test_dataclass_logical_type_factory.py | 5 +- .../test_type_identity_failure.py | 12 ++- tests/test_user_types/child_harness.py | 2 + tests/test_user_types/test_field_order.py | 29 ++++++ tests/test_user_types/test_identity_record.py | 23 +++++ tests/test_user_types/test_read_path.py | 8 ++ tests/test_user_types/test_register_type.py | 93 +++++++++++++++++++ .../test_workshop_scenarios.py | 20 ++++ 10 files changed, 200 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8b7a9a95..d074b8e8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,7 +94,9 @@ class defined in a module, or the name declared with `op.register_type`. a run returned 0 rows with only log warnings. - **A stored shape that disagrees with the registered class raises.** `DeltaTableDatabase` raises the new `orcapod.errors.TypeIdentityError` as is. It is not a `StoreReadError`: - the table is intact and the class differs. Reading a column whose class cannot be found raises + the table is intact and the class differs. Like a store error, it fails the run under + every `error_policy`, with both orchestrators: a stored row that cannot be decoded is + never skipped as a failed row. Reading a column whose class cannot be found raises `ImportError` naming the column, the type and the `op.register_type` call to declare it. - **A subclass is its own type.** A dataclass or pydantic subclass was stored as its parent when the parent was registered first: its extra fields were silently dropped from @@ -199,6 +201,12 @@ that raised. See *When the store cannot be written* in the function-pods guide. - `PolarsFilter(predicates=[...])` and `stream.polars_filter(...)` with the same predicates now have the same hashes (`predicates` is stored as a tuple). +#### A nested class stored as `None` reads back (ITL-881) + +A dataclass or pydantic model with an optional nested class (`child: Inner | None`) that +stored `None` raised `TypeError: 'NoneType' object is not subscriptable` when the value +was read back. It now reads back as `None`. + #### Stored dataclass and pydantic values stay readable after Delta compaction (ITL-881) Delta compaction (`optimize.compact()`) rewrites a `large_string` struct field as diff --git a/superpowers/plans/2026-11-11-itl-881-field-order-identity.md b/superpowers/plans/2026-11-11-itl-881-field-order-identity.md index f6eb76d55..cb1ef7795 100644 --- a/superpowers/plans/2026-11-11-itl-881-field-order-identity.md +++ b/superpowers/plans/2026-11-11-itl-881-field-order-identity.md @@ -91,3 +91,9 @@ Full suite on 3.11, 3.12, 3.13 one at a time; ruff; basedpyright; push; CI; Code - Out of scope, to report: the semantic hasher expands a namedtuple with fields sorted by name (`semantic_hasher._expand_namedtuple`), so namedtuple field order does not enter its hash, although a namedtuple compares as an ordered tuple. +- Second CodeRabbit round (plan approved by the coordinator): identity errors fatal under + `error_policy="continue"` in both orchestrators (own commit); `None` nested class on read; + stored record checked against its extension name; lazy candidate lookup; distinct scenario + values; test docstrings; spec example. Declined: stale read converter after rebinding (not + reproduced). Deferred: pydantic field alias fails on read (also on `main`). +- Full suite after these: 7794 passed, 96 skipped, 20 xfailed on 3.11, 3.12 and 3.13. diff --git a/tests/test_logical_types/test_dataclass_logical_type_factory.py b/tests/test_logical_types/test_dataclass_logical_type_factory.py index b40a6a978..c4e940057 100644 --- a/tests/test_logical_types/test_dataclass_logical_type_factory.py +++ b/tests/test_logical_types/test_dataclass_logical_type_factory.py @@ -727,8 +727,9 @@ def test_reconstruct_rejects_a_record_that_does_not_match_the_extension_name(): # This test is about the factory's read entry point: an inconsistent stored # column cannot be produced through the public interface. metadata = _metadata(_Flat) - record = dict(metadata["identity"]) # pyright: ignore[reportCallIssue, reportArgumentType] # the identity is a JSON object - record["fields"] = [["name", "str"], ["count", "str"]] + stored = metadata["identity"] + assert isinstance(stored, dict) + record = {**stored, "fields": [["name", "str"], ["count", "str"]]} storage = pa.struct( [pa.field("name", pa.large_string()), pa.field("count", pa.int64())] ) diff --git a/tests/test_pipeline/test_type_identity_failure.py b/tests/test_pipeline/test_type_identity_failure.py index b62c4f44c..584e324d9 100644 --- a/tests/test_pipeline/test_type_identity_failure.py +++ b/tests/test_pipeline/test_type_identity_failure.py @@ -28,6 +28,8 @@ from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +Orchestrator = SyncPipelineOrchestrator | AsyncPipelineOrchestrator + _CALLS: list[int] = [] @@ -47,7 +49,7 @@ def pair(x: int) -> Pair: return Pair(x, x) -ORCHESTRATORS: dict[str, Callable[[], object]] = { +ORCHESTRATORS: dict[str, Callable[[], Orchestrator]] = { "sync": lambda: SyncPipelineOrchestrator(error_policy="continue"), "async": lambda: AsyncPipelineOrchestrator(error_policy="continue"), } @@ -62,7 +64,7 @@ def _reset_calls(): @pytest.fixture(params=list(ORCHESTRATORS)) -def make_orchestrator(request: pytest.FixtureRequest) -> Callable[[], object]: +def make_orchestrator(request: pytest.FixtureRequest) -> Callable[[], Orchestrator]: """Return a factory for the parametrized orchestrator (``continue`` policy).""" return ORCHESTRATORS[request.param] @@ -121,7 +123,7 @@ def store_another_shape(result_table: Path) -> None: @pytest.fixture -def stored(tmp_path: Path, make_orchestrator: Callable[[], object]) -> Path: +def stored(tmp_path: Path, make_orchestrator: Callable[[], Orchestrator]) -> Path: """A store in which the job has computed and stored all three rows.""" store_path = tmp_path / "store" build_job(store_path).run(orchestrator=make_orchestrator()) @@ -131,7 +133,7 @@ def stored(tmp_path: Path, make_orchestrator: Callable[[], object]) -> Path: def test_other_stored_shape_fails_the_run( - stored: Path, make_orchestrator: Callable[[], object] + stored: Path, make_orchestrator: Callable[[], Orchestrator] ) -> None: """A result table holding another shape fails the run before any function call.""" result_table, _ = table_dirs(stored) @@ -145,7 +147,7 @@ def test_other_stored_shape_fails_the_run( def test_other_stored_shape_fails_the_run_in_the_per_row_lookup( - stored: Path, make_orchestrator: Callable[[], object] + stored: Path, make_orchestrator: Callable[[], Orchestrator] ) -> None: """The per-row lookup's identity error fails the run under ``continue``.""" # With the pipeline table gone, the first read of the result table is the per-row diff --git a/tests/test_user_types/child_harness.py b/tests/test_user_types/child_harness.py index 46d813f7b..aa6b7268d 100644 --- a/tests/test_user_types/child_harness.py +++ b/tests/test_user_types/child_harness.py @@ -103,6 +103,7 @@ def error_text(exc: BaseException) -> str: def table_paths(root): + """Return the sorted relative paths of the Delta tables under ``root``.""" return sorted( str(p.parent.relative_to(root)) for p in Path(root).rglob("_delta_log") if p.is_dir() ) @@ -128,6 +129,7 @@ def make_pod(point_cls): calls = [] def shift(p): + """Record the call and return ``p`` unchanged.""" calls.append(p) return p diff --git a/tests/test_user_types/test_field_order.py b/tests/test_user_types/test_field_order.py index c69596533..8f8395a29 100644 --- a/tests/test_user_types/test_field_order.py +++ b/tests/test_user_types/test_field_order.py @@ -27,6 +27,8 @@ @op.register_type(name="itl881.order.Data") @dataclasses.dataclass class DataXY: + """Dataclass declared under ``itl881.order.Data`` with fields ``x``, ``y``.""" + x: int y: int @@ -34,18 +36,24 @@ class DataXY: @op.register_type(name="itl881.order.Data") @dataclasses.dataclass class DataYX: + """Dataclass declared under ``itl881.order.Data`` with fields ``y``, ``x``.""" + y: int x: int @op.register_type(name="itl881.order.Model") class ModelXY(pydantic.BaseModel): + """Pydantic model declared under ``itl881.order.Model`` with fields ``x``, ``y``.""" + x: int y: int @op.register_type(name="itl881.order.Model") class ModelYX(pydantic.BaseModel): + """Pydantic model declared under ``itl881.order.Model`` with fields ``y``, ``x``.""" + y: int x: int @@ -53,12 +61,16 @@ class ModelYX(pydantic.BaseModel): @op.register_type(name="itl881.order.Outer") @dataclasses.dataclass class OuterOfXY: + """Declared under ``itl881.order.Outer``; nests ``DataXY``.""" + inner: DataXY @op.register_type(name="itl881.order.Outer") @dataclasses.dataclass class OuterOfYX: + """Declared under ``itl881.order.Outer``; nests ``DataYX``.""" + inner: DataYX @@ -71,20 +83,24 @@ class OuterOfYX: def _source(value: Any) -> DictSource: + """Return a one-row ``DictSource`` holding ``value`` in data column ``p``.""" return DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"], source_id="s") def _column_type(value: Any) -> pa.ExtensionType: + """Return the Arrow extension type of the column holding ``value``.""" column_type = _source(value).as_table().schema.field("p").type assert isinstance(column_type, pa.ExtensionType) return column_type def _type_name(value: Any) -> str: + """Return the Arrow extension name of the column holding ``value``.""" return _column_type(value).extension_name def _row_hash(value: Any) -> str: + """Return the content hash of the row holding ``value``.""" _, data = next(iter(_source(value).iter_data())) return data.content_hash().to_string() @@ -96,11 +112,13 @@ def pair(request: pytest.FixtureRequest) -> tuple[Any, Any]: def test_other_field_order_is_another_type(pair: tuple[Any, Any]) -> None: + """The same fields in another declared order give another extension name.""" first, second = pair assert _type_name(first) != _type_name(second) def test_other_field_order_has_other_hashes(pair: tuple[Any, Any]) -> None: + """Another field order changes the row, content and pipeline hashes.""" first, second = pair assert _row_hash(first) != _row_hash(second) assert _source(first).content_hash() != _source(second).content_hash() @@ -119,6 +137,7 @@ def test_other_field_order_has_other_hashes(pair: tuple[Any, Any]) -> None: def test_stored_struct_lists_fields_in_declared_order( value: Any, names: list[str] ) -> None: + """The stored struct lists its fields in the class's declared order.""" storage = _column_type(value).storage_type assert isinstance(storage, pa.StructType) assert storage.names == names @@ -127,6 +146,7 @@ def test_stored_struct_lists_fields_in_declared_order( def test_values_of_both_orders_read_back_as_their_own_class( pair: tuple[Any, Any], ) -> None: + """A value of either order reads back equal and as its own class.""" for value in pair: _, data = next(iter(_source(value).iter_data())) read = data.as_dict()["p"] @@ -135,12 +155,16 @@ def test_values_of_both_orders_read_back_as_their_own_class( def test_pods_over_the_two_orders_have_different_pipeline_hashes() -> None: + """Pods over the two orders differ in pipeline hash and output type.""" + @function_pod(output_keys="q") def keep_xy(p: DataXY) -> DataXY: + """Return the ``DataXY`` value unchanged.""" return p @function_pod(output_keys="q") def keep_yx(p: DataYX) -> DataYX: + """Return the ``DataYX`` value unchanged.""" return p xy = keep_xy.pod(_source(DataXY(x=1, y=2))) @@ -155,6 +179,8 @@ def keep_yx(p: DataYX) -> DataYX: @op.register_type(name="itl881.order.Same") @dataclasses.dataclass class SameFirst: + """First class declared under ``itl881.order.Same``.""" + a: int b: str @@ -162,11 +188,14 @@ class SameFirst: @op.register_type(name="itl881.order.Same") @dataclasses.dataclass class SameSecond: + """Second class under ``itl881.order.Same``, same fields in the same order.""" + a: int b: str def test_same_fields_in_the_same_order_are_one_type() -> None: + """Two classes with one name and the same ordered fields are one type.""" with pytest.warns(UserWarning, match="same identity"): assert _type_name(SameFirst(1, "x")) == _type_name(SameSecond(1, "x")) assert _row_hash(SameFirst(1, "x")) == _row_hash(SameSecond(1, "x")) diff --git a/tests/test_user_types/test_identity_record.py b/tests/test_user_types/test_identity_record.py index dc85b56b8..75ee5ad0e 100644 --- a/tests/test_user_types/test_identity_record.py +++ b/tests/test_user_types/test_identity_record.py @@ -20,10 +20,12 @@ @pytest.fixture def converter(): + """Return the default data context's type converter.""" return get_default_context().type_converter def _make_class(module: str, qualname: str = "Point") -> type: + """Return a one-field dataclass with the given module and qualified name.""" cls = dataclasses.make_dataclass(qualname, [("x", int)]) cls.__module__ = module cls.__qualname__ = qualname @@ -31,7 +33,10 @@ def _make_class(module: str, qualname: str = "Point") -> type: class TestIdentityString: + """The identity record, its canonical JSON and the identity string.""" + def test_documented_example(self) -> None: + """The documented example record gives the documented identity string.""" record = uci.identity_record( "dataclass", "Point", [("x", "int"), ("y", "float")] ) @@ -40,6 +45,7 @@ def test_documented_example(self) -> None: ) def test_canonical_json_is_compact_and_keeps_field_order(self) -> None: + """Canonical JSON is compact, sorts keys and keeps the field order.""" record = uci.identity_record("pydantic", "P", [("b", "int"), ("a", "str")]) assert ( uci.canonical_json(record) @@ -47,6 +53,7 @@ def test_canonical_json_is_compact_and_keeps_field_order(self) -> None: ) def test_field_order_is_part_of_the_identity(self) -> None: + """Records differing only in field order have different identity strings.""" a = uci.identity_record("dataclass", "P", [("x", "int"), ("y", "str")]) b = uci.identity_record("dataclass", "P", [("y", "str"), ("x", "int")]) assert uci.identity_string(a) != uci.identity_string(b) @@ -65,18 +72,22 @@ def test_field_order_is_part_of_the_identity(self) -> None: def test_each_component_changes_the_identity( self, other: tuple[str, str, list[tuple[str, str]]] ) -> None: + """Changing the kind, the name or any field changes the identity string.""" base = uci.identity_record("dataclass", "Point", [("x", "int"), ("y", "float")]) assert uci.identity_string(base) != uci.identity_string( uci.identity_record(*other) ) def test_describe_fields_keeps_the_given_order(self) -> None: + """``describe_fields`` renders the fields in the order given.""" assert ( uci.describe_fields([["y", "float"], ["x", "int"]]) == "{y: float, x: int}" ) class TestTypeTokens: + """Type tokens of field annotations.""" + @pytest.mark.parametrize( ("annotation", "token"), [ @@ -99,42 +110,53 @@ class TestTypeTokens: ], ) def test_token(self, converter, annotation: object, token: str) -> None: + """Each annotation maps to its canonical type token.""" assert uci.type_token(annotation, converter) == token class TestNames: + """Name validation, default names and declared names.""" + @pytest.mark.parametrize( "name", ["", "a@b", "a b", "orcapod.thing", "tab\tname", 3] ) def test_invalid_names_rejected(self, name: object) -> None: + """Empty, non-string, reserved, ``@`` and whitespace names are rejected.""" with pytest.raises((ValueError, TypeError)): uci.validate_name(name) # pyright: ignore[reportArgumentType] @pytest.mark.parametrize("name", ["Point", "geo.Point", "my-types.Point_2"]) def test_valid_names_accepted(self, name: str) -> None: + """Dotted names with letters, digits, ``-`` and ``_`` are accepted.""" uci.validate_name(name) def test_module_class_defaults_to_qualified(self) -> None: + """A module class is named by its module path and class name by default.""" assert uci.resolve_name(_make_class("pkg.models")) == "pkg.models.Point" @pytest.mark.parametrize("module", ["__main__", "__mp_main__"]) def test_main_class_defaults_to_name(self, module: str) -> None: + """A class in ``__main__`` or ``__mp_main__`` is named by its class name.""" assert uci.resolve_name(_make_class(module)) == "Point" def test_nested_main_class_uses_qualname(self) -> None: + """A nested ``__main__`` class is named by its qualified name.""" assert uci.resolve_name(_make_class("__main__", "Outer.Point")) == "Outer.Point" def test_default_name_qualified_name_false(self) -> None: + """``qualified_name=False`` names a module class by its class name alone.""" assert ( uci.default_name(_make_class("pkg.models"), qualified_name=False) == "Point" ) @pytest.mark.parametrize("module", ["__main__", "__mp_main__"]) def test_default_name_qualified_name_true_for_main(self, module: str) -> None: + """``qualified_name=True`` names a main-module class ``__main__.``.""" cls = _make_class(module) assert uci.default_name(cls, qualified_name=True) == "__main__.Point" def test_declared_names_are_nfc_normalized(self) -> None: + """Composed and decomposed spellings of a name declare one NFC name.""" composed, decomposed = "Caf\u00e9", "Cafe\u0301" first, second = _make_class("pkg.a"), _make_class("pkg.b") uci.declare(first, composed) @@ -142,6 +164,7 @@ def test_declared_names_are_nfc_normalized(self) -> None: assert uci.resolve_name(first) == uci.resolve_name(second) == composed def test_declared_name_wins(self) -> None: + """A declared name replaces the default name.""" cls = _make_class("pkg.models") uci.declare(cls, "geo.Point") assert uci.resolve_name(cls) == "geo.Point" diff --git a/tests/test_user_types/test_read_path.py b/tests/test_user_types/test_read_path.py index 7d13f80ef..43e893766 100644 --- a/tests/test_user_types/test_read_path.py +++ b/tests/test_user_types/test_read_path.py @@ -25,11 +25,14 @@ @op.register_type(name="itl881.StoredPoint") @dataclasses.dataclass class StoredPoint: + """Declared under ``itl881.StoredPoint``.""" + x: int y: int def _extension_field_metadata() -> dict[bytes | str, bytes | str]: + """Return Arrow field metadata marking a column as a ``StoredPoint`` column.""" value: Any = StoredPoint(1, 2) source = DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) ext = source.as_table().schema.field("p").type @@ -40,6 +43,7 @@ def _extension_field_metadata() -> dict[bytes | str, bytes | str]: def test_storage_mismatch_raises_instead_of_reading_nothing(tmp_path: Path) -> None: + """A stored column whose storage differs from its class raises on read.""" wrong = pa.struct([pa.field("x", pa.float64()), pa.field("y", pa.int64())]) field = pa.field("q", wrong).with_metadata(_extension_field_metadata()) table = pa.table( @@ -55,17 +59,21 @@ def test_storage_mismatch_raises_instead_of_reading_nothing(tmp_path: Path) -> N @op.register_type(name="itl881.Labelled") @dataclasses.dataclass class Labelled: + """Declared under ``itl881.Labelled``; has a string field.""" + label: str n: int def _run_labelled(db_root: Path) -> tuple[int, list[Labelled]]: + """Run an echo pod over two ``Labelled`` rows; return the call count and values.""" from orcapod import PipelineJob, function_pod calls: list[Labelled] = [] @function_pod(output_keys="q") def echo(p: Labelled) -> Labelled: + """Record the call and return ``p`` unchanged.""" calls.append(p) return p diff --git a/tests/test_user_types/test_register_type.py b/tests/test_user_types/test_register_type.py index 899c54cdc..b7019970a 100644 --- a/tests/test_user_types/test_register_type.py +++ b/tests/test_user_types/test_register_type.py @@ -20,14 +20,17 @@ def _source(value: Any) -> DictSource: + """Return a one-row ``DictSource`` holding ``value`` in data column ``p``.""" return DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) def _type_name(value: Any) -> str: + """Return the Arrow extension name of the column holding ``value``.""" return _source(value).as_table().schema.field("p").type.extension_name def _name_part(value: Any) -> str: + """Return the name component of the extension name, the part before ``@``.""" return _type_name(value).split("@")[0] @@ -39,50 +42,67 @@ def _name_part(value: Any) -> str: @op.register_type(name="itl881.Declared") @dataclasses.dataclass class Declared: + """Declared under ``itl881.Declared``.""" + x: int @dataclasses.dataclass class Undeclared: + """Module-level class with no declared name.""" + x: int @op.register_type @dataclasses.dataclass class BareDecorated: + """Registered with the bare decorator, so named by default.""" + x: int @dataclasses.dataclass class CalledLater: + """Registered by a ``register_type`` call inside a test.""" + x: int def test_declared_name_is_the_name_component() -> None: + """A declared name is the name component of the type's identity.""" assert _name_part(Declared(1)) == "itl881.Declared" def test_module_class_defaults_to_module_path() -> None: + """An undeclared module class is named by its module path and class name.""" assert _name_part(Undeclared(1)) == f"{__name__}.Undeclared" def test_bare_decorator_uses_the_naming_setting() -> None: + """The bare decorator names the class by the default, qualified name.""" assert _name_part(BareDecorated(1)) == f"{__name__}.BareDecorated" def test_call_form_returns_the_class() -> None: + """The call form returns the class and declares the name.""" assert op.register_type(CalledLater, name="itl881.CalledLater") is CalledLater assert _name_part(CalledLater(1)) == "itl881.CalledLater" @pytest.mark.parametrize("name", ["", "a@b", "a b", "orcapod.Point"]) def test_invalid_names_raise(name: str) -> None: + """Empty, reserved, ``@`` and whitespace names raise ``ValueError``.""" with pytest.raises(ValueError): op.register_type(name=name)(Undeclared) def test_non_dataclass_raises_type_error() -> None: + """A class that is neither a dataclass nor a pydantic model raises ``TypeError``.""" + class Plain: + """Plain class, neither a dataclass nor a pydantic model.""" + x: int = 0 with pytest.raises(TypeError, match="dataclasses and pydantic models"): @@ -90,8 +110,12 @@ class Plain: def test_local_class_raises() -> None: + """Registering a class defined inside a function raises ``ValueError``.""" + @dataclasses.dataclass class Local: + """Dataclass defined inside a function.""" + x: int with pytest.raises(ValueError, match="inside a function"): @@ -100,10 +124,13 @@ class Local: @dataclasses.dataclass class UsedFirst: + """Used under its default name before a name is declared.""" + x: int def test_declaring_after_use_under_another_name_raises() -> None: + """Declaring another name for a class already used raises ``ValueError``.""" _type_name(UsedFirst(1)) with pytest.raises(ValueError, match="already registered as"): op.register_type(UsedFirst, name="itl881.UsedFirst") @@ -112,24 +139,31 @@ def test_declaring_after_use_under_another_name_raises() -> None: @op.register_type(qualified_name=False) @dataclasses.dataclass class ShortNamed: + """Registered with ``qualified_name=False``.""" + x: int def test_qualified_name_false_names_a_module_class_by_its_class_name() -> None: + """``qualified_name=False`` names a module class by its class name alone.""" assert _name_part(ShortNamed(1)) == "ShortNamed" def test_name_and_qualified_name_together_raise() -> None: + """Passing both ``name`` and ``qualified_name`` raises ``ValueError``.""" with pytest.raises(ValueError, match="either name"): op.register_type(Undeclared, name="itl881.X", qualified_name=False) @dataclasses.dataclass class DeclaredTwice: + """Declared more than once inside a test.""" + x: int def test_redeclaring_with_the_same_name_is_a_no_op_and_another_name_raises() -> None: + """Redeclaring the same name returns the class; another name raises.""" op.register_type(DeclaredTwice, name="itl881.Twice") assert op.register_type(DeclaredTwice, name="itl881.Twice") is DeclaredTwice with pytest.raises(ValueError, match="already registered as 'itl881.Twice'"): @@ -138,10 +172,13 @@ def test_redeclaring_with_the_same_name_is_a_no_op_and_another_name_raises() -> @dataclasses.dataclass class NotMutated: + """Class whose attributes are compared before and after registration.""" + x: int def test_registration_does_not_modify_the_class() -> None: + """Registering and using a class leaves its attributes unchanged.""" before = dict(vars(NotMutated)) params = NotMutated.__dataclass_params__ # pyright: ignore[reportAttributeAccessIssue] op.register_type(NotMutated, name="itl881.NotMutated") @@ -158,6 +195,8 @@ def test_registration_does_not_modify_the_class() -> None: @op.register_type(name="itl881.Shape") @dataclasses.dataclass class ShapeInt: + """Declared under ``itl881.Shape`` with an ``int`` field ``y``.""" + x: int y: int @@ -165,36 +204,46 @@ class ShapeInt: @op.register_type(name="itl881.Shape") @dataclasses.dataclass class ShapeFloat: + """Declared under ``itl881.Shape`` with a ``float`` field ``y``.""" + x: int y: float @op.register_type(name="itl881.Shape") class ShapeModel(pydantic.BaseModel): + """Pydantic model under ``itl881.Shape`` with the fields of ``ShapeInt``.""" + x: int y: int def test_same_name_different_field_type_is_a_different_type() -> None: + """One name with a different field type is another type and pipeline hash.""" a, b = _source(ShapeInt(1, 2)), _source(ShapeFloat(1, 2.0)) assert _type_name(ShapeInt(1, 2)) != _type_name(ShapeFloat(1, 2.0)) assert a.pipeline_hash() != b.pipeline_hash() def test_dataclass_and_pydantic_model_are_different_types() -> None: + """A dataclass and a pydantic model with one name and fields are two types.""" assert _type_name(ShapeInt(1, 2)) != _type_name(ShapeModel(x=1, y=2)) @op.register_type(name="itl881.Defaults") @dataclasses.dataclass class DefaultsA: + """Declared under ``itl881.Defaults`` with a default, a method and a property.""" + x: int = 0 def method(self) -> int: + """Return ``x``.""" return self.x @property def doubled(self) -> int: + """Return twice ``x``.""" return 2 * self.x @@ -207,6 +256,7 @@ class DefaultsB: def test_defaults_methods_and_properties_are_not_identity() -> None: + """Defaults, methods and properties are not part of the identity.""" first = _type_name(DefaultsA(1)) with pytest.warns(UserWarning, match="same identity"): second = _type_name(DefaultsB(1)) @@ -215,20 +265,26 @@ def test_defaults_methods_and_properties_are_not_identity() -> None: @op.register_type(name="itl881.Computed") class ComputedA(pydantic.BaseModel): + """Pydantic model under ``itl881.Computed`` with a computed field.""" + x: int @pydantic.computed_field @property def doubled(self) -> int: + """Return twice ``x``.""" return 2 * self.x @op.register_type(name="itl881.Computed") class ComputedB(pydantic.BaseModel): + """Pydantic model under ``itl881.Computed`` without the computed field.""" + x: int def test_pydantic_computed_fields_are_not_identity() -> None: + """Pydantic computed fields are not part of the identity.""" first = _type_name(ComputedA(x=1)) with pytest.warns(UserWarning, match="same identity"): second = _type_name(ComputedB(x=1)) @@ -243,55 +299,73 @@ def test_pydantic_computed_fields_are_not_identity() -> None: @op.register_type(name="itl881.InnerA") @dataclasses.dataclass class InnerA: + """Nested class declared under ``itl881.InnerA``.""" + value: int @op.register_type(name="itl881.InnerB") @dataclasses.dataclass class InnerB: + """Nested class declared under ``itl881.InnerB`` with the fields of ``InnerA``.""" + value: int @op.register_type(name="itl881.InnerA") @dataclasses.dataclass class InnerAFloat: + """Nested class declared under ``itl881.InnerA`` with a ``float`` field.""" + value: float @op.register_type(name="itl881.Outer") @dataclasses.dataclass class OuterA: + """Declared under ``itl881.Outer``; nests ``InnerA``.""" + inner: InnerA @op.register_type(name="itl881.Outer") @dataclasses.dataclass class OuterB: + """Declared under ``itl881.Outer``; nests ``InnerB``.""" + inner: InnerB @op.register_type(name="itl881.Outer") @dataclasses.dataclass class OuterAFloat: + """Declared under ``itl881.Outer``; nests ``InnerAFloat``.""" + inner: InnerAFloat def test_nested_class_contributes_its_name() -> None: + """An outer class's identity changes with the name of its nested class.""" assert _type_name(OuterA(InnerA(1))) != _type_name(OuterB(InnerB(1))) def test_nested_class_contributes_its_fields() -> None: + """An outer class's identity changes with the fields of its nested class.""" assert _type_name(OuterA(InnerA(1))) != _type_name(OuterAFloat(InnerAFloat(1.0))) @dataclasses.dataclass class LateInner: + """Nested class declared only after ``EarlyOuter`` was decorated.""" + value: int @op.register_type(name="itl881.OrderOuter") @dataclasses.dataclass class EarlyOuter: + """Declared under ``itl881.OrderOuter`` before its nested class is declared.""" + inner: LateInner @@ -302,16 +376,21 @@ class EarlyOuter: @op.register_type(name="itl881.OrderInner") @dataclasses.dataclass class FirstInner: + """Nested class declared under ``itl881.OrderInner`` before its outer class.""" + value: int @op.register_type(name="itl881.OrderOuter") @dataclasses.dataclass class SecondOuter: + """Declared under ``itl881.OrderOuter`` after its nested class.""" + inner: FirstInner def test_registration_order_does_not_matter() -> None: + """Declaring a nested class before or after its outer class gives one identity.""" late_order = _type_name(EarlyOuter(LateInner(1))) with pytest.warns(UserWarning, match="same identity"): natural_order = _type_name(SecondOuter(FirstInner(1))) @@ -319,6 +398,7 @@ def test_registration_order_does_not_matter() -> None: def test_nested_values_round_trip() -> None: + """A nested value reads back equal, with the inner value as its own class.""" _, data = next(iter(_source(OuterA(InnerA(3))).iter_data())) value = data.as_dict()["p"] assert value == OuterA(InnerA(3)) @@ -327,26 +407,34 @@ def test_nested_values_round_trip() -> None: @dataclasses.dataclass class Node: + """Class that refers to itself.""" + value: int next: Node | None = None @dataclasses.dataclass class CycleA: + """Class that refers to ``CycleB``, which refers back.""" + b: CycleB | None = None @dataclasses.dataclass class CycleB: + """Class that refers to ``CycleA``, which refers back.""" + a: CycleA | None = None def test_self_referential_class_is_rejected_naming_the_field() -> None: + """A self-referential class raises ``RecursiveTypeError`` naming the field.""" with pytest.raises(RecursiveTypeError, match=r"Node\.next -> Node"): _source(Node(1)) def test_cyclic_classes_are_rejected_naming_the_path() -> None: + """Mutually referential classes raise ``RecursiveTypeError`` naming the path.""" with pytest.raises(RecursiveTypeError, match=r"CycleA\.b -> CycleB\.a -> CycleA"): _source(CycleA()) @@ -358,15 +446,20 @@ def test_cyclic_classes_are_rejected_naming_the_path() -> None: @dataclasses.dataclass class Base: + """Base class of ``Child``.""" + x: int @dataclasses.dataclass class Child(Base): + """Subclass of ``Base`` with one more field.""" + z: int = 0 def test_subclass_has_its_own_identity_and_keeps_its_fields() -> None: + """A subclass has its own identity and keeps its fields.""" assert _type_name(Base(1)) != _type_name(Child(1, 2)) _, data = next(iter(_source(Child(1, 2)).iter_data())) assert data.as_dict()["p"] == Child(1, 2) diff --git a/tests/test_user_types/test_workshop_scenarios.py b/tests/test_user_types/test_workshop_scenarios.py index af00319d4..a159583f4 100644 --- a/tests/test_user_types/test_workshop_scenarios.py +++ b/tests/test_user_types/test_workshop_scenarios.py @@ -68,23 +68,27 @@ class Point: @pytest.fixture def workdir(tmp_path: Path) -> Path: + """Return a working directory with the job modules and an empty ``db`` directory.""" write_files(tmp_path, {"wsjob.py": WSJOB, "wsmodels.py": WSMODELS}) (tmp_path / "db").mkdir() return tmp_path def _kernel(workdir: Path, cells: list[str]) -> list[dict[str, Any]]: + """Run ``cells`` in a fresh kernel process and return one log entry per cell.""" code = f"SCENARIO = {cells!r}\n" + KERNEL return run_child(["-c", code], cwd=workdir, env={"WS_DB": str(workdir / "db")}) def _runs(log: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Assert that no cell failed and return the log entries of the run cells.""" errors = [e for e in log if "error" in e] assert not errors, errors return [e for e in log if e["cell"].startswith("run")] def _script(workdir: Path, code: str, **env: str) -> Any: + """Run ``code`` in a fresh process against the working store; return its report.""" return run_child( ["-c", code], cwd=workdir, env={"WS_DB": str(workdir / "db"), **env} ) @@ -208,6 +212,7 @@ def test_subclass_identity_and_fields_do_not_depend_on_registration_order( def test_w1_rerun_define_cell_same_fields_rebinds(workdir: Path) -> None: + """Re-running the class cell unchanged reuses the stored results.""" log = _kernel(workdir, ["setup", "define", "run", "define", "run"]) first, second = _runs(log) assert (first["calls"], first["rows"]) == (2, 2) @@ -226,6 +231,7 @@ def test_w1_old_pod_cell_with_redefined_class(workdir: Path) -> None: def test_w1_run_all_twice(workdir: Path) -> None: + """Running all cells a second time reuses the stored results.""" log = _kernel(workdir, ["setup", "define", "pod", "run_pod"] * 2) first, second = _runs(log) assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) @@ -277,6 +283,7 @@ def test_class_redefined_with_an_added_field_is_a_new_type(workdir: Path) -> Non def test_w2_rerun_with_changed_fields_is_a_new_type(workdir: Path) -> None: + """Re-running with a changed field type gives a new type with its own tables.""" log = _kernel(workdir, ["setup", "define", "run", "define_float", "run"]) first, second = _runs(log) assert (second["calls"], second["rows"]) == (2, 2) @@ -289,6 +296,7 @@ def test_w2_rerun_with_changed_fields_is_a_new_type(workdir: Path) -> None: def test_w3_pydantic_rerun_rebinds(workdir: Path) -> None: + """Re-running a pydantic class cell unchanged reuses the stored results.""" log = _kernel(workdir, ["setup", "define_model", "run", "define_model", "run"]) first, second = _runs(log) assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) @@ -297,6 +305,7 @@ def test_w3_pydantic_rerun_rebinds(workdir: Path) -> None: def test_dataclass_and_pydantic_with_same_name_and_fields_are_distinct( workdir: Path, ) -> None: + """A dataclass and a pydantic model with one name and fields are distinct types.""" log = _kernel(workdir, ["setup", "define", "run", "define_model", "run"]) first, second = _runs(log) assert second["calls"] == 2 @@ -305,6 +314,7 @@ def test_dataclass_and_pydantic_with_same_name_and_fields_are_distinct( def test_w4_module_reload_rebinds(workdir: Path) -> None: + """Reloading the class's module reuses the stored results.""" log = _kernel(workdir, ["setup", "import_mod", "run", "reload_mod", "run"]) first, second = _runs(log) assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) @@ -312,6 +322,7 @@ def test_w4_module_reload_rebinds(workdir: Path) -> None: def test_main_class_notice_once(workdir: Path) -> None: + """The notebook-or-script class notice is given once and names the class.""" log = _kernel(workdir, ["setup", "define", "run", "define", "run"]) notices = [w for e in log for w in e["warnings"] if "notebook or script" in w] assert len(notices) == 1 @@ -324,6 +335,7 @@ def test_main_class_notice_once(workdir: Path) -> None: def test_w5_restart_same_class_hits_cache(workdir: Path) -> None: + """The same class in a restarted kernel reuses the stored results.""" first = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] second = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) @@ -340,6 +352,7 @@ def test_w6_restart_with_changed_fields_recomputes(workdir: Path) -> None: def test_w7_script_as_main_twice(workdir: Path) -> None: + """A script run as a file and then with ``-m`` reuses the stored results.""" write_files( workdir, { @@ -369,6 +382,7 @@ def main(): def test_w8_script_then_import_hits_with_declared_name(workdir: Path) -> None: + """A declared name lets a module run as main and then imported share results.""" write_files( workdir, { @@ -436,6 +450,7 @@ class Point: def test_w11_module_without_declared_name_is_a_different_type(workdir: Path) -> None: + """A module class without a declared name is another type than the notebook's.""" notebook = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] module = _runs(_kernel(workdir, ["setup", "import_mod", "run"]))[0] assert module["calls"] == 2 @@ -469,10 +484,12 @@ def test_w11_module_without_declared_name_is_a_different_type(workdir: Path) -> def _store_from_notebook(workdir: Path) -> None: + """Store results for the notebook ``Point`` class in the working store.""" _runs(_kernel(workdir, ["setup", "define", "run"])) def test_w12_read_notebook_results_from_script_with_same_class(workdir: Path) -> None: + """A script defining the same class reads the notebook's stored values.""" _store_from_notebook(workdir) define = "from dataclasses import dataclass\n@dataclass\nclass Point:\n x: int\n y: int\n" out = _script(workdir, READER.format(define=define)) @@ -482,6 +499,7 @@ def test_w12_read_notebook_results_from_script_with_same_class(workdir: Path) -> def test_read_with_declared_class_from_a_module(workdir: Path) -> None: + """A module class declared under the stored name reads the stored values.""" _store_from_notebook(workdir) write_files( workdir, @@ -504,6 +522,7 @@ class Point: def test_read_without_the_class_names_it_and_how_to_register(workdir: Path) -> None: + """A read without the class raises, naming it, its fields and ``register_type``.""" _store_from_notebook(workdir) out = _script(workdir, READER.format(define="")) assert out["error"].startswith("ImportError: Column 'q'") @@ -513,6 +532,7 @@ def test_read_without_the_class_names_it_and_how_to_register(workdir: Path) -> N def test_read_with_different_fields_raises_naming_both_shapes(workdir: Path) -> None: + """A read with a class of other fields raises, naming both shapes.""" _store_from_notebook(workdir) define = "from dataclasses import dataclass\n@dataclass\nclass Point:\n x: int\n y: float\n" out = _script(workdir, READER.format(define=define)) From 38c583a346446491e012eebba03cd02bbfa37c06 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 19:51:41 -0800 Subject: [PATCH 28/31] fix(pipeline): failing observer callbacks do not replace an identity error (ITL-881) keep_store_error and note_callback_failure also accept a TypeIdentityError, so an observer callback or fail-loud post-run hook that raises while it propagates is noted on it instead of replacing it, in both orchestrators. Co-Authored-By: Claude Fable 5.2 --- src/orcapod/core/function_pod.py | 10 +++-- src/orcapod/core/nodes/function_node.py | 6 ++- src/orcapod/pipeline/async_orchestrator.py | 5 ++- src/orcapod/pipeline/sync_orchestrator.py | 9 ++++- src/orcapod/utils/store_errors.py | 20 ++++++---- .../test_type_identity_failure.py | 39 +++++++++++++++++++ 6 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 5dbecf059..f1b04c11d 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -329,7 +329,8 @@ def add_post_run_hook(self, hook: PostRunHook) -> None: def _fire_post_run_hooks(self, payload: PostRunPayload) -> None: """Fire all registered hooks with payload in registration order. - When the invocation failed with a ``StoreError``, a fail-loud hook that + When the invocation failed with a ``StoreError`` or a ``TypeIdentityError`` + (both fail the run under every error policy), a fail-loud hook that raises does not replace it: its failure is logged and noted on the store error, the remaining hooks are skipped as for any fail-loud failure, and the caller re-raises the store error. @@ -338,7 +339,9 @@ def _fire_post_run_hooks(self, payload: PostRunPayload) -> None: payload: The post-run payload to pass to each hook. """ error = payload.stats.error - store_error = error if isinstance(error, StoreError) else None + store_error = ( + error if isinstance(error, (StoreError, TypeIdentityError)) else None + ) for hook in self._post_run_hooks: fn = hook.fn if isinstance(hook, HookConfig) else hook on_error = hook.on_error if isinstance(hook, HookConfig) else "raise" @@ -609,7 +612,8 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: except TypeIdentityError as exc: # A stored row that cannot be decoded as its class is never # skipped as a row failure (ITL-881). - obs.on_data_crash(pod_label, tag, data, exc) + with keep_store_error(exc, "Observer on_data_crash"): + obs.on_data_crash(pod_label, tag, data, exc) raise except Exception as exc: logger.debug( diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 89ff8cafa..a02a7403d 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1321,8 +1321,10 @@ def execute( except TypeIdentityError as exc: # A stored row that cannot be decoded as its class is never # skipped as a row failure, whatever the error policy (ITL-881). - ctx_obs.on_data_crash(node_label, tag, data, exc) - ctx_obs.on_node_end(node_label, node_hash) + with keep_store_error(exc, "Observer on_data_crash"): + ctx_obs.on_data_crash(node_label, tag, data, exc) + with keep_store_error(exc, "Observer on_node_end"): + ctx_obs.on_node_end(node_label, node_hash) raise except Exception as exc: logger.warning( diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 049440dd9..16b85ecde 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -141,7 +141,7 @@ async def _run_async( effective_observer.on_run_start(run_id, pipeline_uri=pipeline_uri) topo_order: list[Any] = [] - store_error: StoreError | None = None + store_error: StoreError | TypeIdentityError | None = None try: topo_order = list(graph.topological_sort()) buf = self._buffer_size @@ -276,7 +276,8 @@ async def _run_async( exc for exc in _leaves(group) if isinstance(exc, TypeIdentityError) ] if identity_errors: - raise identity_errors[0] + store_error = identity_errors[0] + raise store_error raise finally: # A failing on_run_end must not replace a failed store read or write. diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index dfe238f5e..981706352 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -10,7 +10,7 @@ import uuid from typing import TYPE_CHECKING, Any, Literal -from orcapod.errors import StoreError +from orcapod.errors import StoreError, TypeIdentityError from orcapod.pipeline.result import OrchestratorResult from orcapod.utils import arrow_utils from orcapod.utils.store_errors import keep_store_error @@ -79,7 +79,7 @@ def run( effective_observer = observer if observer is not None else NoOpObserver() effective_observer.on_run_start(run_id, pipeline_uri=pipeline_uri) - store_error: StoreError | None = None + store_error: StoreError | TypeIdentityError | None = None current: Any = None try: topo_order = list(graph.topological_sort()) @@ -142,6 +142,11 @@ def run( exc.add_context(node_label=current.label) store_error = exc raise + except TypeIdentityError as exc: + # A stored row that cannot be decoded as its class stops the run as a + # store error does; on_run_end must not replace it (ITL-881). + store_error = exc + raise finally: # A failing on_run_end must not replace a failed store write. with keep_store_error(store_error, "Observer on_run_end"): diff --git a/src/orcapod/utils/store_errors.py b/src/orcapod/utils/store_errors.py index a28c1a640..d4f21ad4d 100644 --- a/src/orcapod/utils/store_errors.py +++ b/src/orcapod/utils/store_errors.py @@ -1,8 +1,9 @@ """Helpers that keep a ``StoreError`` intact on its way out of a run. -A failed store read or write always fails the run (ITL-878, ITL-884). User callbacks that run while -the error propagates (a post-run hook, an observer callback) may fail too; their -failure is logged and noted on the store error, never raised in its place. +A failed store read or write always fails the run (ITL-878, ITL-884), and so does a +stored row that cannot be decoded as its class (``TypeIdentityError``, ITL-881). User +callbacks that run while the error propagates (a post-run hook, an observer callback) +may fail too; their failure is logged and noted on the error, never raised in its place. """ from __future__ import annotations @@ -11,18 +12,18 @@ from collections.abc import Iterator from contextlib import contextmanager -from orcapod.errors import StoreError +from orcapod.errors import StoreError, TypeIdentityError logger = logging.getLogger(__name__) def note_callback_failure( - store_error: StoreError, during: str, failure: Exception + store_error: StoreError | TypeIdentityError, during: str, failure: Exception ) -> None: """Log a callback failure and record it on the propagating store error. Args: - store_error: The store error on its way out of the run. + store_error: The store error (or identity error) on its way out of the run. during: What was running, for the log and the note, such as ``"Observer on_data_crash"``. failure: The exception the callback raised. @@ -41,7 +42,9 @@ def note_callback_failure( @contextmanager -def keep_store_error(store_error: StoreError | None, during: str) -> Iterator[None]: +def keep_store_error( + store_error: StoreError | TypeIdentityError | None, during: str +) -> Iterator[None]: """Run a callback so that it cannot replace a propagating store error. With ``store_error`` set, an ``Exception`` the block raises is logged and noted on @@ -50,7 +53,8 @@ def keep_store_error(store_error: StoreError | None, during: str) -> Iterator[No unchanged and its exceptions propagate. Args: - store_error: The store error on its way out, or ``None`` when none is. + store_error: The store error (or identity error) on its way out, or ``None`` + when none is. during: What the block runs, for the log and the note. Yields: diff --git a/tests/test_pipeline/test_type_identity_failure.py b/tests/test_pipeline/test_type_identity_failure.py index 584e324d9..9e532e356 100644 --- a/tests/test_pipeline/test_type_identity_failure.py +++ b/tests/test_pipeline/test_type_identity_failure.py @@ -26,6 +26,7 @@ from orcapod.errors import StoreError, TypeIdentityError from orcapod.pipeline import PipelineJob from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator Orchestrator = SyncPipelineOrchestrator | AsyncPipelineOrchestrator @@ -161,3 +162,41 @@ def test_other_stored_shape_fails_the_run_in_the_per_row_lookup( assert not isinstance(info.value, StoreError) assert _CALLS == [] + + +class _CrashingObserver(NoOpObserver): + """An observer whose callbacks raise while the pod's failure propagates.""" + + def on_data_crash( + self, node_label: str, tag: Any, data: Any, error: Exception + ) -> None: + """Raise from the row-crash callback.""" + raise RuntimeError("observer failed in on_data_crash") + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Raise from the node-end callback of the pod's node.""" + if node_label == "pair": + raise RuntimeError("observer failed in on_node_end") + + def on_run_end(self, run_id: str) -> None: + """Raise from the run-end callback.""" + raise RuntimeError("observer failed in on_run_end") + + +def test_failing_observer_does_not_replace_the_identity_error( + stored: Path, make_orchestrator: Callable[[], Orchestrator] +) -> None: + """Callbacks that raise while the identity error propagates are noted, not raised.""" + result_table, pipeline_table = table_dirs(stored) + store_another_shape(result_table) + shutil.rmtree(pipeline_table) + + with pytest.raises(TypeIdentityError) as info: + build_job(stored).run( + orchestrator=make_orchestrator(), observer=_CrashingObserver() + ) + + notes = getattr(info.value, "__notes__", []) + assert any("observer failed in on_data_crash" in note for note in notes), notes + assert any("observer failed in on_run_end" in note for note in notes), notes + assert _CALLS == [] From e18f05d56abc5a6180dd0e35736c967eb986cd6a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 22:12:20 -0800 Subject: [PATCH 29/31] docs(changelog): move the ITL-881 entry to a changelog fragment (ITL-961) Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 69 --------------------------------------- changelog.d/ITL-881.md | 74 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 69 deletions(-) create mode 100644 changelog.d/ITL-881.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d074b8e8d..dfdc26d3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,46 +70,6 @@ first `InputValidationError`, with each other failure attached as a note and log stay queued for the next flush. A flush that fails only for other reasons still raises `RuntimeError`, with the first failure as its cause. -#### Dataclasses and pydantic models are identified by name, kind and fields (ITL-881) - -Orcapod 0.3 identifies a dataclass or pydantic model by its name, its kind (dataclass or -pydantic) and its fields' names and types, in declared order. The name is the class name for a -class defined in a notebook or script (`__main__`), the module path and class name for a -class defined in a module, or the name declared with `op.register_type`. - -- **Hashes change for every pipeline that uses such a class.** The class's logical type - name and Arrow extension name become `Name@`; the - stored struct lists fields in declared order, as before. Every schema hash, pipeline hash, table path, - system-tag column name and data hash that involves a dataclass or pydantic value moves. - The first run under 0.3 computes those pod and operator results again, into new tables. - Tables written by earlier versions are left untouched and are not read by 0.3 pipelines; - reading such a column directly raises `TypeIdentityError`. There is no migration tool. - Results that involve no dataclass or pydantic value are unaffected. -- **Changing a class's fields makes a new type.** Adding, removing, renaming, retyping or - reordering fields all count: a dataclass compares its fields as an ordered tuple and a - pydantic model dumps in declared order, so two classes with the same fields in another - order are different types. Defaults, methods and pydantic computed fields are not part - of the identity. The new type's pods get new tables; the results of - the old definition stay where they are. Before, a changed class shared the old tables and - a run returned 0 rows with only log warnings. -- **A stored shape that disagrees with the registered class raises.** `DeltaTableDatabase` - raises the new `orcapod.errors.TypeIdentityError` as is. It is not a `StoreReadError`: - the table is intact and the class differs. Like a store error, it fails the run under - every `error_policy`, with both orchestrators: a stored row that cannot be decoded is - never skipped as a failed row. Reading a column whose class cannot be found raises - `ImportError` naming the column, the type and the `op.register_type` call to declare it. -- **A subclass is its own type.** A dataclass or pydantic subclass was stored as its - parent when the parent was registered first: its extra fields were silently dropped from - stored data, and the pod's hash and table path depended on the registration order. It - now has its own identity and keeps all its fields, in any registration order. -- **Recursive classes** raise `RecursiveTypeError` (a `TypeError`) naming the field path, - for example `Node.next -> Node`. -- Pinned golden hash entries changed (approved): `semantic.json` `type.dataclass`, - `type.pydantic`; `schema.json` `multi_field.logical_mix`, - `single_field.{dataclass,pydantic}.{arrow_type,schema_hash}`; `datagram.json` - `{dataclass,pydantic}.{content_hash,pipeline_hash,system_tag_columns}` and the per-row - `data_hash`, `_tag_record_id::…` and `_tag_source_id::…` entries of both cases. - ### Added - **`orcapod.errors.StoreWriteError`** — raised when a node's results or records cannot be @@ -121,19 +81,6 @@ class defined in a module, or the name declared with `op.register_type`. - **`orcapod.errors.StoreError`** — the base of `StoreReadError` and `StoreWriteError`; `except StoreError` handles both. (ITL-884 #290) -#### `op.register_type` — declare the name of your own data types (ITL-881) - -`@op.register_type(name="Point")` declares the name a dataclass or pydantic model is -identified by, wherever the class is defined. `@op.register_type(qualified_name=False)` names a -module class by its class name alone. Registration never modifies the class; declaring the -same class under a second name raises `ValueError`. See -[Your Own Data Types](docs/concepts/your-own-data-types.md). - -Re-running a notebook cell that defines a dataclass or pydantic model, or reloading its -module, no longer breaks later pods with `ValueError: … already bound`: a class with the -same name and the same fields in the same order is the same type, and its stored results -are found again. - ### Fixed #### A Delta table that cannot be read is no longer reported as having no rows (ITL-884) @@ -201,22 +148,6 @@ that raised. See *When the store cannot be written* in the function-pods guide. - `PolarsFilter(predicates=[...])` and `stream.polars_filter(...)` with the same predicates now have the same hashes (`predicates` is stored as a tuple). -#### A nested class stored as `None` reads back (ITL-881) - -A dataclass or pydantic model with an optional nested class (`child: Inner | None`) that -stored `None` raised `TypeError: 'NoneType' object is not subscriptable` when the value -was read back. It now reads back as `None`. - -#### Stored dataclass and pydantic values stay readable after Delta compaction (ITL-881) - -Delta compaction (`optimize.compact()`) rewrites a `large_string` struct field as -`string`. Reading a stored dataclass or pydantic column with a `str` or `bytes` field then -failed, and a cached run returned 0 rows. Extension columns now accept large and non-large -layouts of the same type at any nesting depth, as they already did at the top level -(ITL-602). Known limitation: such a non-large column read over Arrow IPC or the C stream -interface (not Parquet or Delta) gives invalid arrays, at any depth; Orcapod's own writers -never produce one (ITL-894). - ## [0.2.0] — 2026-11-10 **Distribution:** Orcapod is MIT licensed starting with `0.2.0` (it was proprietary on diff --git a/changelog.d/ITL-881.md b/changelog.d/ITL-881.md new file mode 100644 index 000000000..10652626a --- /dev/null +++ b/changelog.d/ITL-881.md @@ -0,0 +1,74 @@ +### Breaking changes and migration + +#### Dataclasses and pydantic models are identified by name, kind and fields (ITL-881) + +Orcapod 0.3 identifies a dataclass or pydantic model by its name, its kind (dataclass or +pydantic) and its fields' names and types, in declared order. The name is the class name for a +class defined in a notebook or script (`__main__`), the module path and class name for a +class defined in a module, or the name declared with `op.register_type`. + +- **Hashes change for every pipeline that uses such a class.** The class's logical type + name and Arrow extension name become `Name@`; the + stored struct lists fields in declared order, as before. Every schema hash, pipeline hash, table path, + system-tag column name and data hash that involves a dataclass or pydantic value moves. + The first run under 0.3 computes those pod and operator results again, into new tables. + Tables written by earlier versions are left untouched and are not read by 0.3 pipelines; + reading such a column directly raises `TypeIdentityError`. There is no migration tool. + Results that involve no dataclass or pydantic value are unaffected. +- **Changing a class's fields makes a new type.** Adding, removing, renaming, retyping or + reordering fields all count: a dataclass compares its fields as an ordered tuple and a + pydantic model dumps in declared order, so two classes with the same fields in another + order are different types. Defaults, methods and pydantic computed fields are not part + of the identity. The new type's pods get new tables; the results of + the old definition stay where they are. Before, a changed class shared the old tables and + a run returned 0 rows with only log warnings. +- **A stored shape that disagrees with the registered class raises.** `DeltaTableDatabase` + raises the new `orcapod.errors.TypeIdentityError` as is. It is not a `StoreReadError`: + the table is intact and the class differs. Like a store error, it fails the run under + every `error_policy`, with both orchestrators: a stored row that cannot be decoded is + never skipped as a failed row. Reading a column whose class cannot be found raises + `ImportError` naming the column, the type and the `op.register_type` call to declare it. +- **A subclass is its own type.** A dataclass or pydantic subclass was stored as its + parent when the parent was registered first: its extra fields were silently dropped from + stored data, and the pod's hash and table path depended on the registration order. It + now has its own identity and keeps all its fields, in any registration order. +- **Recursive classes** raise `RecursiveTypeError` (a `TypeError`) naming the field path, + for example `Node.next -> Node`. +- Pinned golden hash entries changed (approved): `semantic.json` `type.dataclass`, + `type.pydantic`; `schema.json` `multi_field.logical_mix`, + `single_field.{dataclass,pydantic}.{arrow_type,schema_hash}`; `datagram.json` + `{dataclass,pydantic}.{content_hash,pipeline_hash,system_tag_columns}` and the per-row + `data_hash`, `_tag_record_id::…` and `_tag_source_id::…` entries of both cases. + +### Added + +#### `op.register_type` — declare the name of your own data types (ITL-881) + +`@op.register_type(name="Point")` declares the name a dataclass or pydantic model is +identified by, wherever the class is defined. `@op.register_type(qualified_name=False)` names a +module class by its class name alone. Registration never modifies the class; declaring the +same class under a second name raises `ValueError`. See +[Your Own Data Types](docs/concepts/your-own-data-types.md). + +Re-running a notebook cell that defines a dataclass or pydantic model, or reloading its +module, no longer breaks later pods with `ValueError: … already bound`: a class with the +same name and the same fields in the same order is the same type, and its stored results +are found again. + +### Fixed + +#### A nested class stored as `None` reads back (ITL-881) + +A dataclass or pydantic model with an optional nested class (`child: Inner | None`) that +stored `None` raised `TypeError: 'NoneType' object is not subscriptable` when the value +was read back. It now reads back as `None`. + +#### Stored dataclass and pydantic values stay readable after Delta compaction (ITL-881) + +Delta compaction (`optimize.compact()`) rewrites a `large_string` struct field as +`string`. Reading a stored dataclass or pydantic column with a `str` or `bytes` field then +failed, and a cached run returned 0 rows. Extension columns now accept large and non-large +layouts of the same type at any nesting depth, as they already did at the top level +(ITL-602). Known limitation: such a non-large column read over Arrow IPC or the C stream +interface (not Parquet or Delta) gives invalid arrays, at any depth; Orcapod's own writers +never produce one (ITL-894). From e99a0c2ddde7002b35645d9d66f2321a14a43c04 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 23:15:37 -0800 Subject: [PATCH 30/31] docs(design-issues): move the ITL-881 entries to design_issues/ files (ITL-968) Co-Authored-By: Claude Fable 5.2 --- DESIGN_ISSUES.md | 40 ----------------------------------- design_issues/ITL-881-UCI1.md | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 40 deletions(-) create mode 100644 design_issues/ITL-881-UCI1.md diff --git a/DESIGN_ISSUES.md b/DESIGN_ISSUES.md index f90c7ccdd..e322a9507 100644 --- a/DESIGN_ISSUES.md +++ b/DESIGN_ISSUES.md @@ -2710,43 +2710,3 @@ never hidden). Follow-up idea, unverified: a manifest-only row count (`spiral.core.manifest_stats(snapshot).key_space_stats.total_rows`, see the ITL-186 design) could replace the shortcut cheaply once confirmed live. -## `src/orcapod/logical_types/` — identity of user classes (dataclasses, pydantic models) - -### UCI1 — A user class was identified by `module.qualname` only, without its fields -**Status:** resolved -**Severity:** high -**Issue:** ITL-881 - -A dataclass or pydantic model was identified by `f"{module}.{qualname}"`. The same class -defined in a notebook (`__main__`) and in a module were different types; two classes with -the same name and different fields shared pipeline hashes and tables, so a changed class -read the old table, failed, and the run returned 0 rows with only log warnings; re-running -the notebook cell that defines a class raised `ValueError: … already bound` until a kernel -restart; a subclass was stored as its parent, dropping its extra fields. - -**Fix:** a user class is identified by `{kind, name, fields}` (spec -`superpowers/specs/2026-11-10-user-class-identity-v0.3.md`). The identity string -`Name@` is the logical type name and Arrow extension -name, with the record in the extension metadata. `fields` lists `[name, type]` pairs in -declared order and the stored struct follows it: field order is part of the identity -(developer, November 11, 2026), so the same fields in another order are another type. -`op.register_type` declares names without modifying the class; `__main__` classes default -to their class name. The registry rebinds an unchanged identity to a re-defined class and -matches user classes by exact class only. The read path finds the class by kind and name, -checks the identity, and raises `ImportError` or `TypeIdentityError` naming the column; -`DeltaTableDatabase` raises `TypeIdentityError` as is, never wrapped in `StoreReadError`. -Testing a read candidate registers neither the class nor its nested classes -(`probe_record`), so a rejected candidate can still be declared under a name. Resolves the entry PR #288 proposes for -the same issue. Not covered: computed fields (deferred by the developer), local classes, -Enum/NamedTuple/TypedDict. - -**Known limitation (ITL-894):** reading a stored column whose storage differs from the -registered one only by layout (`string` vs `large_string`) is accepted at any depth, so -Delta-compacted tables stay readable. `__arrow_ext_deserialize__` then returns the -registered large storage type, which is correct for Parquet and Delta reads (the decoder -converts the data) but not over Arrow IPC or the C stream interface, where Arrow reads -32-bit offsets as 64-bit and the arrays are invalid. Top-level columns had this since -ITL-602; nested struct fields since this fix. Orcapod's own writers and Polars round trips -never produce non-large storage. Pinned by a strict xfail in -`tests/test_user_types/test_read_path.py`. - diff --git a/design_issues/ITL-881-UCI1.md b/design_issues/ITL-881-UCI1.md new file mode 100644 index 000000000..b5a090f36 --- /dev/null +++ b/design_issues/ITL-881-UCI1.md @@ -0,0 +1,39 @@ +## `src/orcapod/logical_types/` — identity of user classes (dataclasses, pydantic models) + +### UCI1 — A user class was identified by `module.qualname` only, without its fields +**Status:** resolved +**Severity:** high +**Issue:** ITL-881 + +A dataclass or pydantic model was identified by `f"{module}.{qualname}"`. The same class +defined in a notebook (`__main__`) and in a module were different types; two classes with +the same name and different fields shared pipeline hashes and tables, so a changed class +read the old table, failed, and the run returned 0 rows with only log warnings; re-running +the notebook cell that defines a class raised `ValueError: … already bound` until a kernel +restart; a subclass was stored as its parent, dropping its extra fields. + +**Fix:** a user class is identified by `{kind, name, fields}` (spec +`superpowers/specs/2026-11-10-user-class-identity-v0.3.md`). The identity string +`Name@` is the logical type name and Arrow extension +name, with the record in the extension metadata. `fields` lists `[name, type]` pairs in +declared order and the stored struct follows it: field order is part of the identity +(developer, November 11, 2026), so the same fields in another order are another type. +`op.register_type` declares names without modifying the class; `__main__` classes default +to their class name. The registry rebinds an unchanged identity to a re-defined class and +matches user classes by exact class only. The read path finds the class by kind and name, +checks the identity, and raises `ImportError` or `TypeIdentityError` naming the column; +`DeltaTableDatabase` raises `TypeIdentityError` as is, never wrapped in `StoreReadError`. +Testing a read candidate registers neither the class nor its nested classes +(`probe_record`), so a rejected candidate can still be declared under a name. Resolves the entry PR #288 proposes for +the same issue. Not covered: computed fields (deferred by the developer), local classes, +Enum/NamedTuple/TypedDict. + +**Known limitation (ITL-894):** reading a stored column whose storage differs from the +registered one only by layout (`string` vs `large_string`) is accepted at any depth, so +Delta-compacted tables stay readable. `__arrow_ext_deserialize__` then returns the +registered large storage type, which is correct for Parquet and Delta reads (the decoder +converts the data) but not over Arrow IPC or the C stream interface, where Arrow reads +32-bit offsets as 64-bit and the arrays are invalid. Top-level columns had this since +ITL-602; nested struct fields since this fix. Orcapod's own writers and Polars round trips +never produce non-large storage. Pinned by a strict xfail in +`tests/test_user_types/test_read_path.py`. From c6837a1512e700112d59ba6cfb77ef3d7fbf263e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Wed, 11 Nov 2026 23:53:53 -0800 Subject: [PATCH 31/31] fix(types): review round on user class identity: rebound containers, union parameters, null storage (ITL-881) - registry: when an identity is rebound to a re-defined class, the list and set logical types over it are rebuilt, at any nesting depth, so a schema reports list[Point] with the class Point is now bound to. - schema_utils: a column of an identity-equivalent class is accepted by a union parameter (New | None) when one of its members accepts it. - registry, user_class_logical_type: storage comparison keeps null types (keep_null=True), so a null field stored where the class has a string raises TypeIdentityError. - tests: run_child rejects a report from a child that exits non-zero; docstrings on nested helpers and in the generated scenario scripts. No hash, identity string or stored schema changes. Co-Authored-By: Claude Fable 5.2 --- .../list_logical_type_factory.py | 13 ++ src/orcapod/logical_types/registry.py | 34 ++- .../logical_types/user_class_identity.py | 1 + .../logical_types/user_class_logical_type.py | 6 +- src/orcapod/user_types.py | 1 + src/orcapod/utils/schema_utils.py | 47 ++++- tests/test_user_types/child_harness.py | 4 + .../test_rebind_and_compatibility.py | 196 ++++++++++++++++++ .../test_workshop_scenarios.py | 20 ++ 9 files changed, 308 insertions(+), 14 deletions(-) create mode 100644 tests/test_user_types/test_rebind_and_compatibility.py diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index 9a173c69d..771229cb2 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -124,6 +124,19 @@ def element_logical_type(self) -> LogicalTypeProtocol: """ return self._element_logical_type + def with_element( + self, element_logical_type: "LogicalTypeProtocol" + ) -> "ListLogicalType": + """Return this list or set logical type built over another element logical type. + + Args: + element_logical_type: The logical type of the elements. + + Returns: + A new ``ListLogicalType`` with the same list or set semantics. + """ + return ListLogicalType(element_logical_type, is_set=self._is_set) + @property def python_type(self) -> type: """The Python generic alias this logical type represents. diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index dd8b765f9..da61498c1 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -104,11 +104,13 @@ def _deserialize( # vice-versa) are genuine semantic errors and always raise. from orcapod.utils.arrow_utils import normalize_to_large_types - if normalize_to_large_types(_storage) == _storage: + if normalize_to_large_types(_storage, keep_null=True) == _storage: # _storage is the canonical large form — accept any same-family variant, # at any nesting depth (a struct field rewritten from large_string to # string by Delta compaction, ITL-881). - storage_ok = normalize_to_large_types(storage_type) == _storage + storage_ok = ( + normalize_to_large_types(storage_type, keep_null=True) == _storage + ) else: # Non-canonical _storage — require exact match. storage_ok = storage_type == _storage @@ -380,12 +382,38 @@ def _rebind( UserWarning, stacklevel=4, ) + self._move_bindings(previous, logical_type) + + def _move_bindings( + self, previous: LogicalTypeProtocol, logical_type: LogicalTypeProtocol + ) -> None: + """Bind an identity's names to ``logical_type`` and rebuild its containers. + + The logical-name and Arrow-name bindings move to ``logical_type`` and its + Python type is bound to it; the previous Python type keeps its binding. Each + registered list or set logical type whose element is ``previous`` is rebuilt + over ``logical_type`` and rebound the same way, at any nesting depth, so + ``list[Point]`` names the class ``Point`` is now bound to. + + Args: + previous: The logical type the names were bound to. + logical_type: The logical type to bind them to, with the same names. + """ + containers = [ + registered + for registered in self._by_logical_name.values() + if getattr(registered, "element_logical_type", None) is previous + ] name = logical_type.logical_type_name self._by_logical_name[name] = logical_type self._by_arrow_name[logical_type.get_arrow_extension_type().extension_name] = ( logical_type ) - self._by_python_type[new_cls] = logical_type + self._by_python_type[logical_type.python_type] = logical_type + for container in containers: + with_element = getattr(container, "with_element", None) + if with_element is not None: + self._move_bindings(container, with_element(logical_type)) def get_by_logical_name(self, name: str) -> LogicalTypeProtocol | None: """Return the logical type registered under *name*, or ``None``.""" diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py index 928150e4e..de1d252e7 100644 --- a/src/orcapod/logical_types/user_class_identity.py +++ b/src/orcapod/logical_types/user_class_identity.py @@ -385,6 +385,7 @@ def candidate_classes(kind: str, name: str) -> Iterator[type]: seen: list[type] = [] def fresh(cls: Any) -> bool: + """Record ``cls`` and return True if it is of the stored kind and not seen yet.""" if class_kind(cls) != kind or cls in seen: return False seen.append(cls) diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py index 84daa189c..758298807 100644 --- a/src/orcapod/logical_types/user_class_logical_type.py +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -410,9 +410,9 @@ def _check_storage( """Raise ``TypeIdentityError`` if stored and registered storage differ beyond layout.""" from orcapod.utils.arrow_utils import normalize_to_large_types - if normalize_to_large_types(lt.storage_type) != normalize_to_large_types( - storage_type - ): + if normalize_to_large_types( + lt.storage_type, keep_null=True + ) != normalize_to_large_types(storage_type, keep_null=True): raise TypeIdentityError( f"Stored values of {uci.describe_record(record)} have Arrow storage type " f"{storage_type}, but the registered class stores {lt.storage_type}." diff --git a/src/orcapod/user_types.py b/src/orcapod/user_types.py index 41afb0022..d63629134 100644 --- a/src/orcapod/user_types.py +++ b/src/orcapod/user_types.py @@ -82,6 +82,7 @@ def register_type( """ def decorate(target: T) -> T: + """Declare ``target`` under the given name and return it unchanged.""" _uci.declare(target, name, qualified_name=qualified_name) return target diff --git a/src/orcapod/utils/schema_utils.py b/src/orcapod/utils/schema_utils.py index c4a236449..b585895f9 100644 --- a/src/orcapod/utils/schema_utils.py +++ b/src/orcapod/utils/schema_utils.py @@ -4,7 +4,8 @@ import logging import sys from collections.abc import Callable, Collection, Mapping, Sequence -from typing import Any, get_args, get_origin, get_type_hints +from types import UnionType +from typing import Any, Union, get_args, get_origin, get_type_hints from orcapod.types import Schema, SchemaLike @@ -62,6 +63,42 @@ def _type_label(type_info: Any) -> str: return str(type_info) +def _union_members(hint: Any) -> tuple[Any, ...] | None: + """Return the members of a union hint, or ``None`` if ``hint`` is not a union.""" + if get_origin(hint) in (Union, UnionType): + return get_args(hint) + return None + + +def _accepts(incoming: Any, receiving: Any) -> bool: + """Return True if a value typed ``incoming`` can be passed where ``receiving`` is expected. + + ``incoming`` must be a subhint of ``receiving``, or the same type up to user-class + identity (a class and its re-definition are one type). A receiving union accepts + what one of its members accepts; an incoming union needs every member accepted. + + Args: + incoming: The type hint of the value offered. + receiving: The type hint of the parameter. + + Returns: + Whether the value can be passed. + """ + from beartype.door import is_subhint + + from orcapod.logical_types.user_class_identity import equivalent_hints + + if is_subhint(incoming, receiving) or equivalent_hints(incoming, receiving): + return True + incoming_members = _union_members(incoming) + if incoming_members is not None: + return all(_accepts(member, receiving) for member in incoming_members) + receiving_members = _union_members(receiving) + if receiving_members is not None: + return any(_accepts(incoming, member) for member in receiving_members) + return False + + def schema_incompatibility( incoming_types: SchemaLike, receiving_types: Schema ) -> str | None: @@ -79,16 +116,10 @@ def schema_incompatibility( ``None`` when the schemas are compatible, otherwise a one-line reason naming the first offending column or parameter. """ - from beartype.door import is_subhint - - from orcapod.logical_types.user_class_identity import equivalent_hints - for key, type_info in incoming_types.items(): if key not in receiving_types: return f"column '{key}' is not a parameter of the function" - if not is_subhint(type_info, receiving_types[key]) and not equivalent_hints( - type_info, receiving_types[key] - ): + if not _accepts(type_info, receiving_types[key]): return ( f"column '{key}' has type {_type_label(type_info)}, " f"the function expects {_type_label(receiving_types[key])}" diff --git a/tests/test_user_types/child_harness.py b/tests/test_user_types/child_harness.py index aa6b7268d..f63083d58 100644 --- a/tests/test_user_types/child_harness.py +++ b/tests/test_user_types/child_harness.py @@ -65,6 +65,10 @@ def run_child( check=False, timeout=timeout, ) + if proc.returncode != 0: + raise AssertionError( + f"child failed (exit {proc.returncode}):\n{proc.stderr[-4000:]}" + ) for line in proc.stdout.splitlines(): if line.startswith(JSON_MARKER): return json.loads(line[len(JSON_MARKER) :]) diff --git a/tests/test_user_types/test_rebind_and_compatibility.py b/tests/test_user_types/test_rebind_and_compatibility.py new file mode 100644 index 000000000..294386bee --- /dev/null +++ b/tests/test_user_types/test_rebind_and_compatibility.py @@ -0,0 +1,196 @@ +"""Rebound user classes in containers, in unions, and foreign storage (ITL-881). + +Two classes declared under one name with the same fields have one identity; the second +definition rebinds it, as a re-run notebook cell does. These tests cover what follows a +rebind: the schema of a list of the class, a pod parameter typed as a union with the +class, and storage that differs from the registered class. Every test declares names under +``itl881.rebind.`` that no other test uses, because declarations are process-wide. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod import function_pod +from orcapod.errors import TypeIdentityError +from orcapod.sources import ArrowTableSource, DictSource +from tests.test_user_types.child_harness import run_child + + +def _source(key: str, value: Any) -> DictSource: + """Return a one-row ``DictSource`` holding ``value`` in data column ``key``.""" + return DictSource(data=[{"k": "a", key: value}], tag_columns=["k"]) + + +def _read(source: Any, key: str) -> Any: + """Return the value of data column ``key`` in the source's only row.""" + return [data.as_dict()[key] for _, data in source.iter_data()][0] + + +# --------------------------------------------------------------------------- +# A list of a rebound class +# --------------------------------------------------------------------------- + + +@op.register_type(name="itl881.rebind.Point") +@dataclasses.dataclass(frozen=True) +class PointV1: + """First definition under ``itl881.rebind.Point``.""" + + x: int + + +@op.register_type(name="itl881.rebind.Point") +@dataclasses.dataclass(frozen=True) +class PointV2: + """Second definition under ``itl881.rebind.Point``, with the same fields.""" + + x: int + + +def test_list_schema_names_the_rebound_class() -> None: + """After a rebind, a list column's schema and values use the new class.""" + first = _source("ps", [PointV1(1)]) + stored = first.as_table() + assert first.output_schema()[1]["ps"] == list[PointV1] + + with pytest.warns(UserWarning, match="same identity"): + second = _source("ps", [PointV2(1)]) + assert second.output_schema()[1]["ps"] == list[PointV2] + assert _read(second, "ps") == [PointV2(1)] + + # A table written before the rebind reads back as the new class as well. + reread = ArrowTableSource(stored.select(["k", "ps"]), tag_columns=["k"]) + assert reread.output_schema()[1]["ps"] == list[PointV2] + assert type(_read(reread, "ps")[0]) is PointV2 + + +@op.register_type(name="itl881.rebind.Cell") +@dataclasses.dataclass(frozen=True) +class CellV1: + """First definition under ``itl881.rebind.Cell``.""" + + x: int + + +@op.register_type(name="itl881.rebind.Cell") +@dataclasses.dataclass(frozen=True) +class CellV2: + """Second definition under ``itl881.rebind.Cell``, with the same fields.""" + + x: int + + +def test_nested_list_schema_names_the_rebound_class() -> None: + """A list of lists follows the rebind of its innermost class.""" + first = _source("grid", [[CellV1(1)]]) + assert first.output_schema()[1]["grid"] == list[list[CellV1]] + with pytest.warns(UserWarning, match="same identity"): + second = _source("grid", [[CellV2(1)]]) + assert second.output_schema()[1]["grid"] == list[list[CellV2]] + assert _read(second, "grid") == [[CellV2(1)]] + + +# --------------------------------------------------------------------------- +# An identity-equivalent class offered to a union parameter +# --------------------------------------------------------------------------- + + +@op.register_type(name="itl881.rebind.Reading") +@dataclasses.dataclass +class ReadingOld: + """First definition under ``itl881.rebind.Reading``.""" + + x: int + + +@op.register_type(name="itl881.rebind.Reading") +@dataclasses.dataclass +class ReadingNew: + """Second definition under ``itl881.rebind.Reading``, with the same fields.""" + + x: int + + +@function_pod(output_keys="p") +def make_old(n: int) -> ReadingOld: + """Return a ``ReadingOld`` holding ``n``.""" + return ReadingOld(n) + + +@function_pod(output_keys="y") +def read_optional(p: ReadingNew | None) -> int: + """Return the reading's ``x``, or ``-1`` for no reading.""" + return -1 if p is None else p.x + + +@function_pod(output_keys="y") +def read_int_or_str(p: int | str) -> int: + """Return ``0``; only the parameter type matters.""" + return 0 + + +def test_old_class_output_feeds_a_union_parameter_of_the_new_class() -> None: + """A pod output typed as the old class is accepted by ``New | None``.""" + source = DictSource(data=[{"k": "a", "n": 3}], tag_columns=["k"]) + out = read_optional.pod(make_old.pod(source)) + assert _read(out, "y") == 3 + + +def test_union_parameter_still_rejects_another_type() -> None: + """A union parameter with no member matching the column is still refused.""" + source = DictSource(data=[{"k": "a", "n": 3}], tag_columns=["k"]) + with pytest.raises(ValueError, match="column 'p' has type"): + read_int_or_str.pod(make_old.pod(source)) + + +# --------------------------------------------------------------------------- +# Storage that differs from the registered class +# --------------------------------------------------------------------------- + + +@op.register_type(name="itl881.rebind.Named") +@dataclasses.dataclass +class Named: + """Declared under ``itl881.rebind.Named``; has a string field.""" + + label: str + n: int + + +def _struct(label_type: pa.DataType) -> pa.DataType: + """Return ``Named``'s storage struct with ``label`` of ``label_type``.""" + return pa.struct([pa.field("label", label_type), pa.field("n", pa.int64())]) + + +def test_null_storage_is_not_the_registered_string_storage() -> None: + """A ``null`` field stored where the class has a string raises ``TypeIdentityError``.""" + ext = _source("p", Named("x", 1)).as_table().schema.field("p").type + metadata = ext.__arrow_ext_serialize__() + # Non-public entry point: pyarrow's extension hook. No public path produces this + # input (Delta refuses a null column; Parquet does not rebuild the extension type). + deserialize = type(ext).__arrow_ext_deserialize__ + # A layout variant of the same type is still accepted (Delta compaction). + assert deserialize(_struct(pa.string()), metadata).storage_type == ext.storage_type + with pytest.raises(TypeIdentityError, match="expected storage_type"): + deserialize(_struct(pa.null()), metadata) + + +# --------------------------------------------------------------------------- +# The child-process harness +# --------------------------------------------------------------------------- + + +def test_run_child_rejects_a_report_from_a_failed_child(tmp_path: Path) -> None: + """A child that reports and then exits non-zero is a failure, not a result.""" + code = "print('JSON:1', flush=True); raise SystemExit(3)" + with pytest.raises(AssertionError, match="exit 3"): + run_child(["-c", code], cwd=tmp_path) + ok = "print('JSON:1', flush=True)" + assert run_child(["-c", ok], cwd=tmp_path) == 1 diff --git a/tests/test_user_types/test_workshop_scenarios.py b/tests/test_user_types/test_workshop_scenarios.py index a159583f4..2efa1a34c 100644 --- a/tests/test_user_types/test_workshop_scenarios.py +++ b/tests/test_user_types/test_workshop_scenarios.py @@ -61,6 +61,7 @@ @dataclass class Point: + '''The scenario's ``Point`` class.''' x: int y: int """ @@ -106,19 +107,23 @@ def _script(workdir: Path, code: str, **env: str) -> Any: @dataclass class Base: + '''The scenario's ``Base`` class.''' x: int @dataclass class Child(Base): + '''The scenario's ``Child`` class.''' z: int = 0 class BaseModel(pydantic.BaseModel): + '''The scenario's ``BaseModel`` class.''' x: int class ChildModel(BaseModel): + '''The scenario's ``ChildModel`` class.''' z: int = 0 """ @@ -135,11 +140,13 @@ class ChildModel(BaseModel): def use_base(): + '''Return the Arrow extension name the base class is stored under.''' source = DictSource(data=[{"k": "a", "p": base(x=1)}], tag_columns=["k"]) return source.as_table().schema.field("p").type.extension_name def use_child(): + '''Run the job on the child class and return its result.''' return wsjob.run_job(child, rows=[{"x": 1, "z": 10}, {"x": 2, "z": 20}]) @@ -364,10 +371,12 @@ def test_w7_script_as_main_twice(workdir: Path) -> None: @dataclass class Point: + '''The scenario's ``Point`` class.''' x: int y: int def main(): + '''Run the job on ``Point`` and report its result.''' report(wsjob.run_job(Point)) if __name__ == "__main__": @@ -396,10 +405,12 @@ def test_w8_script_then_import_hits_with_declared_name(workdir: Path) -> None: @op.register_type(name="Point") @dataclass class Point: + '''The scenario's ``Point`` class.''' x: int y: int def main(): + '''Run the job on ``Point`` and report its result.''' report(wsjob.run_job(Point)) if __name__ == "__main__": @@ -429,6 +440,7 @@ def test_w11_notebook_to_module_with_declared_name(workdir: Path) -> None: @op.register_type(name="Point") @dataclass class Point: + '''The scenario's ``Point`` class.''' x: int y: int """, @@ -511,6 +523,7 @@ def test_read_with_declared_class_from_a_module(workdir: Path) -> None: @op.register_type(name="Point") @dataclass class Point: + '''The scenario's ``Point`` class.''' x: int y: int """ @@ -553,6 +566,7 @@ def test_failed_read_leaves_the_class_free_to_be_declared(workdir: Path) -> None @dataclass class Point: + '''The scenario's ``Point`` class.''' x: int y: int """, @@ -573,6 +587,7 @@ class Point: converter = get_default_context().type_converter def read(): + '''Read the stored result table and return its values as text.''' db = ExtensionAwareDatabase(DeltaTableDatabase(root), converter=converter) table = db.get_all_records(path) return [repr(r["q"]) for r in converter.arrow_table_to_python_dicts(table.select(["q"]))] @@ -603,11 +618,13 @@ def read(): @dataclass class Inner: + '''The scenario's ``Inner`` class.''' v: int @dataclass class Outer: + '''The scenario's ``Outer`` class.''' inner: Inner @@ -629,6 +646,7 @@ class Outer: converter = get_default_context().type_converter def read(): + '''Read the stored result table and return its values as text.''' db = ExtensionAwareDatabase(DeltaTableDatabase(root), converter=converter) table = db.get_all_records(path) return [repr(r["q"]) for r in converter.arrow_table_to_python_dicts(table.select(["q"]))] @@ -663,10 +681,12 @@ def test_failed_read_leaves_nested_classes_free_to_be_declared(workdir: Path) -> @dataclass class Inner: + '''The scenario's ``Inner`` class.''' v: int @dataclass class Outer: + '''The scenario's ``Outer`` class.''' inner: Inner """, },