diff --git a/.zed/rules b/.zed/rules index 66993d81f..c04e386d7 100644 --- a/.zed/rules +++ b/.zed/rules @@ -273,6 +273,37 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension never fall back to the storage type. List an older value in `also_reads` only when the current code decodes that layout. +## User classes: exact and structural + +A class a user defines is one of two kinds (developer, November 12, 2026, ITL-972; see +`docs/concepts/your-own-data-types.md`). + +- An ordinary `@dataclass` or a pydantic model is an **exact** class: named by module path + and qualified name, and one type only with itself. Never make two different exact classes + equivalent (no declared names, no equivalence by description). Python subclassing still + applies at a pod input: a subclass column is accepted at a base-class parameter, and the + subclass keeps its own identity and fields. +- A class made with `orcapod.dataclass` is **structural**: identified by its name (default + the class name) and its fields in declared order. Two structural classes with equal + identity records are compatible in the schema check and their instances compare equal. + Equality and hashing of a structural class cover exactly its stored fields; never let a + per-field `compare=` or `hash=` choice through, because a class rebuilt from metadata + cannot know it. Pydantic has no structural kind (ITL-973). +- **No automatic conversion.** Orcapod checks compatibility and passes a value as it is. A + function annotated with a structural class may receive another class of the same + identity; a structural class promises its fields, not its methods. `from_instance` is the + only conversion, and Orcapod never calls it. +- A value is stored only as its own class (`UserClassLogicalType.accepts_instance`); never + store a subclass instance or a look-alike object in another class's column. +- The identity record (`logical_types/user_class_identity.py`) is `{kind, name, fields}` + with JSON type tokens. The Arrow extension name is `name@` and the + record is the extension metadata; together they are the logical type identifier. A nested + class embeds its whole record and a logical-type field the whole identifier text, so a + structural class can be rebuilt from metadata alone. A change to the record or token + format moves every user-class identifier: treat it as a hash change. +- Arrow-backed data carries the identifier, not a class object: the registry binds an + identifier to the newest class object with that identity, and data is read as that class. + ## Golden hash values Hash values must stay identical across releases (developer, November 10, 2026). Every hash diff --git a/CLAUDE.md b/CLAUDE.md index c6697f850..8ce8050d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -284,6 +284,37 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension never fall back to the storage type. List an older value in `also_reads` only when the current code decodes that layout. +## User classes: exact and structural + +A class a user defines is one of two kinds (developer, November 12, 2026, ITL-972; see +`docs/concepts/your-own-data-types.md`). + +- An ordinary `@dataclass` or a pydantic model is an **exact** class: named by module path + and qualified name, and one type only with itself. Never make two different exact classes + equivalent (no declared names, no equivalence by description). Python subclassing still + applies at a pod input: a subclass column is accepted at a base-class parameter, and the + subclass keeps its own identity and fields. +- A class made with `orcapod.dataclass` is **structural**: identified by its name (default + the class name) and its fields in declared order. Two structural classes with equal + identity records are compatible in the schema check and their instances compare equal. + Equality and hashing of a structural class cover exactly its stored fields; never let a + per-field `compare=` or `hash=` choice through, because a class rebuilt from metadata + cannot know it. Pydantic has no structural kind (ITL-973). +- **No automatic conversion.** Orcapod checks compatibility and passes a value as it is. A + function annotated with a structural class may receive another class of the same + identity; a structural class promises its fields, not its methods. `from_instance` is the + only conversion, and Orcapod never calls it. +- A value is stored only as its own class (`UserClassLogicalType.accepts_instance`); never + store a subclass instance or a look-alike object in another class's column. +- The identity record (`logical_types/user_class_identity.py`) is `{kind, name, fields}` + with JSON type tokens. The Arrow extension name is `name@` and the + record is the extension metadata; together they are the logical type identifier. A nested + class embeds its whole record and a logical-type field the whole identifier text, so a + structural class can be rebuilt from metadata alone. A change to the record or token + format moves every user-class identifier: treat it as a hash change. +- Arrow-backed data carries the identifier, not a class object: the registry binds an + identifier to the newest class object with that identity, and data is read as that class. + ## Golden hash values Hash values must stay identical across releases (developer, November 10, 2026). Every hash diff --git a/changelog.d/ITL-972.md b/changelog.d/ITL-972.md new file mode 100644 index 000000000..115942fbc --- /dev/null +++ b/changelog.d/ITL-972.md @@ -0,0 +1,98 @@ +### Breaking changes and migration + +#### Your own classes come in two kinds: exact and structural (ITL-972) + +Orcapod 0.3 identifies a class you define by its kind, its name and its fields' names and +types, in declared order. The kind is chosen where the class is defined. + +- **An ordinary `@dataclass` or a pydantic model is identified by its exact class.** Its + name is the module path and class name (`pkg.models.Point`; `__main__.Point` for a class + defined in a notebook or script). Two different classes are never the same type, even + with the same name and fields: a pod built for a class refuses the class defined again + (a re-run notebook cell), and the error names the column. A column of a subclass is + accepted where its base class is expected, as before. +- **A class made with the new `@op.dataclass` is structural** (see "Added"): identified by + its name and fields, whatever class object or module it is. +- **Hashes change for every pipeline that uses a dataclass or pydantic model.** The + class's logical type name and Arrow extension name become + `@`, and the extension metadata holds that + description; the stored struct lists fields in declared order, as before. Every schema + hash, pipeline hash, table path, system-tag column name and data hash that involves such + a value moves. The first run under 0.3 computes those pod and operator results again, + into new tables. Tables written by earlier versions are left untouched and are not read + by 0.3 pipelines; reading such a column directly raises `TypeIdentityError`. There is no + migration tool. Results that involve no such value are unaffected. +- **Changing a class's fields makes a new type.** Adding, removing, renaming, retyping or + reordering fields all count. Defaults, methods and pydantic computed fields are not part + of the identity. The new type's pods get new tables; the results of the old definition + stay where they are. Before, a changed class shared the old tables and a run returned 0 + rows with only log warnings. +- **A value is stored only as its own class.** Storing a value in a column of an ordinary + dataclass or pydantic model requires an instance of exactly that class; a subclass + instance or an unrelated object with the same attributes raises `TypeError`. Before, + any object with the right attributes was stored, and a subclass instance lost its own + fields. This holds for a pod's output read in memory as well: its columns are + converted as the pod's declared output types, where the types were taken from the + values before. +- **A stored shape that disagrees with the class raises.** Reading a stored column of an + ordinary dataclass or pydantic model finds the class at its module path. If the class + there has other fields, `DeltaTableDatabase` raises the new + `orcapod.errors.TypeIdentityError` naming the column and both sets of fields. It is not + a `StoreReadError`: the table is intact and the class differs. Like a store error, it + fails the run under every `error_policy`, with both orchestrators: a stored row that + cannot be decoded is never skipped as a failed row. If no class is found, reading raises + `ImportError` naming the column, the class and its stored fields. +- **A subclass is its own type.** A dataclass or pydantic subclass was stored as its + parent when the parent was registered first: its extra fields were silently dropped from + stored data, and the pod's hash and table path depended on the registration order. It + now has its own identity and keeps all its fields, in any registration order. +- **Recursive classes** raise `RecursiveTypeError` (a `TypeError`) naming the field path, + for example `Node.next -> Node`. + +### Added + +#### `op.dataclass` — a dataclass identified by its name and fields (ITL-972) + +`@op.dataclass` is used like `dataclasses.dataclass` (bare, or with `name=` and the +dataclass options except `init=False`) and returns a real dataclass. Orcapod identifies it +by its name (default: the class name) and its fields, not by the class object or its +module: + +- Two such classes with the same name and fields are one type. A class cell run again in + a notebook, or a class moved from a notebook into a module, keeps its pods and its + stored results. +- A pod annotated with one class accepts a column of another class of the same type and + receives the value as it is. Orcapod never converts a value from one class to another: + a structural class promises its fields, not its methods. +- Instances of two classes of the same type compare equal when their field values are + equal. Equality and hashing always cover all stored fields: `field(compare=False)` and + `field(hash=...)` are refused with `TypeError` where the class is defined. +- Stored values can be read in a process that does not define the class: Orcapod rebuilds + a dataclass from the description stored with the column. +- `P.from_instance(other)` builds an instance from an instance of any other dataclass with + exactly the same field names and types. It is the only conversion, and it is explicit. +- Every class inside an `op.dataclass` must be an `op.dataclass`. + +See [Your Own Data Types](docs/concepts/your-own-data-types.md) for the two kinds and when +to use each. + +Re-running a notebook cell that defines a dataclass or pydantic model, or reloading its +module, no longer raises `ValueError: … already bound`. + +### Fixed + +#### A nested class stored as `None` reads back (ITL-881) + +A dataclass or pydantic model with an optional nested class (`child: Inner | None`) that +stored `None` raised `TypeError: 'NoneType' object is not subscriptable` when the value +was read back. It now reads back as `None`. + +#### Stored dataclass and pydantic values stay readable after Delta compaction (ITL-881) + +Delta compaction (`optimize.compact()`) rewrites a `large_string` struct field as +`string`. Reading a stored dataclass or pydantic column with a `str` or `bytes` field then +failed, and a cached run returned 0 rows. Extension columns now accept large and non-large +layouts of the same type at any nesting depth, as they already did at the top level +(ITL-602). Known limitation: such a non-large column read over Arrow IPC or the C stream +interface (not Parquet or Delta) gives invalid arrays, at any depth; Orcapod's own writers +never produce one (ITL-894). diff --git a/design_issues/ITL-972-UCK1.md b/design_issues/ITL-972-UCK1.md new file mode 100644 index 000000000..29b7ac343 --- /dev/null +++ b/design_issues/ITL-972-UCK1.md @@ -0,0 +1,57 @@ +## `src/orcapod/logical_types/` — identity of user classes (dataclasses, pydantic models) + +### UCK1 — A user class was identified by `module.qualname` only; then every class by name, kind and fields +**Status:** resolved +**Severity:** high +**Issue:** ITL-972 (supersedes the model of ITL-881, PR #292) + +In v0.2.0 a dataclass or pydantic model was identified by `f"{module}.{qualname}"`. Two +classes with the same name and different fields shared pipeline hashes and tables, so a +changed class read the old table, failed, and the run returned 0 rows with only log +warnings; re-running the notebook cell that defines a class raised +`ValueError: … already bound` until a kernel restart; a subclass was stored as its parent, +dropping its extra fields. + +PR #292 (ITL-881, not merged) identified every class by a declared name, kind and ordered +fields, so two different classes with the same description were interchangeable. A +measurement on November 12, 2026 showed the consequence: in memory a pod annotated with one +class received the other class's object unchanged, while through a store it received its +own class. + +**Fix:** two explicit kinds (developer, November 12, 2026; spec +`superpowers/specs/2026-11-12-itl-972-user-class-kinds.md`). + +- An ordinary dataclass or pydantic model is an **exact** class: named by module path and + qualified name, never equivalent to another class. Python subclassing still applies at a + pod input. A value is stored only as its own class. +- `orcapod.dataclass` makes a **structural** class: named by its declared name (default the + class name); two structural classes with equal identity records are compatible in the + schema check and their instances compare equal. Orcapod converts nothing: a value is + passed as it is. `from_instance` is the explicit conversion. +- Every kind is described by an identity record `{kind, name, fields}` with structured + type tokens; a nested class embeds its whole record and a logical-type field the whole + logical type identifier. The identity string `name@` is the logical + type name and Arrow extension name (Arrow's and Polars' registries are keyed by name), + and the record is the extension metadata, so a structural class is rebuilt from metadata + when no class is defined (`orcapod.synthesized`). +- Kept from PR #292: a subclass has its own identity; field order is identity; + `TypeIdentityError` is not a `StoreReadError` and fails the run under every error policy; + the registry binds an identifier to the newest class object with that identity. + Removed: `op.register_type`, declared names for exact classes, equivalence of exact + classes. + +Arrow-backed data carries the identifier, not a class object: after an exact class is +defined again and used, tables built before are read as the new class. + +Not covered: a structural pydantic model (ITL-973), computed fields, Enum/NamedTuple/ +TypedDict. + +**Known limitation (ITL-894):** reading a stored column whose storage differs from the +registered one only by layout (`string` vs `large_string`) is accepted at any depth, so +Delta-compacted tables stay readable. `__arrow_ext_deserialize__` then returns the +registered large storage type, which is correct for Parquet and Delta reads (the decoder +converts the data) but not over Arrow IPC or the C stream interface, where Arrow reads +32-bit offsets as 64-bit and the arrays are invalid. Top-level columns had this since +ITL-602; nested struct fields since this fix. Orcapod's own writers and Polars round trips +never produce non-large storage. Pinned by a strict xfail in +`tests/test_user_types/test_read_path.py`. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 6e3e4d4dd..c78c78ee6 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -453,3 +453,5 @@ primary user-facing APIs. - [Operators](operators.md) handle structural transforms (the complement of function pods) - Function pods participate in the [identity chain](identity.md) -- each pod's hash includes the function's identity and its upstream pipeline hashes +- Function pods can take and return dataclasses and pydantic models; see + [Your Own Data Types](your-own-data-types.md) for when two classes count as the same type diff --git a/docs/concepts/identity.md b/docs/concepts/identity.md index 6bf7a2b5a..f849252f0 100644 --- a/docs/concepts/identity.md +++ b/docs/concepts/identity.md @@ -171,6 +171,29 @@ print(step.pod(source).pipeline_hash() == original) When you change what a function computes, bump its major version (or rename it) so that stale results are not reused. +## Your own classes in the identity + +A class enters both chains through its **identity**: its kind, its name and the names and +types of its fields, in declared order. The identity is part of every schema hash, so it +reaches the pipeline hash of each source, pod and node that uses the class, and with it +the database table paths. Each stored value is hashed together with it, so it also +reaches the content hash. + +The name depends on the kind of class: + +- A class made with `@op.dataclass` is named by its class name (or the `name=` you give). + Two such classes with the same name and fields share results, wherever each is + defined. +- An ordinary `@dataclass` or a pydantic model is named by its module path and class + name. Moving it to another module makes a new type. + +In both kinds a class whose fields change or are reordered is a new type: its pods get +new pipeline hashes and new tables, and the results stored with the old definition stay +where they are. + +[Your Own Data Types](your-own-data-types.md) describes the two kinds and when to use +each. + ## How it connects to other concepts - [Sources](sources.md) form the base case of the Merkle chain (schema-only for pipeline hash, @@ -179,3 +202,5 @@ stale results are not reused. - [Operators](operators.md) contribute their own identity to the chain - [Function Pods](function-pods.md) run inside a `PipelineJob` use content hashes for memoization and pipeline hashes for DB scoping +- [Your Own Data Types](your-own-data-types.md) explains the identity of dataclasses and + pydantic models diff --git a/docs/concepts/your-own-data-types.md b/docs/concepts/your-own-data-types.md new file mode 100644 index 000000000..33444c868 --- /dev/null +++ b/docs/concepts/your-own-data-types.md @@ -0,0 +1,314 @@ +# Your Own Data Types + +Function pods can take and return your own classes: Python +[dataclasses](https://docs.python.org/3/library/dataclasses.html) and +[pydantic](https://docs.pydantic.dev/) models. Orcapod stores their values in Arrow +columns and caches results computed from them like any other value. + +Caching and type checking need to know when two classes are **the same type**. Orcapod +has two kinds of class, and you choose the kind where you define the class. + +## Two kinds of class + +| | `@op.dataclass` | `@dataclass`, pydantic model | +|---|---|---| +| Identified by | its **name** and its **fields** (names, types, order) | its **exact class**: module path, class name and fields | +| Two classes with the same name and fields | one type | two types | +| The class cell is run again in a notebook | still the same type; pods and stored results keep working | a new class; pods built for the earlier one refuse it | +| The class moves to another module | same type (the module is not part of it) | another type; results are computed again | +| Reading stored values without the class | works: the class is rebuilt from stored metadata | `ImportError` | +| What a pod is promised | the fields | the class itself, with its methods and validators | + +**Use `@op.dataclass`** for data you pass between pods, keep in a store and work on in +notebooks: records, measurements, parameters. It is the kind to use while a class is +still being edited. + +**Use an ordinary `@dataclass` or a pydantic model** when the pod needs that very class: +its methods, its validators, its place in a library. Orcapod then never accepts another +class in its place. + +In both kinds, adding, removing, renaming, retyping or reordering fields makes a **new +type**. Results computed with the old definition stay in the database, and the new type +computes its results into new tables, so a stored value is never read back with fields +it was not written with. Field order counts because your code can see it: a dataclass +compares, unpacks and prints its fields in declared order. + +Defaults, methods, properties, validators and docstrings are **not** part of a type, +although a pod can observe them. Changing one of them keeps the type and its stored +results. If such a change alters what a pod computes, bump the pod's version, as you +would for any change to its code. + +## `op.dataclass` in a notebook + +`op.dataclass` is used like `dataclasses.dataclass` and returns a real dataclass. Define +a class, use it in a pod and run a job: + +```python +import orcapod as op +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import DictSource + + +@op.dataclass +class Point: + x: float + y: float + + +calls = [] + + +@function_pod(output_keys="norm") +def norm(p: Point) -> float: + calls.append(p) # only here to show when the function really runs + return (p.x**2 + p.y**2) ** 0.5 + + +db = InMemoryArrowDatabase() + + +def run(point_class): + rows = [ + {"k": "a", "p": point_class(x=3.0, y=4.0)}, + {"k": "b", "p": point_class(x=6.0, y=8.0)}, + ] + job = PipelineJob(name="points", store=db) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="pts") + norm.pod(source, label="norm") + job.run() + return job.nodes["norm"] + + +print(run(Point).as_table().to_pandas()) +# k norm +# 0 a 5.0 +# 1 b 10.0 +print("function calls:", len(calls)) +# function calls: 2 +``` + +Running the cell that defines `Point` again creates a new class object with the same +name and fields. It is the same type: the pod defined earlier accepts it, and the next +run finds the stored results. + +```python +@op.dataclass +class Point: + x: float + y: float + + +run(Point) +print("function calls:", len(calls)) +# function calls: 2 +``` + +Changing the fields makes a new type. Re-run the pod cell too, so that `norm` takes the +new `Point`; the run then computes into new tables: + +```python +@op.dataclass +class Point: + x: float + y: float + label: str = "" + + +@function_pod(output_keys="norm") +def norm(p: Point) -> float: + calls.append(p) + return (p.x**2 + p.y**2) ** 0.5 + + +run(Point) +print("function calls:", len(calls)) +# function calls: 4 +``` + +### The name + +The type is named by the class name, `Point`, wherever the class is defined: in a +notebook, a script or any module. Pass `name=` to choose another name, for example to +keep two unrelated `Point` classes apart or to rename a class without losing its +results: + +```python +@op.dataclass(name="Point") +class LabelledPoint: + x: float + y: float + label: str = "" + + +run(LabelledPoint) +print("function calls:", len(calls)) +# function calls: 4 +``` + +A name must not be empty, contain `@`, `;` or spaces, or start with `orcapod.`. Two +classes with the same fields and **different** names are different types. + +`op.dataclass` also takes the options of `dataclasses.dataclass` (`frozen=True`, +`slots=True`, `kw_only=True`, `order=True`, ...), except `init=False`. Equality and +hashing always cover all stored fields, so that every class of a type compares alike: a +field with `field(compare=False)` or `field(hash=...)` is refused with `TypeError`. + +You can see a type on any column that holds it. The part before `@` is the name; the +rest is a hash of the name, the kind and the fields: + +```python +source = DictSource(data=[{"k": "a", "p": LabelledPoint(1.0, 2.0)}], tag_columns=["k"]) +print(source.as_table().schema.field("p").type.extension_name) +# Point@... +``` + +### A structural class promises its fields, not its methods + +When two classes are the same type, Orcapod checks that they are compatible and passes +the value **as it is**. It never converts a value from one class to another. A function +annotated with `Point` can therefore receive an instance of another class of the same +type: the object an upstream pod returned, an instance of the class as it was defined +before the cell was re-run, or a class rebuilt from stored metadata. + +What you can rely on: + +- **The fields.** Every class of the type has the same fields with the same types. +- **Equality.** Instances of two classes of the same type are equal when their field + values are equal. + +What you cannot rely on is a method or a `__post_init__` check of the class named in the +annotation. Keep behavior in functions that take the value, or convert explicitly. + +```python +print(LabelledPoint(1.0, 2.0) == Point(1.0, 2.0)) +# True +``` + +### Converting explicitly + +`from_instance` builds an instance of an `op.dataclass` from an instance of any other +dataclass, ordinary or `op.dataclass`, that has exactly the same field names with the +same types. It is the only conversion, and Orcapod never calls it for you. + +```python +from dataclasses import dataclass + + +@dataclass +class LegacyPoint: + x: float + y: float + label: str = "" + + +converted = Point.from_instance(LegacyPoint(1.0, 2.0, "origin")) +print(type(converted).__name__, converted.label) +# Point origin +``` + +A missing or an extra field raises `TypeError` rather than dropping or inventing data. + +## Ordinary dataclasses and pydantic models + +An ordinary `@dataclass` and a pydantic model are identified by their **exact class**: +the module path, the class name and the fields. + +- A class defined in a **module** is named by its module path: `pkg.models.Reading`. +- A class defined in a **notebook or a script** run directly is named under `__main__`: + `__main__.Reading`. + +Two different classes are never the same type, even with the same name and fields. In a +notebook this shows when a class cell is run again: the new class is a new class, and a +pod built for the earlier one refuses it. Re-run the pod cell as well. + +```python +@dataclass +class Reading: + value: float + + +@function_pod(output_keys="doubled") +def double(r: Reading) -> float: + return 2 * r.value + + +@dataclass +class Reading: # the cell is run again + value: float + + +try: + double.pod(DictSource(data=[{"k": "a", "r": Reading(1.0)}], tag_columns=["k"])) +except ValueError as error: + print("refused:", "column 'r'" in str(error)) +# refused: True +``` + +The stored results are not lost: a pod built for the new class finds them, because the +module path, the name and the fields are unchanged. + +Python's subclass rule holds: a pod annotated with a class accepts a column of a +subclass. The subclass remains its own type, with its own fields and tables. + +A column holds one class. To return a subclass from a pod, declare it: a pod annotated +`-> Child` produces a `Child` column with all of `Child`'s fields, and a downstream +parameter `p: Base` accepts that column. A pod annotated `-> Base` that returns a `Child` +instance is an error (`TypeError`) when its output is turned into a table or stored: the +value is refused, never stored as a `Base` without its own fields. + +A dataclass and a pydantic model are never the same type, and neither is ever the same +type as an `op.dataclass`. + +## Classes inside classes + +A field whose type is another class carries that class's whole description: its name, +its kind and its fields. Renaming the inner class or changing its fields therefore makes +the outer class a new type too. + +Every class inside an `op.dataclass` must itself be an `op.dataclass`, so that the whole +value can be rebuilt from stored metadata. A field typed by an ordinary dataclass or a +pydantic model raises `TypeError` naming the field. An ordinary dataclass or a pydantic +model may hold fields of any kind. + +A class that refers back to itself, directly (`next: Node | None`) or through other +classes, cannot be stored: Arrow has no recursive struct type. Orcapod raises +`RecursiveTypeError` naming the path, for example `Node.next -> Node`. + +## Reading stored values back + +**An `op.dataclass` column** is read with any `op.dataclass` of the stored name and +fields that is defined in the current process. If none is defined, Orcapod builds a +dataclass from the stored description (in the module `orcapod.synthesized`): it has the +stored fields, compares equal to your own class's instances, and converts with +`from_instance`, but it has none of your methods. + +**An ordinary dataclass or pydantic column** is read with the class found at the stored +module path (for a notebook or script class: in the running `__main__`). + +- If there is no class at that path, reading raises `ImportError` naming the column, the + class and its stored fields. +- If the class there has other fields or another field order, reading raises + `TypeIdentityError` (from `orcapod.errors`) naming the column and both sets of fields. + +Neither case is reported as missing data, and nothing stored is changed. + +## Rules and limits + +- An ordinary dataclass or pydantic model defined inside a function cannot be stored: it + has no module path to be found by. An `op.dataclass` may be defined anywhere. +- Each subclass is its own type with its own fields; it is never stored as its parent. A + value is stored only as its own class: a pod annotated `-> Base` that returns a `Child` + instance fails when the result is stored, instead of losing the child's fields. +- There is no structural pydantic model: a pydantic model is always identified by its + exact class. +- Results stored by Orcapod versions before 0.3 identified classes by module and class + name only. Version 0.3 does not read those columns; pipelines compute their results again + into new tables and leave the old ones untouched. + +## How it connects to other concepts + +- [Function Pods](function-pods.md): the pods that take and return these classes. +- [Identity & Hashing](identity.md): how a type's identity enters pipeline and content + hashes. diff --git a/mkdocs.yml b/mkdocs.yml index 0d34575c6..a152c656a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,6 +38,7 @@ nav: - Streams: concepts/streams.md - Operators: concepts/operators.md - Function Pods: concepts/function-pods.md + - Your Own Data Types: concepts/your-own-data-types.md - NumPy Arrays: concepts/arrays.md - Logical Type Identifiers: concepts/logical-type-identifiers.md - Identity & Hashing: concepts/identity.md diff --git a/src/orcapod/__init__.py b/src/orcapod/__init__.py index 994888ec1..6e73ba48a 100644 --- a/src/orcapod/__init__.py +++ b/src/orcapod/__init__.py @@ -61,6 +61,9 @@ # Extension registration API (ITL-473) from .extensions import OrcapodExtension, register_extension +# Your own data types: a dataclass identified by its name and fields (ITL-972) +from .structural import dataclass + # Post-run hook types (ITL-523) from .hooks import ( HookConfig, @@ -108,6 +111,8 @@ # Extension registration API (ITL-473) "OrcapodExtension", "register_extension", + # Your own data types (ITL-972) + "dataclass", # Post-run hook types (ITL-523) "HookConfig", "InvocationStatus", diff --git a/src/orcapod/contexts/logical_types.json b/src/orcapod/contexts/logical_types.json index 271578e66..4135c1daf 100644 --- a/src/orcapod/contexts/logical_types.json +++ b/src/orcapod/contexts/logical_types.json @@ -50,6 +50,13 @@ } ] }, + { + "factory": { + "_class": "orcapod.logical_types.structural_logical_type_factory.StructuralLogicalTypeFactory", + "_config": {} + }, + "category": "orcapod.structural" + }, { "factory": { "_class": "orcapod.logical_types.pydantic_logical_type_factory.PydanticLogicalTypeFactory", diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index de51791be..1580962fa 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -14,7 +14,7 @@ from orcapod.core.data_function import CachedDataFunction, PythonDataFunction from orcapod.core.streams.base import StreamBase from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.errors import StoreError +from orcapod.errors import StoreError, TypeIdentityError from orcapod.utils.store_errors import keep_store_error, note_callback_failure from orcapod.protocols.core_protocols import ( ArgumentGroup, @@ -29,6 +29,7 @@ ) from orcapod.protocols.database_protocols import ArrowDatabaseProtocol from orcapod.protocols.observability_protocols import DataExecutionLoggerProtocol +from orcapod.semantic_types import infer_python_schema_from_pylist_data from orcapod.system_constants import constants from orcapod.types import ( ColumnConfig, @@ -329,7 +330,8 @@ def add_post_run_hook(self, hook: PostRunHook) -> None: def _fire_post_run_hooks(self, payload: PostRunPayload) -> None: """Fire all registered hooks with payload in registration order. - When the invocation failed with a ``StoreError``, a fail-loud hook that + When the invocation failed with a ``StoreError`` or a ``TypeIdentityError`` + (both fail the run under every error policy), a fail-loud hook that raises does not replace it: its failure is logged and noted on the store error, the remaining hooks are skipped as for any fail-loud failure, and the caller re-raises the store error. @@ -338,7 +340,9 @@ def _fire_post_run_hooks(self, payload: PostRunPayload) -> None: payload: The post-run payload to pass to each hook. """ error = payload.stats.error - store_error = error if isinstance(error, StoreError) else None + store_error = ( + error if isinstance(error, (StoreError, TypeIdentityError)) else None + ) for hook in self._post_run_hooks: fn = hook.fn if isinstance(hook, HookConfig) else hook on_error = hook.on_error if isinstance(hook, HookConfig) else "raise" @@ -606,6 +610,12 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: with keep_store_error(exc, "Observer on_data_crash"): obs.on_data_crash(pod_label, tag, data, exc) raise + except TypeIdentityError as exc: + # A stored row that cannot be decoded as its class is never + # skipped as a row failure (ITL-881). + with keep_store_error(exc, "Observer on_data_crash"): + obs.on_data_crash(pod_label, tag, data, exc) + raise except Exception as exc: logger.debug( "Data processing failed, skipping: %s", exc, exc_info=True @@ -1040,7 +1050,16 @@ def as_table( # TODO: re-verify the implemetation of this conversion converter = self.data_context.type_converter - struct_data = converter.python_dicts_to_struct_dicts(all_data) + # The data columns have declared types: convert each value as its column's + # type, never as the type the value happens to have. A subclass instance + # returned for a base-class column is then refused instead of being stored + # as the base without its own fields (ITL-972). Only the remaining columns + # (meta, source info) are typed from their values. + python_schema = dict(infer_python_schema_from_pylist_data(all_data)) + python_schema.update(self.output_schema()[1]) + struct_data = converter.python_dicts_to_struct_dicts( + all_data, python_schema=python_schema + ) all_tags_as_tables: pa.Table = pa.Table.from_pylist( all_tags, schema=tag_schema ) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 826123f68..e6bd9ff2f 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -37,6 +37,7 @@ InputValidationError, PipelineJobRequiredError, StoreError, + TypeIdentityError, StoreReadError, StoreWriteError, ) @@ -1608,6 +1609,14 @@ def execute( with keep_store_error(exc, "Observer on_node_end"): ctx_obs.on_node_end(node_label, node_hash) raise + except TypeIdentityError as exc: + # A stored row that cannot be decoded as its class is never + # skipped as a row failure, whatever the error policy (ITL-881). + with keep_store_error(exc, "Observer on_data_crash"): + ctx_obs.on_data_crash(node_label, tag, data, exc) + with keep_store_error(exc, "Observer on_node_end"): + ctx_obs.on_node_end(node_label, node_hash) + raise except Exception as exc: logger.warning( "Data execution failed in %s: %s", diff --git a/src/orcapod/databases/delta_lake_databases.py b/src/orcapod/databases/delta_lake_databases.py index 32e6126d1..f80302f29 100644 --- a/src/orcapod/databases/delta_lake_databases.py +++ b/src/orcapod/databases/delta_lake_databases.py @@ -21,6 +21,7 @@ InputValidationError, StoreReadError, TableListingNotSupportedError, + TypeIdentityError, ) from orcapod.databases.storage_utils import ( delta_to_pyarrow_dataset, @@ -879,6 +880,9 @@ def _stored_ids(self, record_path: tuple[str, ...]) -> set[bytes]: if snapshot.ids is None: try: snapshot.ids = self._scan_ids(delta_table) + except TypeIdentityError: + # The table is intact; the registered class differs (ITL-881). + raise except Exception as e: raise self._read_failed(record_path) from e return snapshot.ids @@ -896,6 +900,9 @@ def _stored_rows(self, record_path: tuple[str, ...]) -> KeyIndex | None: if snapshot.rows is None: try: table, files, schema = self._scan_rows(delta_table) + except TypeIdentityError: + # The table is intact; the registered class differs (ITL-881). + raise except Exception as e: raise self._read_failed(record_path) from e snapshot.rows = KeyIndex(table) @@ -1129,6 +1136,10 @@ def _handle_delta_schema_compatibility( # Get Delta table schema and convert from arro3 to pyarrow arro3_schema = delta_table.schema().to_arrow() delta_schema = pa.schema(arro3_schema) # type: ignore + except TypeIdentityError: + # Stored and registered user-class shapes disagree: never report + # this as missing data (ITL-881). + raise except Exception as e: logger.warning(f"Could not get Delta table schema: {e}") return table @@ -1332,6 +1343,10 @@ def get_all_records( delta_table_data = delta_to_pyarrow_dataset( delta_table, as_large_types=True ).to_table() + except TypeIdentityError: + # Stored and registered user-class shapes disagree: the table is + # intact, so report the identity mismatch itself (ITL-881). + raise except Exception as e: raise self._read_failed(record_path) from e if delta_table_data.num_rows > 0: diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 71f21a7cd..94ada08a4 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -24,6 +24,26 @@ class SchemaInconsistencyError(InputValidationError): """ +class TypeIdentityError(ValueError): + """Raised when stored values and a registered user class disagree on identity. + + A stored dataclass or pydantic column records its class's identity (name, kind and + field types). This error is raised when the class found for that name has a + different identity or a different Arrow storage type, or when the column was + written by an Orcapod version that did not record the identity. The message names + the column where it is known, the class, and both shapes. + """ + + +class RecursiveTypeError(TypeError): + """Raised when a class refers back to itself through its fields. + + Arrow cannot store a recursive struct, so a dataclass or pydantic model whose + fields reach the class again (``next: Node | None``) cannot be stored. The message + names the field path, e.g. ``Node.next -> Node``. + """ + + class DuplicateTagError(ValueError): """Raised when duplicate tag values are found and skip_duplicates=False""" diff --git a/src/orcapod/logical_types/database_hooks.py b/src/orcapod/logical_types/database_hooks.py index ded2b8f17..f4f3798c5 100644 --- a/src/orcapod/logical_types/database_hooks.py +++ b/src/orcapod/logical_types/database_hooks.py @@ -24,6 +24,7 @@ import logging from typing import TYPE_CHECKING, Any, cast +from orcapod.errors import TypeIdentityError from orcapod.logical_types.identifier import LogicalTypeIdentifierError from orcapod.logical_types.identifier import require_readable from orcapod.logical_types.registry import LogicalTypeRegistry @@ -42,7 +43,7 @@ def register_discovered_logical_types( ) -> None: """Register any extension types found in ``schema`` that are not yet known. - Walks ``schema`` recursively via ``walk_schema`` to discover all Arrow extension + Walks each column of ``schema`` recursively via ``walk_field`` to discover all Arrow extension types at any nesting depth (both in-memory and field-metadata channels). For each discovered type, delegates to ``converter.register_logical_type_from_arrow_metadata``. @@ -61,6 +62,10 @@ def register_discovered_logical_types( error names the column. ValueError: Propagated from the converter if an extension type's metadata has no registered factory or is malformed. + TypeIdentityError: If a stored user-class column does not match the class + found for it; the message starts with the column name. + ImportError: If no class can be found for a stored user-class column; the + message starts with the column name. """ if converter is None: logger.debug( @@ -85,6 +90,9 @@ def register_discovered_logical_types( ) except LogicalTypeIdentifierError as exc: raise exc.for_column(field.name) from exc + except (TypeIdentityError, ImportError) as exc: + # The factory knows the type, not the column that holds it. + raise type(exc)(f"Column {field.name!r}: {exc}") from exc def apply_logical_types( diff --git a/src/orcapod/logical_types/dataclass_logical_type_factory.py b/src/orcapod/logical_types/dataclass_logical_type_factory.py index ac328847a..e26fd2998 100644 --- a/src/orcapod/logical_types/dataclass_logical_type_factory.py +++ b/src/orcapod/logical_types/dataclass_logical_type_factory.py @@ -1,70 +1,37 @@ """DataclassLogicalType and DataclassLogicalTypeFactory. -Provides the ``DataclassLogicalType`` logical type implementation and the -``DataclassLogicalTypeFactory`` that synthesises and reconstructs ``DataclassLogicalType`` -instances for Python dataclasses. - -Write path (``create_for_python_type``): - Iterates dataclass fields, delegates field Arrow-type resolution to the converter - via ``register_python_class``, and returns a ``DataclassLogicalType`` backed by - a ``pa.struct`` extension type. - -Read path (``reconstruct_from_arrow``): - Imports the dataclass by fully-qualified class name, resolves field annotations - against the (already bottom-up resolved) storage type, and returns a - ``DataclassLogicalType``. +An ordinary dataclass is an exact class: it is identified by its module path, qualified +name and fields, and is one type only with itself (ITL-972). The shared write and read +paths live in ``user_class_logical_type``. A class made by ``orcapod.dataclass`` is a +dataclass too; this factory hands it to ``StructuralLogicalTypeFactory``. Category tag: ``"orcapod.dataclass"`` """ from __future__ import annotations -import dataclasses -import json -import logging -from typing import TYPE_CHECKING, Any -from orcapod.logical_types.base_logical_type import BaseLogicalType -from orcapod.logical_types.registry import ( - make_arrow_extension_type, - make_polars_extension_type, +import orcapod.logical_types.user_class_identity as uci +from typing import TYPE_CHECKING + +from orcapod.logical_types.user_class_logical_type import ( + UserClassLogicalType, + UserClassLogicalTypeFactory, ) -from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: - import polars as pl - import pyarrow as pa from orcapod.logical_types.protocols import TypeConverterProtocol -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) #: Category tag embedded in Arrow extension metadata. Used as the factory dispatch key. DATACLASS_CATEGORY = "orcapod.dataclass" -class DataclassLogicalType(BaseLogicalType): - """Logical type binding a Python dataclass to its Arrow extension type representation. - - Stores the dataclass's fully-qualified class name as the Arrow extension name - and a ``pa.struct`` of the dataclass fields as the storage type. - - No Arrow-type reasoning lives here — all field-type resolution is owned by the - converter and completed before this object is constructed. - - Args: - logical_name: Fully-qualified class name (e.g. ``"mymodule.sub.MyData"``). - Used as both the logical type name and the Arrow extension name. - python_type: The Python dataclass ``type`` object. - storage_type: The Arrow ``pa.StructType`` for the dataclass fields. - field_annotations: Ordered list of ``(field_name, python_annotation)`` pairs - matching the fields in ``storage_type``. +class DataclassLogicalType(UserClassLogicalType): + """Logical type binding a dataclass to its Arrow extension type. Example: >>> lt = DataclassLogicalType( - ... "mymod.Point", Point, + ... "Point@76d961e4...", Point, ... pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]), ... [("x", int), ("y", int)], ... ) @@ -72,133 +39,13 @@ class DataclassLogicalType(BaseLogicalType): {"x": 1, "y": 2} """ - def __init__( - self, - logical_name: str, - python_type: type, - storage_type: pa.StructType, - field_annotations: list[tuple[str, Any]], - ) -> None: - self._logical_name = logical_name - self._python_type = python_type - self._storage_type = storage_type - self._field_annotations = field_annotations - - _metadata = json.dumps({"category": DATACLASS_CATEGORY}).encode("utf-8") - self._arrow_ext_class = make_arrow_extension_type( - logical_name, storage_type, metadata=_metadata - ) - self._arrow_ext: pa.ExtensionType | None = None - # ``storage_type`` must not contain nested extension types (ET1 in design_issues/). - # On the write path, ``DataclassLogicalTypeFactory.create_for_python_type`` strips any - # top-level extension type from each field's Arrow type before inserting it into the - # struct. On the read path, ``reconstruct_from_arrow`` receives a ``storage_type`` - # already guaranteed storage-safe by ``register_storage_type``. - self._polars_ext_class = make_polars_extension_type( - logical_name, - storage_type, - metadata=json.dumps({"category": DATACLASS_CATEGORY}), - ) - self._polars_ext: pl.BaseExtension | None = None - - @property - def logical_type_name(self) -> str: - """Fully-qualified class name used as the logical type identifier.""" - return self._logical_name - - @property - def python_type(self) -> type: - """The Python dataclass type this logical type represents.""" - return self._python_type - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for this dataclass. - - Returns: - A cached ``pa.ExtensionType`` instance with ``extension_name`` equal to - the fully-qualified class name and ``storage_type`` equal to the struct - of the dataclass fields. - """ - if self._arrow_ext is None: - self._arrow_ext = self._arrow_ext_class() - return self._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for this dataclass. - - Returns: - A cached ``pl.BaseExtension`` instance. - """ - if self._polars_ext is None: - self._polars_ext = self._polars_ext_class() - return self._polars_ext - - def python_to_storage( - self, value: Any, converter: TypeConverterProtocol | None - ) -> dict[str, Any]: - """Convert a dataclass instance to an Arrow-compatible struct dict. - - Iterates ``_field_annotations`` and delegates each field's conversion to - ``converter.python_to_storage``. - - Args: - value: A dataclass instance of type ``python_type``. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A dict mapping field names to their Arrow storage values. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "DataclassLogicalType.python_to_storage requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - return { - name: converter.python_to_storage(getattr(value, name), annotation) - for name, annotation in self._field_annotations - } - - def storage_to_python( - self, storage_value: Any, converter: TypeConverterProtocol | None - ) -> Any: - """Reconstruct a dataclass instance from an Arrow struct dict. - - Args: - storage_value: A dict mapping field names to Arrow storage values. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A dataclass instance of type ``python_type``. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "DataclassLogicalType.storage_to_python requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - kwargs = { - name: converter.storage_to_python(storage_value[name], annotation) - for name, annotation in self._field_annotations - } - return self._python_type(**kwargs) - + category = DATACLASS_CATEGORY -class DataclassLogicalTypeFactory: - """Stateless factory that synthesises and reconstructs ``DataclassLogicalType`` instances. - **Write path** (``create_for_python_type``): derives Arrow struct type from the - dataclass fields by delegating to ``converter.register_python_class`` per field. +class DataclassLogicalTypeFactory(UserClassLogicalTypeFactory): + """Factory for dataclass logical types. - **Read path** (``reconstruct_from_arrow``): imports the dataclass by FQCN, matches - fields against the already-resolved ``storage_type``, and returns a - ``DataclassLogicalType``. - - Category tag: ``"orcapod.dataclass"`` + Stored fields are the dataclass fields with ``init=True``. Register with:: @@ -207,168 +54,37 @@ class DataclassLogicalTypeFactory: category="orcapod.dataclass", python_bases=[object], ) - - Example: - >>> factory = DataclassLogicalTypeFactory() - >>> factory.supports_class(MyDataclass) - True - >>> factory.supports_class(str) - False """ - def supports_class(self, python_type: type) -> bool: - """Return True if ``python_type`` is a dataclass. - - Args: - python_type: Any Python type. + kind = uci.KIND_DATACLASS + category = DATACLASS_CATEGORY + logical_type_class = DataclassLogicalType - Returns: - True if ``dataclasses.is_dataclass(python_type)`` is True. - """ - return dataclasses.is_dataclass(python_type) and isinstance(python_type, type) + def supports_class(self, python_type: type) -> bool: + """Return True for every dataclass, ordinary or structural.""" + return uci.class_kind(python_type) in (uci.KIND_DATACLASS, uci.KIND_STRUCTURAL) def create_for_python_type( self, python_type: type, converter: TypeConverterProtocol, - ) -> DataclassLogicalType: - """Synthesise a ``DataclassLogicalType`` for a Python dataclass (write path). - - Derives the FQCN, obtains type hints, and resolves each field's Arrow type - via ``converter.register_python_class``. Rejects local / unnamed classes. + ) -> UserClassLogicalType: + """Build the logical type of a dataclass (write path). Args: - python_type: A Python dataclass type. + python_type: An ordinary dataclass or a structural class. converter: The active converter for field-type resolution. Returns: - A ``DataclassLogicalType`` ready for registration. - - Raises: - ValueError: If ``python_type`` is a local class (``__qualname__`` contains - ``""``). + A ``DataclassLogicalType``, or a ``StructuralLogicalType`` for a class made + by ``orcapod.dataclass``. """ - import typing - - fqcn = f"{python_type.__module__}.{python_type.__qualname__}" - if "" in fqcn: - raise ValueError( - f"Cannot register local class {python_type!r} as a DataclassLogicalType — " - f"local classes have no stable fully-qualified class name and cannot be " - f"reconstructed on read. Define the dataclass at module level." + if uci.is_structural(python_type): + from orcapod.logical_types.structural_logical_type_factory import ( + StructuralLogicalTypeFactory, ) - try: - hints = typing.get_type_hints(python_type) - except Exception as exc: - raise ValueError( - f"Cannot get type hints for {python_type!r}: {exc}" - ) from exc - - arrow_fields = [] - field_annotations = [] - for field in dataclasses.fields(python_type): - if not field.init: - continue - annotation = hints.get(field.name, Any) - arrow_type = converter.register_python_class(annotation) - # register_python_class returns a storage-safe type: may be extension at the - # top level, but struct fields are always plain. Strip the top-level extension - # type here before inserting into the struct (ET1; see design_issues/). - if isinstance(arrow_type, pa.ExtensionType): - arrow_type = arrow_type.storage_type - arrow_fields.append(pa.field(field.name, arrow_type)) - field_annotations.append((field.name, annotation)) - - storage_type = pa.struct(arrow_fields) - logger.debug( - "DataclassLogicalTypeFactory: synthesised %r for %r", fqcn, python_type - ) - return DataclassLogicalType(fqcn, python_type, storage_type, field_annotations) - - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: pa.DataType, - metadata: dict[str, Any], - converter: TypeConverterProtocol, - ) -> DataclassLogicalType: - """Reconstruct a ``DataclassLogicalType`` from Arrow schema metadata (read path). - - Imports the dataclass from its FQCN (``arrow_extension_name``), then matches - the dataclass field annotations against the fields in ``storage_type``. - ``storage_type`` is already bottom-up resolved by ``register_storage_type`` - before this method is called. - - Args: - arrow_extension_name: FQCN of the dataclass (Arrow extension name). - storage_type: Already-resolved ``pa.StructType`` for the dataclass fields. - metadata: Full parsed metadata JSON dict (always contains ``"category"``). - converter: The active converter (not needed here but required by protocol). - - Returns: - A ``DataclassLogicalType`` ready for registration. - - Raises: - ImportError: If the class cannot be imported from ``arrow_extension_name``. - ValueError: If ``storage_type`` is not a struct type. - """ - import typing - - if not pa.types.is_struct(storage_type): - raise ValueError( - f"DataclassLogicalTypeFactory.reconstruct_from_arrow: expected a struct " - f"storage type for {arrow_extension_name!r}, got {storage_type!r}." + return StructuralLogicalTypeFactory().create_for_python_type( + python_type, converter ) - - # Import class from FQCN using longest-prefix module walk - cls = _import_from_fqcn(arrow_extension_name) - - try: - hints = typing.get_type_hints(cls) - except Exception as exc: - raise ValueError(f"Cannot get type hints for {cls!r}: {exc}") from exc - - field_annotations = [] - for field in dataclasses.fields(cls): - if not field.init: - continue - annotation = hints.get(field.name, Any) - # Register any logical type the field annotation maps to (registration - # completeness invariant: all nested logical types must be registered when - # the outer type is registered). The return value is discarded; only the - # side effect of registration matters here. - converter.register_python_class(annotation) - field_annotations.append((field.name, annotation)) - - logger.debug( - "DataclassLogicalTypeFactory: reconstructed %r from Arrow", - arrow_extension_name, - ) - return DataclassLogicalType( - arrow_extension_name, cls, storage_type, field_annotations - ) - - -def _import_from_fqcn(fqcn: str) -> type: - """Import a dataclass from its fully-qualified class name. - - Delegates the module-prefix walk to ``type_utils._walk_fqcn``, then - validates the resolved object is a dataclass type. - - Args: - fqcn: Fully-qualified class name, e.g. ``"mypackage.sub.MyClass"``. - - Returns: - The imported dataclass type. - - Raises: - ImportError: If no valid module+attribute split can be found, or if the - resolved object is not a dataclass type. - """ - from orcapod.logical_types.type_utils import _walk_fqcn - - obj: Any = _walk_fqcn(fqcn) - if not dataclasses.is_dataclass(obj) or not isinstance(obj, type): - raise ImportError(f"{fqcn!r} does not resolve to a dataclass type.") - return obj + return super().create_for_python_type(python_type, converter) diff --git a/src/orcapod/logical_types/list_logical_type_factory.py b/src/orcapod/logical_types/list_logical_type_factory.py index 90ff2c660..2d229cfaa 100644 --- a/src/orcapod/logical_types/list_logical_type_factory.py +++ b/src/orcapod/logical_types/list_logical_type_factory.py @@ -168,6 +168,19 @@ def element_logical_type(self) -> LogicalTypeProtocol: """ return self._element_logical_type + def with_element( + self, element_logical_type: "LogicalTypeProtocol" + ) -> "ListLogicalType": + """Return this list or set logical type built over another element logical type. + + Args: + element_logical_type: The logical type of the elements. + + Returns: + A new ``ListLogicalType`` with the same list or set semantics. + """ + return ListLogicalType(element_logical_type, is_set=self._is_set) + @property def python_type(self) -> type: """The Python generic alias this logical type represents. diff --git a/src/orcapod/logical_types/pydantic_logical_type_factory.py b/src/orcapod/logical_types/pydantic_logical_type_factory.py index 85fa9e25e..dbd3bb43f 100644 --- a/src/orcapod/logical_types/pydantic_logical_type_factory.py +++ b/src/orcapod/logical_types/pydantic_logical_type_factory.py @@ -1,379 +1,51 @@ """PydanticLogicalType and PydanticLogicalTypeFactory. -Provides the ``PydanticLogicalType`` logical type implementation and the -``PydanticLogicalTypeFactory`` that synthesises and reconstructs -``PydanticLogicalType`` instances for pydantic v2 ``BaseModel`` subclasses. - -Write path (``create_for_python_type``): - Iterates model fields via ``model_fields`` (pydantic v2 API), delegates - field Arrow-type resolution to the converter via ``register_python_class``, - and returns a ``PydanticLogicalType`` backed by a ``pa.struct`` extension - type. - -Read path (``reconstruct_from_arrow``): - Imports the model by fully-qualified class name, resolves field annotations - against the (already bottom-up resolved) storage type, and returns a - ``PydanticLogicalType``. +Pydantic v2 models are user classes: their logical type name and Arrow extension name +are the class's identity string (name, kind ``"pydantic"`` and field types). A pydantic +model and a dataclass with the same name and fields are different types. The shared +write and read paths live in ``user_class_logical_type``. Category tag: ``"orcapod.pydantic"`` """ from __future__ import annotations -import json -import logging -from typing import TYPE_CHECKING, Any - -from orcapod.logical_types.base_logical_type import BaseLogicalType -from orcapod.logical_types.registry import ( - make_arrow_extension_type, - make_polars_extension_type, +import orcapod.logical_types.user_class_identity as uci +from orcapod.logical_types.user_class_logical_type import ( + UserClassLogicalType, + UserClassLogicalTypeFactory, ) -from orcapod.utils.lazy_module import LazyModule - -if TYPE_CHECKING: - import polars as pl - import pyarrow as pa - from orcapod.logical_types.protocols import TypeConverterProtocol -else: - pa = LazyModule("pyarrow") - pl = LazyModule("polars") - -logger = logging.getLogger(__name__) #: Category tag embedded in Arrow extension metadata. Used as the factory dispatch key. PYDANTIC_CATEGORY = "orcapod.pydantic" -class PydanticLogicalType(BaseLogicalType): +class PydanticLogicalType(UserClassLogicalType): """Logical type binding a pydantic ``BaseModel`` subclass to its Arrow extension type. - Stores the model's fully-qualified class name as the Arrow extension name - and a ``pa.struct`` of the model fields as the storage type. - - No Arrow-type reasoning lives here — all field-type resolution is owned by - the converter and completed before this object is constructed. - - Args: - logical_name: Fully-qualified class name (e.g. ``"mymodule.sub.MyModel"``). - Used as both the logical type name and the Arrow extension name. - python_type: The pydantic ``BaseModel`` subclass. - storage_type: The Arrow ``pa.StructType`` for the model fields. - field_annotations: Ordered list of ``(field_name, python_annotation)`` - pairs matching the fields in ``storage_type``. - - Example: - >>> lt = PydanticLogicalType( - ... "mymod.Point", Point, - ... pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]), - ... [("x", int), ("y", int)], - ... ) - >>> lt.python_to_storage(Point(x=1, y=2), converter) - {"x": 1, "y": 2} + ``storage_to_python`` constructs the model, so pydantic validation runs on read. """ - def __init__( - self, - logical_name: str, - python_type: type, - storage_type: pa.StructType, - field_annotations: list[tuple[str, Any]], - ) -> None: - self._logical_name = logical_name - self._python_type = python_type - self._storage_type = storage_type - self._field_annotations = field_annotations - - _metadata = json.dumps({"category": PYDANTIC_CATEGORY}).encode("utf-8") - self._arrow_ext_class = make_arrow_extension_type( - logical_name, storage_type, metadata=_metadata - ) - self._arrow_ext: pa.ExtensionType | None = None - # ``storage_type`` must not contain nested extension types (ET1 in design_issues/). - # On the write path, ``PydanticLogicalTypeFactory.create_for_python_type`` strips any - # top-level extension type from each field's Arrow type before inserting it into the - # struct. On the read path, ``reconstruct_from_arrow`` receives a ``storage_type`` - # already guaranteed storage-safe by ``register_storage_type``. - self._polars_ext_class = make_polars_extension_type( - logical_name, - storage_type, - metadata=json.dumps({"category": PYDANTIC_CATEGORY}), - ) - self._polars_ext: pl.BaseExtension | None = None - - @property - def logical_type_name(self) -> str: - """Fully-qualified class name used as the logical type identifier.""" - return self._logical_name - - @property - def python_type(self) -> type: - """The pydantic ``BaseModel`` subclass this logical type represents.""" - return self._python_type - - def get_arrow_extension_type(self) -> pa.ExtensionType: - """Return the Arrow extension type for this model. - - Returns: - A cached ``pa.ExtensionType`` instance with ``extension_name`` equal to - the fully-qualified class name and ``storage_type`` equal to the struct - of the model fields. - """ - if self._arrow_ext is None: - self._arrow_ext = self._arrow_ext_class() - return self._arrow_ext - - def get_polars_extension_type(self) -> pl.BaseExtension: - """Return the Polars extension type for this model. - - Returns: - A cached ``pl.BaseExtension`` instance. - """ - if self._polars_ext is None: - self._polars_ext = self._polars_ext_class() - return self._polars_ext + category = PYDANTIC_CATEGORY - def python_to_storage( - self, value: Any, converter: TypeConverterProtocol | None - ) -> dict[str, Any]: - """Convert a pydantic model instance to an Arrow-compatible struct dict. - Iterates ``_field_annotations`` and delegates each field's conversion to - ``converter.python_to_storage``. +class PydanticLogicalTypeFactory(UserClassLogicalTypeFactory): + """Factory for pydantic model logical types. - Args: - value: A pydantic model instance of type ``python_type``. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A dict mapping field names to their Arrow storage values. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "PydanticLogicalType.python_to_storage requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - return { - name: converter.python_to_storage(getattr(value, name), annotation) - for name, annotation in self._field_annotations - } - - def storage_to_python( - self, storage_value: Any, converter: TypeConverterProtocol | None - ) -> Any: - """Reconstruct a pydantic model instance from an Arrow struct dict. - - Args: - storage_value: A dict mapping field names to Arrow storage values. - converter: The active converter for per-field delegation. Must not be ``None``. - - Returns: - A pydantic model instance of type ``python_type``. Pydantic validation - runs on construction, ensuring the model is always in a valid state. - - Raises: - ValueError: If ``converter`` is ``None``. - """ - if converter is None: - raise ValueError( - "PydanticLogicalType.storage_to_python requires a converter — " - "pass a TypeConverterProtocol instance for field-level conversion." - ) - kwargs = { - name: converter.storage_to_python(storage_value[name], annotation) - for name, annotation in self._field_annotations - } - return self._python_type(**kwargs) - - -class PydanticLogicalTypeFactory: - """Stateless factory that synthesises and reconstructs ``PydanticLogicalType`` instances. - - **Write path** (``create_for_python_type``): derives Arrow struct type from the - model fields by delegating to ``converter.register_python_class`` per field. - Only fields in ``model_fields`` are stored — computed fields and private - attributes are excluded. - - **Read path** (``reconstruct_from_arrow``): imports the model by FQCN, matches - fields against the already-resolved ``storage_type``, and returns a - ``PydanticLogicalType``. - - Category tag: ``"orcapod.pydantic"`` + Stored fields are the model's ``model_fields``; computed fields and private + attributes are not stored. Register with:: from pydantic import BaseModel + converter.register_logical_type_factory( PydanticLogicalTypeFactory(), category="orcapod.pydantic", python_bases=[BaseModel], ) - - Example: - >>> factory = PydanticLogicalTypeFactory() - >>> factory.supports_class(MyModel) - True - >>> factory.supports_class(str) - False """ - def supports_class(self, python_type: type) -> bool: - """Return True if ``python_type`` is a pydantic ``BaseModel`` subclass. - - Args: - python_type: Any Python type. - - Returns: - True if ``python_type`` is a ``BaseModel`` subclass. - """ - from pydantic import BaseModel - - return isinstance(python_type, type) and issubclass(python_type, BaseModel) - - def create_for_python_type( - self, - python_type: type, - converter: TypeConverterProtocol, - ) -> PydanticLogicalType: - """Synthesise a ``PydanticLogicalType`` for a pydantic model (write path). - - Derives the FQCN, obtains type hints, and resolves each field's Arrow type - via ``converter.register_python_class``. Only fields present in - ``model_fields`` are stored — computed fields and private attributes are - excluded. Rejects local / unnamed classes. - - Args: - python_type: A pydantic ``BaseModel`` subclass. - converter: The active converter for field-type resolution. - - Returns: - A ``PydanticLogicalType`` ready for registration. - - Raises: - ValueError: If ``python_type`` is a local class (``__qualname__`` contains - ``""``). - """ - import typing - - fqcn = f"{python_type.__module__}.{python_type.__qualname__}" - if "" in fqcn: - raise ValueError( - f"Cannot register local class {python_type!r} as a PydanticLogicalType — " - f"local classes have no stable fully-qualified class name and cannot be " - f"reconstructed on read. Define the model at module level." - ) - - try: - hints = typing.get_type_hints(python_type) - except Exception as exc: - raise ValueError( - f"Cannot get type hints for {python_type!r}: {exc}" - ) from exc - - arrow_fields = [] - field_annotations = [] - for field_name in python_type.model_fields: - annotation = hints.get(field_name, Any) - arrow_type = converter.register_python_class(annotation) - # Strip top-level extension type before inserting into the struct (ET1; - # see design_issues/): Arrow cannot represent extension types inside - # struct field types. - if isinstance(arrow_type, pa.ExtensionType): - arrow_type = arrow_type.storage_type - arrow_fields.append(pa.field(field_name, arrow_type)) - field_annotations.append((field_name, annotation)) - - storage_type = pa.struct(arrow_fields) - logger.debug( - "PydanticLogicalTypeFactory: synthesised %r for %r", fqcn, python_type - ) - return PydanticLogicalType(fqcn, python_type, storage_type, field_annotations) - - def reconstruct_from_arrow( - self, - arrow_extension_name: str, - storage_type: pa.DataType, - metadata: dict[str, Any], - converter: TypeConverterProtocol, - ) -> PydanticLogicalType: - """Reconstruct a ``PydanticLogicalType`` from Arrow schema metadata (read path). - - Imports the model from its FQCN (``arrow_extension_name``), then matches - the model field annotations against the fields in ``storage_type``. - ``storage_type`` is already bottom-up resolved by ``register_storage_type`` - before this method is called. - - Args: - arrow_extension_name: FQCN of the pydantic model (Arrow extension name). - storage_type: Already-resolved ``pa.StructType`` for the model fields. - metadata: Full parsed metadata JSON dict (always contains ``"category"``). - converter: The active converter (used for registration completeness invariant). - - Returns: - A ``PydanticLogicalType`` ready for registration. - - Raises: - ImportError: If the class cannot be imported from ``arrow_extension_name``. - ValueError: If ``storage_type`` is not a struct type. - """ - import typing - - if not pa.types.is_struct(storage_type): - raise ValueError( - f"PydanticLogicalTypeFactory.reconstruct_from_arrow: expected a struct " - f"storage type for {arrow_extension_name!r}, got {storage_type!r}." - ) - - cls = _import_pydantic_model_from_fqcn(arrow_extension_name) - - try: - hints = typing.get_type_hints(cls) - except Exception as exc: - raise ValueError(f"Cannot get type hints for {cls!r}: {exc}") from exc - - field_annotations = [] - for field_name in cls.model_fields: - annotation = hints.get(field_name, Any) - # Register any logical type the field annotation maps to (registration - # completeness invariant: all nested logical types must be registered when - # the outer type is registered). The return value is discarded. - converter.register_python_class(annotation) - field_annotations.append((field_name, annotation)) - - logger.debug( - "PydanticLogicalTypeFactory: reconstructed %r from Arrow", - arrow_extension_name, - ) - return PydanticLogicalType( - arrow_extension_name, cls, storage_type, field_annotations - ) - - -def _import_pydantic_model_from_fqcn(fqcn: str) -> type: - """Import a pydantic ``BaseModel`` subclass from its fully-qualified class name. - - Delegates the module-prefix walk to ``type_utils._walk_fqcn``, then - validates the resolved object is a ``BaseModel`` subclass. - - Args: - fqcn: Fully-qualified class name, e.g. ``"mypackage.sub.MyModel"``. - - Returns: - The imported ``BaseModel`` subclass. - - Raises: - ImportError: If no valid module+attribute split can be found, or if the - resolved object is not a ``BaseModel`` subclass. - """ - from pydantic import BaseModel - from orcapod.logical_types.type_utils import _walk_fqcn - - obj: Any = _walk_fqcn(fqcn) - if not (isinstance(obj, type) and issubclass(obj, BaseModel)): - raise ImportError( - f"{fqcn!r} does not resolve to a pydantic BaseModel subclass." - ) - return obj + kind = uci.KIND_PYDANTIC + category = PYDANTIC_CATEGORY + logical_type_class = PydanticLogicalType diff --git a/src/orcapod/logical_types/registry.py b/src/orcapod/logical_types/registry.py index 00d480360..9233ea36b 100644 --- a/src/orcapod/logical_types/registry.py +++ b/src/orcapod/logical_types/registry.py @@ -37,27 +37,6 @@ def _sanitize(name: str) -> str: return re.sub(r"[^A-Za-z0-9]", "_", name) -def _canonical_storage(t: pa.DataType) -> pa.DataType: - """Return the canonical large-variant for string/binary families; identity elsewhere. - - Maps: - ``string`` / ``string_view`` → ``large_string`` - ``binary`` / ``binary_view`` → ``large_binary`` - ``large_string`` / ``large_binary`` → unchanged (already canonical) - anything else → unchanged - - Used in ``__arrow_ext_deserialize__`` to accept physically-equivalent but - layout-different string/binary variants that Delta Lake or other compaction - tools may write (e.g. delta-rs ``optimize.compact()`` normalises - ``large_string`` columns to plain ``string`` in the rewritten Parquet files). - """ - if pa.types.is_string(t) or pa.types.is_string_view(t): - return pa.large_string() - if pa.types.is_binary(t) or pa.types.is_binary_view(t): - return pa.large_binary() - return t - - def make_arrow_extension_type( extension_name: str, storage_type: pa.DataType, @@ -132,14 +111,22 @@ def _deserialize( # # Cross-family mismatches (string where large_binary was registered, or # vice-versa) are genuine semantic errors and always raise. - if _canonical_storage(_storage) == _storage: - # _storage is the canonical large form — accept any same-family variant. - storage_ok = _canonical_storage(storage_type) == _storage + from orcapod.utils.arrow_utils import normalize_to_large_types + + if normalize_to_large_types(_storage, keep_null=True) == _storage: + # _storage is the canonical large form — accept any same-family variant, + # at any nesting depth (a struct field rewritten from large_string to + # string by Delta compaction, ITL-881). + storage_ok = ( + normalize_to_large_types(storage_type, keep_null=True) == _storage + ) else: # Non-canonical _storage — require exact match. storage_ok = storage_type == _storage + from orcapod.errors import TypeIdentityError + if not storage_ok: - raise ValueError( + raise TypeIdentityError( f"Arrow extension type '{_name}': expected storage_type " f"{_storage!r} but got {storage_type!r}." ) @@ -229,6 +216,15 @@ def ext_from_params( ) +def _same_arrow_extension(a: pa.ExtensionType, b: pa.ExtensionType) -> bool: + """Return True if two extension types have the same name, storage and metadata.""" + return ( + a.extension_name == b.extension_name + and a.storage_type == b.storage_type + and a.__arrow_ext_serialize__() == b.__arrow_ext_serialize__() + ) + + class LogicalTypeRegistry: """Registry for ``LogicalType`` instances. @@ -311,6 +307,19 @@ def register_logical_type(self, logical_type: LogicalTypeProtocol) -> None: existing_by_arrow = self._by_arrow_name.get(arrow_ext_name) existing_by_python = self._by_python_type.get(py_type) + if ( + existing_by_python is None + and existing_by_logical is not None + and existing_by_logical is existing_by_arrow + and existing_by_logical is not logical_type + and getattr(logical_type, "exact_python_type_only", False) + and _same_arrow_extension( + existing_by_logical.get_arrow_extension_type(), arrow_ext + ) + ): + self._rebind(existing_by_logical, logical_type) + return + # Triplet conflict check: raise if any key is bound to a different instance. for existing, label, key in [ (existing_by_logical, "logical_type_name", logical_name), @@ -361,6 +370,56 @@ def register_logical_type(self, logical_type: LogicalTypeProtocol) -> None: self._by_arrow_name[arrow_ext_name] = logical_type self._by_python_type[py_type] = logical_type + def _rebind( + self, previous: LogicalTypeProtocol, logical_type: LogicalTypeProtocol + ) -> None: + """Rebind an identity to a logical type built for a new class object. + + Used when a user class with an unchanged identity is defined again (a notebook + cell re-run, a module reload, a second structural class with the same name and + fields). It makes the newest class the one stored values are read as; it does + not make two exact classes one type. The logical-name and Arrow-name bindings move to + the new logical type; the previous class object keeps its binding, so instances + created before the re-definition still convert. Arrow's and Polars' global + registrations are reused: name, storage type and metadata are identical. + + Args: + previous: The logical type currently bound to the identity. + logical_type: The logical type for the new class object. + """ + self._move_bindings(previous, logical_type) + + def _move_bindings( + self, previous: LogicalTypeProtocol, logical_type: LogicalTypeProtocol + ) -> None: + """Bind an identity's names to ``logical_type`` and rebuild its containers. + + The logical-name and Arrow-name bindings move to ``logical_type`` and its + Python type is bound to it; the previous Python type keeps its binding. Each + registered list or set logical type whose element is ``previous`` is rebuilt + over ``logical_type`` and rebound the same way, at any nesting depth, so + ``list[Point]`` names the class ``Point`` is now bound to. + + Args: + previous: The logical type the names were bound to. + logical_type: The logical type to bind them to, with the same names. + """ + containers = [ + registered + for registered in self._by_logical_name.values() + if getattr(registered, "element_logical_type", None) is previous + ] + name = logical_type.logical_type_name + self._by_logical_name[name] = logical_type + self._by_arrow_name[logical_type.get_arrow_extension_type().extension_name] = ( + logical_type + ) + self._by_python_type[logical_type.python_type] = logical_type + for container in containers: + with_element = getattr(container, "with_element", None) + if with_element is not None: + self._move_bindings(container, with_element(logical_type)) + def get_by_logical_name(self, name: str) -> LogicalTypeProtocol | None: """Return the logical type registered under *name*, or ``None``.""" return self._by_logical_name.get(name) @@ -371,7 +430,9 @@ def get_by_python_type(self, python_type: type) -> LogicalTypeProtocol | None: The most specific registered type wins: *python_type* itself, then each class of its MRO in order, so a subclass registered with its own logical type is used for itself and its subclasses whatever the registration - order (an ``np.ndarray`` subclass without one is an array). Types that are + order (an ``np.ndarray`` subclass without one is an array). A logical type + with ``exact_python_type_only`` (a dataclass or pydantic model) is never + returned for a subclass. Types that are subclasses only through ``__subclasshook__``/ABC registration fall back to an ``issubclass`` scan, where the first registered match wins. """ @@ -380,9 +441,15 @@ class of its MRO in order, so a subclass registered with its own logical return result for base in getattr(python_type, "__mro__", ())[1:]: result = self._by_python_type.get(base) - if result is not None: + # A user class matches exactly: its subclass is a different class with + # its own fields and identity, never stored as the base (ITL-881). + if result is not None and not getattr( + result, "exact_python_type_only", False + ): return result for registered_type, registered_lt in self._by_python_type.items(): + if getattr(registered_lt, "exact_python_type_only", False): + continue try: if issubclass(python_type, registered_type): return registered_lt diff --git a/src/orcapod/logical_types/structural_class.py b/src/orcapod/logical_types/structural_class.py new file mode 100644 index 000000000..aef7c8531 --- /dev/null +++ b/src/orcapod/logical_types/structural_class.py @@ -0,0 +1,479 @@ +"""Structural classes: rebuilding from stored metadata, equality and conversion (ITL-972). + +A structural class is made by ``orcapod.dataclass``. This module holds what such a class +needs beyond its identity (``user_class_identity``): turning a stored identity record +back into a class when none is defined, the ``__eq__`` that compares across classes of +one type, and ``from_instance``, the explicit conversion. +""" + +from __future__ import annotations + +import ast +import dataclasses +import functools +import operator +import re +import types +import typing +from typing import TYPE_CHECKING, Any, Literal + +import orcapod.logical_types.user_class_identity as uci + +if TYPE_CHECKING: + from orcapod.logical_types.protocols import TypeConverterProtocol + +_UNIONS = (typing.Union, types.UnionType) + +# Structural classes built from stored metadata, by identity string. +_synthesized: dict[str, type] = {} + +_SCALARS: dict[str, Any] = { + "int": int, + "float": float, + "str": str, + "bool": bool, + "bytes": bytes, + "None": type(None), +} +_GENERIC_ORIGINS: dict[str, Any] = { + "list": list, + "set": set, + "frozenset": frozenset, + "tuple": tuple, + "dict": dict, +} + + +# --------------------------------------------------------------------------- +# Rebuilding annotations and structural classes from tokens +# --------------------------------------------------------------------------- + + +def annotation_from_token(token: Any, converter: TypeConverterProtocol) -> Any: + """Return the Python annotation a type token describes. + + Args: + token: A type token, as ``type_token`` builds it. + converter: The type converter of the reading context. + + Returns: + The annotation. + + Raises: + ImportError: If a class named by the token cannot be imported, or a logical + type named by it is not registered in the reading context. + ValueError: If the token is malformed, or an exact class it names is defined + with other fields than the token records. + """ + from orcapod.logical_types.type_utils import _walk_fqcn + + if isinstance(token, str): + if token in _SCALARS: + return _SCALARS[token] + if token == uci.ELLIPSIS_TOKEN: + return Ellipsis + return _walk_fqcn(token) + if not isinstance(token, dict): + raise ValueError(f"Malformed type token: {token!r}.") + if "logical" in token: + name = token["logical"].split(";", 1)[0] + lt = converter.get_logical_type_by_arrow_extension_name(name) + if lt is None: + raise ImportError( + f"The logical type {name!r} is not registered in this data context." + ) + from orcapod.logical_types.identifier import logical_type_identifier + + if logical_type_identifier(lt).text != token["logical"]: + raise ValueError( + f"The logical type {name!r} registered in this data context has the " + f"identifier {logical_type_identifier(lt).text!r}, not the stored " + f"{token['logical']!r}." + ) + return lt.python_type + if "generic" in token: + origin = _GENERIC_ORIGINS.get(token["generic"]) or _walk_fqcn(token["generic"]) + args = tuple(annotation_from_token(arg, converter) for arg in token["args"]) + return origin[args] if args else origin + if "union" in token: + members = [ + annotation_from_token(member, converter) for member in token["union"] + ] + return functools.reduce(operator.or_, members) + if "literal" in token: + return Literal[tuple(ast.literal_eval(text) for text in token["literal"])] # pyright: ignore[reportInvalidTypeForm] # built at run time from stored values + if "class" in token: + return class_for_record(token["class"], converter) + raise ValueError(f"Malformed type token: {token!r}.") + + +def class_for_record(record: dict[str, Any], converter: TypeConverterProtocol) -> type: + """Return the class a stored identity record describes. + + A structural record gives the most recently defined structural class with that + record, or a class synthesized from the record when none is defined. An exact + record gives the class its name refers to, which must have the stored record. + + Args: + record: A stored identity record. + converter: The type converter of the reading context. + + Returns: + The class. + + Raises: + ImportError: If an exact class cannot be found by its name. + ValueError: If the record is malformed, or the exact class found is described + by another record. + """ + kind, name = record.get("kind"), record.get("name") + if not isinstance(name, str) or kind not in (*uci.EXACT_KINDS, uci.KIND_STRUCTURAL): + raise ValueError(f"Malformed identity record: {record!r}.") + if kind == uci.KIND_STRUCTURAL: + if not isinstance(record.get("fields"), list): + raise ValueError(f"Malformed identity record: {record!r}.") + for cls in uci.structural_classes(name): + try: + if uci.class_record(cls, converter) == record: + return cls + except (TypeError, ValueError): + continue + return synthesize_structural_class(record, converter) + cls = uci.find_exact_class(name) + if uci.class_kind(cls) != kind: + raise ImportError( + f"Cannot find the {kind} {name!r} to read stored values of type " + f"{uci.describe_record(record)}: no such class is defined at that module path." + ) + current = uci.class_record(cls, converter) + if current != record: + raise ValueError( + f"Stored values of {uci.describe_record(record)} do not match the class " + f"defined now: {uci.describe_record(current)}." + ) + return cls + + +def synthesize_structural_class( + record: dict[str, Any], converter: TypeConverterProtocol +) -> type: + """Build a structural class from a stored record when no class is defined for it. + + The class is a dataclass with the stored fields, named after the last component of + the stored name, in the module ``orcapod.synthesized``. It is built once per + identity in a process. + + Args: + record: A stored structural identity record. + converter: The type converter of the reading context. + + Returns: + A structural class whose record equals ``record``. + + Raises: + ValueError: If the record cannot be turned into a class with the same record. + """ + identity = uci.identity_string(record) + cached = _synthesized.get(identity) + if cached is not None: + return cached + name = record["name"] + python_name = re.sub(r"\W", "_", name.rsplit(".", 1)[-1]) + if not python_name.isidentifier(): + python_name = f"_{python_name}" + fields = [ + (field, annotation_from_token(token, converter)) + for field, token in record["fields"] + ] + # Hashable (by field values, as ``==`` compares): the class may have to rebuild a + # ``set`` of its instances, and the stored record does not say whether the class + # that wrote them was frozen. + cls = dataclasses.make_dataclass(python_name, fields, unsafe_hash=True) + cls.__module__ = uci.SYNTHESIZED_MODULE + cls.__doc__ = f"Structural class {name!r}, built from stored metadata." + finish_structural_class(cls, name) + if uci.class_record(cls, converter) != record: + raise ValueError( + f"Cannot rebuild the structural class {uci.describe_record(record)} from its " + f"stored record." + ) + _synthesized[identity] = cls + return cls + + +def finish_structural_class( + cls: type, name: str, *, structural_equality: bool = True +) -> None: + """Turn a dataclass that was created for this purpose into a structural class. + + Marks the class with its name and adds ``from_instance`` and, unless the class + opted out of generated equality, the ``__eq__`` that compares across classes of + one type. Used by ``orcapod.dataclass`` and for classes built from stored metadata. + + Args: + cls: The dataclass just created. + name: The validated name of the type. + structural_equality: Whether to install ``structural_eq`` as ``__eq__``. + """ + uci.make_structural(cls, name) + if structural_equality: + setattr(cls, "__eq__", structural_eq) + setattr(cls, "from_instance", classmethod(_from_instance)) + + +# --------------------------------------------------------------------------- +# Equality across classes of one type +# --------------------------------------------------------------------------- + +_MISSING = object() + + +def structural_eq(self: Any, other: Any) -> Any: + """Compare two instances of structural classes of the same type by field values. + + Installed as ``__eq__`` by ``orcapod.dataclass``. Instances of the same class + and instances of two structural classes with the same name and fields (a class + defined again, a class built from stored metadata) are equal when all their stored + fields are equal. Anything else is not comparable. + + Args: + self: An instance of a structural class. + other: Any object. + + Returns: + ``True`` or ``False`` for an instance of a class of the same type, else + ``NotImplemented``. + """ + mine, theirs = self.__class__, other.__class__ + if theirs is not mine and not ( + uci.is_structural(theirs) and uci.same_user_type(mine, theirs) + ): + return NotImplemented + # Every stored field, and only those: ``orcapod.dataclass`` refuses a per-field + # ``compare=`` or ``hash=`` choice, so every class of the type compares alike. + names = uci.stored_field_names(mine) + return tuple(getattr(self, name) for name in names) == tuple( + getattr(other, name, _MISSING) for name in names + ) + + +# --------------------------------------------------------------------------- +# Conversion: an instance of the class from an instance of another dataclass +# --------------------------------------------------------------------------- + + +def _from_instance(cls: type, other: Any) -> Any: + """Build an instance of this class from an instance of another dataclass. + + This is the only conversion between classes, and Orcapod never calls it for you. + + Args: + other: An instance of an ordinary dataclass or an ``orcapod.dataclass`` with + exactly the same field names and the same declared field types. + + Returns: + A new instance of this class with ``other``'s field values. Fields typed by + an ``orcapod.dataclass`` are converted the same way. + + Raises: + TypeError: If ``other`` is not a dataclass instance, or its fields differ + from this class's fields. + """ + return cls(**converted_fields(cls, other)) + + +def converted_fields(cls: type, other: Any) -> dict[str, Any]: + """Return the keyword arguments that build a ``cls`` from ``other``. + + This is the acceptance rule of ``from_instance``: ``other`` is a dataclass instance whose stored fields have exactly the names of + ``cls``'s fields, and whose declared field types agree with ``cls``'s. + + Args: + cls: A structural class. + other: The instance to convert. + + Returns: + Field name to value, values of structural field types converted. + + Raises: + TypeError: If ``other`` is not a dataclass instance, a field is missing or + extra, or a field's declared type differs. + """ + source = type(other) + if not dataclasses.is_dataclass(other) or isinstance(other, type): + raise TypeError( + f"{cls.__qualname__} can be built from an instance of a dataclass, not " + f"from {other!r}." + ) + wanted = uci.stored_field_names(cls) + offered = uci.stored_field_names(source) + if set(wanted) != set(offered): + missing = sorted(set(wanted) - set(offered)) + extra = sorted(set(offered) - set(wanted)) + raise TypeError( + f"Cannot build {cls.__qualname__} from an instance of " + f"{source.__qualname__}: the fields differ (missing: {missing}, not in " + f"{cls.__qualname__}: {extra}). The two classes must have the same fields." + ) + wanted_hints = uci.type_hints(cls) + offered_hints = uci.type_hints(source) + values: dict[str, Any] = {} + for field in wanted: + target = wanted_hints.get(field, Any) + if not _convertible(offered_hints.get(field, Any), target): + raise TypeError( + f"Cannot build {cls.__qualname__} from an instance of " + f"{source.__qualname__}: field {field!r} is declared " + f"{offered_hints.get(field, Any)!r} there and {target!r} here." + ) + values[field] = _converted_value(getattr(other, field), target) + return values + + +def _convertible(offered: Any, wanted: Any) -> bool: + """Return True if a field declared ``offered`` can fill a field declared ``wanted``. + + The declarations are equal, or differ only where ``wanted`` names a structural + class and ``offered`` a dataclass (converted by ``from_instance``, which checks + its fields in turn). Generics are compared by origin and arguments, union members + in any order. + """ + if offered == wanted: + return True + if isinstance(offered, type) and isinstance(wanted, type): + return uci.is_structural(wanted) and dataclasses.is_dataclass(offered) + origin_offered, origin_wanted = ( + typing.get_origin(offered), + typing.get_origin(wanted), + ) + if origin_offered is None or origin_wanted is None: + return False + args_offered, args_wanted = typing.get_args(offered), typing.get_args(wanted) + if origin_offered in _UNIONS and origin_wanted in _UNIONS: + return all( + any(_convertible(x, y) for y in args_wanted) for x in args_offered + ) and all(any(_convertible(x, y) for x in args_offered) for y in args_wanted) + if origin_offered != origin_wanted or len(args_offered) != len(args_wanted): + return False + return all(_convertible(x, y) for x, y in zip(args_offered, args_wanted)) + + +def _converted_union_value(value: Any, members: tuple[Any, ...]) -> Any: + """Return a field value converted to the union member it belongs to. + + A value that already is an instance of a member class is kept. Otherwise a + dataclass instance goes to the first structural member it can be converted to + (the same field names with the same declared types), and a container to the first + member with the container's origin whose conversion succeeds. A value no member + claims (an ``int`` in ``int | P``) is returned as it is. + + Args: + value: The field value, not ``None``. + members: The members of the union annotation. + + Returns: + The value to store in the new instance. + + Raises: + TypeError: If members claim the value (structural members for a dataclass + instance, container members of the value's kind) and it can be converted + to none of them. + """ + cls = type(value) + classes = [ + m for m in members if isinstance(m, type) and typing.get_origin(m) is None + ] + if cls in classes: + return value + if dataclasses.is_dataclass(value): + structural = [m for m in classes if uci.is_structural(m)] + for member in structural: + if _fields_agree(cls, member): + return _converted_value(value, member) + if structural: + names = ", ".join(m.__qualname__ for m in structural) + raise TypeError( + f"An instance of {cls.__qualname__} cannot be converted to any member " + f"of the union ({names}): no member has the same fields with the " + f"same types." + ) + return value + failure: TypeError | None = None + for member in members: + if typing.get_origin(member) is cls: + try: + return _converted_value(value, member) + except TypeError as exc: + failure = exc + if failure is not None: + raise TypeError( + f"A {cls.__name__} value cannot be converted to any member of the union " + f"that is a {cls.__name__}: {failure}" + ) from failure + return value + + +def _fields_agree(source: type, target: type) -> bool: + """Return True if an instance of ``source`` can be converted to ``target``. + + The two dataclasses have the same stored field names, and each field's declared + types agree (``_convertible``): the acceptance rule of ``from_instance``. + """ + wanted = uci.stored_field_names(target) + if set(wanted) != set(uci.stored_field_names(source)): + return False + wanted_hints = uci.type_hints(target) + offered_hints = uci.type_hints(source) + return all( + _convertible(offered_hints.get(field, Any), wanted_hints.get(field, Any)) + for field in wanted + ) + + +def _mentions_structural(annotation: Any) -> bool: + """Return True if an annotation names a structural class at any depth.""" + if isinstance(annotation, type) and typing.get_origin(annotation) is None: + return uci.is_structural(annotation) + return any(_mentions_structural(arg) for arg in typing.get_args(annotation)) + + +def _converted_value(value: Any, annotation: Any) -> Any: + """Return a field value converted to the structural classes its annotation names. + + Used by ``from_instance`` for the fields of the instance it builds. Where the + annotation names a structural class and the value is an instance of another + dataclass, the value is replaced by ``from_instance`` of the annotated class. + Lists, sets, tuples and dict values are rebuilt only when their annotation names a + structural class; every other value is returned as it is. + + Args: + value: The field value of the instance being converted. + annotation: The annotation of the field receiving it. + + Returns: + The value to store in the new instance. + + Raises: + TypeError: If a value cannot be converted to the annotated structural class. + """ + if value is None or not _mentions_structural(annotation): + return value + origin = typing.get_origin(annotation) + args = typing.get_args(annotation) + if origin is None: + if type(value) is annotation: + return value + return annotation.from_instance(value) + if origin in _UNIONS: + return _converted_union_value(value, args) + if origin in (list, set, frozenset) and len(args) == 1: + return origin(_converted_value(item, args[0]) for item in value) + if origin is tuple: + if len(args) == 2 and args[1] is Ellipsis: + return tuple(_converted_value(item, args[0]) for item in value) + if len(args) == len(value): + return tuple(_converted_value(item, arg) for item, arg in zip(value, args)) + return value + if origin is dict and len(args) == 2: + return {key: _converted_value(item, args[1]) for key, item in value.items()} + return value diff --git a/src/orcapod/logical_types/structural_logical_type_factory.py b/src/orcapod/logical_types/structural_logical_type_factory.py new file mode 100644 index 000000000..6f4ed19f6 --- /dev/null +++ b/src/orcapod/logical_types/structural_logical_type_factory.py @@ -0,0 +1,74 @@ +"""StructuralLogicalType and StructuralLogicalTypeFactory (ITL-972). + +A structural class is made by ``orcapod.dataclass``. It is identified by its declared +name and its fields in declared order, not by the class object: two structural classes +with the same name and fields are one type, and a stored column can be read with no +class defined, because the Arrow extension metadata holds the whole identity record. + +Category tag: ``"orcapod.structural"`` +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import orcapod.logical_types.user_class_identity as uci +from orcapod.logical_types.user_class_logical_type import ( + UserClassLogicalType, + UserClassLogicalTypeFactory, +) + +if TYPE_CHECKING: + from orcapod.logical_types.protocols import TypeConverterProtocol + +#: Category tag embedded in Arrow extension metadata. Used as the factory dispatch key. +STRUCTURAL_CATEGORY = "orcapod.structural" + + +class StructuralLogicalType(UserClassLogicalType): + """Logical type binding a structural class to its Arrow extension type.""" + + category = STRUCTURAL_CATEGORY + + instance_requirement = ( + "a structural column stores instances of an orcapod.dataclass with the same " + "name and fields. Build one explicitly, for example " + ".from_instance(value)." + ) + + def accepts_instance(self, value: Any, converter: TypeConverterProtocol) -> bool: + """Return True if ``value`` is an instance of a structural class of this type. + + Args: + value: The value offered. + converter: The active converter. + + Returns: + Whether the value's class is this type's class, or another structural + class with the same identity record. + """ + cls = type(value) + if cls is self._python_type: + return True + if not uci.is_structural(cls): + return False + return uci.class_record(cls, converter) == self._identity_record + + +class StructuralLogicalTypeFactory(UserClassLogicalTypeFactory): + """Factory for structural class logical types. + + On the write path a structural class is a dataclass, so the dataclass factory, + which is registered for every class, hands it to this factory. On the read path + the registry dispatches on the category. + + Register with:: + + converter.register_logical_type_factory( + StructuralLogicalTypeFactory(), category="orcapod.structural" + ) + """ + + kind = uci.KIND_STRUCTURAL + category = STRUCTURAL_CATEGORY + logical_type_class = StructuralLogicalType diff --git a/src/orcapod/logical_types/user_class_identity.py b/src/orcapod/logical_types/user_class_identity.py new file mode 100644 index 000000000..c18a9a0b8 --- /dev/null +++ b/src/orcapod/logical_types/user_class_identity.py @@ -0,0 +1,628 @@ +"""Identity of user classes (dataclasses and pydantic models). + +Orcapod knows three kinds of user class (ITL-972): + +- ``"dataclass"``: an ordinary dataclass. An **exact** class: two classes are one type + only when they are the same class object. Named by module path and qualified name. +- ``"pydantic"``: a ``pydantic.BaseModel`` subclass. Exact as well. +- ``"structural"``: a class made by ``orcapod.dataclass``. Named by its declared name + (default: the class name); two structural classes with the same name and the same + fields in the same order are one type, whatever class object they are. + +A class is described by an *identity record*, a JSON object with ``kind``, ``name`` and +``fields`` (``[name, type token]`` pairs in declared order; order is part of the +identity). A type token is JSON as well and describes a field type completely: a class +typed by another user class embeds that class's whole record, so a structural class can +be rebuilt from its record with no class importable. + +The *identity string* ``f"{name}@{sha256 of the record}"`` is the class's logical type +name and Arrow extension name; ``identity_string`` is the only place that defines its +format. Arrow's and Polars' extension registries are keyed by name, so the digest keeps +two shapes of one name apart within a process. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import re +import sys +import typing +import unicodedata +import weakref +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Literal + +from orcapod.errors import RecursiveTypeError +from orcapod.utils.canonical_type_names import ( + canonical_module_name, + canonical_type_name, +) + +if TYPE_CHECKING: + from orcapod.logical_types.protocols import TypeConverterProtocol + +KIND_DATACLASS = "dataclass" +KIND_PYDANTIC = "pydantic" +KIND_STRUCTURAL = "structural" + +#: Kinds identified by the class object: two classes are one type only if they are +#: the same object. +EXACT_KINDS: frozenset[str] = frozenset({KIND_DATACLASS, KIND_PYDANTIC}) + +#: Modules whose classes are "defined in a notebook or script". ``__mp_main__`` is the +#: name a ``multiprocessing`` spawn child gives the main module. +MAIN_MODULES: tuple[str, ...] = ("__main__", "__mp_main__") + +#: Prefix reserved for Orcapod's own types (``orcapod.path``, ``orcapod.uuid``, ...). +RESERVED_PREFIX = "orcapod." + +#: Number of hex digits of the SHA-256 fingerprint in an identity string (the full digest). +FINGERPRINT_HEX_DIGITS = 64 + +#: Type token of ``...`` in ``tuple[T, ...]``. +ELLIPSIS_TOKEN = "..." + +#: Class attribute that holds the declared name of a structural class. It is looked up +#: in the class's own namespace, so a subclass is structural only if it is decorated too. +STRUCTURAL_NAME_ATTRIBUTE = "__orcapod_structural_name__" + +#: Module name given to structural classes built from stored metadata. +SYNTHESIZED_MODULE = "orcapod.synthesized" + +_NAME_FORBIDDEN = re.compile(r"[@;\s]") + +# The identity record each class object was last described with. +_record_of: weakref.WeakKeyDictionary[type, dict[str, Any]] = ( + weakref.WeakKeyDictionary() +) +# Structural classes defined in this process under each name, most recent last. +_structural_classes: dict[str, list[weakref.ref[type]]] = {} +# Resolved type hints per class. +_hints_of: weakref.WeakKeyDictionary[type, dict[str, Any]] = weakref.WeakKeyDictionary() + + +# --------------------------------------------------------------------------- +# Kinds and names +# --------------------------------------------------------------------------- + + +def structural_name(cls: Any) -> str | None: + """Return the declared name of a structural class, or ``None`` for any other object. + + Args: + cls: Any object. + + Returns: + The name given by ``orcapod.dataclass`` to this very class. + """ + if not isinstance(cls, type): + return None + return vars(cls).get(STRUCTURAL_NAME_ATTRIBUTE) + + +def class_kind(cls: Any) -> str | None: + """Return the kind of a user class, or ``None`` if it is not one. + + Args: + cls: Any object. + + Returns: + ``"structural"`` for a class made by ``orcapod.dataclass``, ``"dataclass"`` for + any other dataclass type (including pydantic dataclasses), ``"pydantic"`` for a + ``pydantic.BaseModel`` subclass, else ``None``. + """ + if not isinstance(cls, type): + return None + if dataclasses.is_dataclass(cls): + if structural_name(cls) is not None: + return KIND_STRUCTURAL + return KIND_DATACLASS + from pydantic import BaseModel + + if issubclass(cls, BaseModel): + return KIND_PYDANTIC + return None + + +def is_structural(cls: Any) -> bool: + """Return True if ``cls`` is a class made by ``orcapod.dataclass``.""" + return class_kind(cls) == KIND_STRUCTURAL + + +def stored_field_names(cls: type) -> list[str]: + """Return the names of the fields stored for a user class, in declared order. + + Args: + cls: A dataclass or pydantic model class. + + Returns: + The ``init=True`` fields of a dataclass, or the ``model_fields`` of a pydantic + model. Computed fields and private attributes are not stored. + + Raises: + TypeError: If ``cls`` is not a dataclass or pydantic model class. + """ + kind = class_kind(cls) + if kind in (KIND_DATACLASS, KIND_STRUCTURAL): + return [f.name for f in dataclasses.fields(cls) if f.init] + if kind == KIND_PYDANTIC: + return list(cls.model_fields) # pyright: ignore[reportAttributeAccessIssue] # class_kind guarantees a BaseModel subclass + raise TypeError(f"{cls!r} is not a dataclass or pydantic model class.") + + +def type_hints(cls: type) -> dict[str, Any]: + """Return the resolved type hints of a class. + + Args: + cls: A class. + + Returns: + Field name to annotation, with ``Annotated`` extras stripped. + + Raises: + ValueError: If the hints cannot be resolved. + """ + cached = _hints_of.get(cls) + if cached is not None: + return cached + try: + hints = typing.get_type_hints(cls) + except Exception as exc: + raise ValueError(f"Cannot get type hints for {cls!r}: {exc}") from exc + _hints_of[cls] = hints + return hints + + +def validate_name(name: str) -> str: + """Check a declared name of a structural class and return its normal form. + + Args: + name: The proposed name. + + Returns: + The name, NFC-normalized, so one name typed in two Unicode forms is one name. + + Raises: + TypeError: If ``name`` is not a string. + ValueError: If ``name`` is empty, contains ``@``, ``;`` or whitespace, or starts + with the reserved ``orcapod.`` prefix. + """ + if not isinstance(name, str): + raise TypeError(f"A type name must be a string, got {name!r}.") + if not name: + raise ValueError("A type name must not be empty.") + if _NAME_FORBIDDEN.search(name): + raise ValueError( + f"Invalid type name {name!r}: a name must not contain '@', ';' or " + f"whitespace." + ) + if name.startswith(RESERVED_PREFIX): + raise ValueError( + f"Invalid type name {name!r}: the prefix {RESERVED_PREFIX!r} is reserved " + f"for Orcapod's own types." + ) + return unicodedata.normalize("NFC", name) + + +def exact_class_name(cls: type) -> str: + """Return the name of an exact class: its module path and qualified name. + + A class defined in a notebook or script is named ``__main__.``; a + ``multiprocessing`` spawn child calls that module ``__mp_main__``, which is written + ``__main__`` so that the child finds the same type. + + Args: + cls: An ordinary dataclass or a pydantic model class. + + Returns: + ``f"{module}.{qualname}"``. + """ + if cls.__module__ in MAIN_MODULES: + return f"{MAIN_MODULES[0]}.{cls.__qualname__}" + module = canonical_module_name(cls) or cls.__module__ + return f"{module}.{cls.__qualname__}" + + +def class_name(cls: type) -> str: + """Return the name component of a user class's identity. + + Args: + cls: A user class. + + Returns: + The declared name of a structural class, else ``exact_class_name(cls)``. + """ + declared = structural_name(cls) + if declared is not None: + return declared + return exact_class_name(cls) + + +def class_path(cls: type) -> str: + """Return ``module.qualname`` of a class as Python reports it, for messages.""" + return f"{cls.__module__}.{cls.__qualname__}" + + +def reject_local_class(cls: type) -> None: + """Raise if an exact class is defined inside a function. + + Args: + cls: A class. + + Raises: + ValueError: If ``cls.__qualname__`` contains ````. + """ + if "" in cls.__qualname__: + raise ValueError( + f"Cannot store local class {cls!r}: a class defined inside a function " + f"cannot be found again by its module path when stored values are read. " + f"Define the class at module level, or make it an orcapod.dataclass, " + f"which is identified by its name and fields." + ) + + +# --------------------------------------------------------------------------- +# Structural classes known in this process +# --------------------------------------------------------------------------- + + +def remember_structural(name: str, cls: type) -> None: + """Record ``cls`` as a structural class defined under ``name`` in this process. + + Args: + name: The declared name. + cls: The class object. + """ + refs = _structural_classes.setdefault(name, []) + refs[:] = [r for r in refs if r() is not None and r() is not cls] + refs.append(weakref.ref(cls)) + + +def structural_classes(name: str) -> list[type]: + """Return the live structural classes defined under ``name``, most recent first.""" + classes = [r() for r in _structural_classes.get(name, [])] + return [c for c in reversed(classes) if c is not None] + + +# --------------------------------------------------------------------------- +# Identity record and string +# --------------------------------------------------------------------------- + + +def identity_record( + kind: str, name: str, fields: Iterable[tuple[str, Any]] +) -> dict[str, Any]: + """Build the identity record of a user class. + + Args: + kind: ``"dataclass"``, ``"pydantic"`` or ``"structural"``. + name: The name component. + fields: ``(stored field name, type token)`` pairs in declared order. + + Returns: + ``{"kind": kind, "name": name, "fields": [[field, token], ...]}``. The fields + keep the given order, which is part of the identity. + """ + return { + "kind": kind, + "name": name, + "fields": [[field, token] for field, token in fields], + } + + +def canonical_json(obj: Any) -> str: + """Serialize ``obj`` as canonical JSON (sorted object keys, no whitespace). + + Arrays keep their order, so the ``fields`` of an identity record stay in declared + order. + """ + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def identity_string(record: dict[str, Any]) -> str: + """Return the identity string of an identity record. + + The format is ``f"{name}@{fingerprint}"``, where the fingerprint is the first + ``FINGERPRINT_HEX_DIGITS`` hex digits of the SHA-256 of ``canonical_json(record)``. + This is the only definition of the format. + + Args: + record: An identity record from ``identity_record``. + + Returns: + The identity string, e.g. ``"Point@76d961e4..."``. + """ + digest = hashlib.sha256(canonical_json(record).encode("utf-8")).hexdigest() + return f"{record['name']}@{digest[:FINGERPRINT_HEX_DIGITS]}" + + +def describe_token(token: Any) -> str: + """Render a type token for messages, e.g. ``list[int]`` or ``int | None``.""" + if isinstance(token, str): + return token + if "logical" in token: + return token["logical"].split(";", 1)[0] + if "generic" in token: + inner = ", ".join(describe_token(arg) for arg in token["args"]) + return f"{token['generic']}[{inner}]" + if "union" in token: + return " | ".join(describe_token(member) for member in token["union"]) + if "literal" in token: + return f"Literal[{', '.join(token['literal'])}]" + if "class" in token: + return str(token["class"].get("name")) + return canonical_json(token) + + +def describe_record(record: dict[str, Any]) -> str: + """Render an identity record for messages, e.g. ``Point (dataclass) {x: int}``.""" + text = f"{record.get('name')} ({record.get('kind')})" + fields = record.get("fields") + if fields is None: + return text + inner = ", ".join(f"{name}: {describe_token(token)}" for name, token in fields) + return f"{text} {{{inner}}}" + + +# --------------------------------------------------------------------------- +# Describing a class: record and type tokens +# --------------------------------------------------------------------------- + + +def class_record( + cls: type, converter: TypeConverterProtocol, _path: tuple[type, ...] = () +) -> dict[str, Any]: + """Return the identity record of a user class. + + The record depends only on the class definition and on the logical types the + converter knows for its field types; no user class is registered. + + Args: + cls: A dataclass or pydantic model class. + converter: The type converter that resolves the logical types of field types + that are not user classes. + + Returns: + The identity record. + + Raises: + TypeError: If ``cls`` is not a user class, a field type has no type token, or + a structural class has a field typed by an exact class. + RecursiveTypeError: If the class refers back to itself. + ValueError: If an exact class is local or type hints cannot be resolved. + """ + kind = class_kind(cls) + if kind is None: + raise TypeError(f"{cls!r} is not a dataclass or pydantic model class.") + if cls in _path: + raise RecursiveTypeError( + " -> ".join(c.__qualname__ for c in (*_path[_path.index(cls) :], cls)) + ) + if kind in EXACT_KINDS: + reject_local_class(cls) + hints = type_hints(cls) + path = (*_path, cls) + fields: list[tuple[str, Any]] = [] + for name in stored_field_names(cls): + token = _type_token(hints.get(name, Any), converter, path) + if kind == KIND_STRUCTURAL: + exact = _first_exact_class(token) + if exact is not None: + raise TypeError( + f"Field {name!r} of the orcapod.dataclass {class_name(cls)!r} is " + f"typed by {describe_record(exact)}, which is identified by its " + f"exact class. Every class inside an orcapod.dataclass must be an " + f"orcapod.dataclass, so that it can be rebuilt from stored " + f"metadata alone." + ) + fields.append((name, token)) + record = identity_record(kind, class_name(cls), fields) + _record_of[cls] = record + return record + + +def known_record(cls: type) -> dict[str, Any] | None: + """Return the record ``cls`` was last described with in this process, or ``None``.""" + return _record_of.get(cls) + + +def _first_exact_class(token: Any) -> dict[str, Any] | None: + """Return the record of the first exact class a token mentions, or ``None``.""" + if isinstance(token, str): + return None + if "class" in token: + record = token["class"] + if record["kind"] in EXACT_KINDS: + return record + for _, inner in record.get("fields", []): + found = _first_exact_class(inner) + if found is not None: + return found + return None + for members in (token.get("args"), token.get("union")): + for member in members or []: + found = _first_exact_class(member) + if found is not None: + return found + return None + + +def type_token(annotation: Any, converter: TypeConverterProtocol) -> Any: + """Return the type token of a field annotation. + + A token is JSON. Builtin scalars and classes without a logical type are strings + (``"int"``, ``"datetime.datetime"``); a type with a logical type is + ``{"logical": }``, the Arrow extension name and metadata; a + generic is ``{"generic": , "args": [...]}``; a union is + ``{"union": [...]}`` with members sorted and ``"None"`` last; a ``Literal`` is + ``{"literal": [...]}``; another user class is ``{"class": }``. The + ``...`` of ``tuple[T, ...]`` is the string ``"..."``. + + Args: + annotation: A field annotation (``Annotated`` extras already stripped). + converter: The type converter that resolves logical types. + + Returns: + The type token. + + Raises: + TypeError: If the annotation is not a type the token rules cover. + """ + return _type_token(annotation, converter, ()) + + +def _type_token( + annotation: Any, converter: TypeConverterProtocol, path: tuple[type, ...] +) -> Any: + """Return a type token; ``path`` holds the user classes being described.""" + import types as _types + + from orcapod.logical_types.identifier import logical_type_identifier + + if annotation is None or annotation is type(None): + return "None" + if annotation is Ellipsis: + # ``tuple[T, ...]``: the marker of a tuple of any length. + return ELLIPSIS_TOKEN + origin = typing.get_origin(annotation) + args = typing.get_args(annotation) + if origin is typing.Union or origin is _types.UnionType: + members = [_type_token(a, converter, path) for a in args] + non_none = sorted((m for m in members if m != "None"), key=canonical_json) + return {"union": non_none + (["None"] if "None" in members else [])} + if origin is Literal: + return {"literal": sorted(repr(a) for a in args)} + if isinstance(annotation, type) and origin is None: + if class_kind(annotation) is not None: + return {"class": class_record(annotation, converter, path)} + _ = converter.register_python_class(annotation) # raises for unsupported types + lt = converter.get_logical_type_for_python_type(annotation) + if lt is not None: + return {"logical": logical_type_identifier(lt).text} + return _class_token(annotation) + if origin is not None: + return { + "generic": _class_token(origin), + "args": [_type_token(a, converter, path) for a in args], + } + raise TypeError(f"Cannot build a type token for annotation {annotation!r}.") + + +def _class_token(cls: Any) -> str: + """Return the type token of a class: its canonical name, builtins written bare.""" + name = canonical_type_name(cls) + return name.removeprefix("builtins.") + + +def make_structural(cls: type, name: str) -> None: + """Mark a dataclass as the structural class ``name``. + + Called through ``structural_class.finish_structural_class`` on a class that + ``orcapod.dataclass`` has just created or that was built from stored metadata. + + Args: + cls: A dataclass created for this purpose. + name: The validated declared name. + """ + setattr(cls, STRUCTURAL_NAME_ATTRIBUTE, name) + remember_structural(name, cls) + + +def find_exact_class(name: str) -> Any: + """Return the object an exact class's name refers to now, or ``None``. + + Args: + name: ``module.qualname``; ``__main__.`` is looked up in the running + main module (``__main__`` or ``__mp_main__``). + + Returns: + The object found by attribute walk or import, or ``None`` if there is none. + """ + from orcapod.logical_types.type_utils import _walk_fqcn + + main_prefix = f"{MAIN_MODULES[0]}." + if name.startswith(main_prefix): + for module_name in MAIN_MODULES: + module = sys.modules.get(module_name) + obj = module + for part in name[len(main_prefix) :].split("."): + obj = getattr(obj, part, None) + if obj is None: + break + if obj is not None: + return obj + return None + try: + return _walk_fqcn(name) + except ImportError: + return None + + +# --------------------------------------------------------------------------- +# Same type? +# --------------------------------------------------------------------------- + + +def same_user_type(a: type, b: type) -> bool: + """Return True if two user classes are one type. + + Exact classes are one type only when they are the same object. Structural classes + are one type when their identity records are equal. + + Args: + a: A class. + b: A class. + + Returns: + Whether Orcapod treats the two classes as the same type. + """ + if a is b: + return True + if not (is_structural(a) and is_structural(b)): + return False + record_a, record_b = _record_of.get(a), _record_of.get(b) + if record_a is None or record_b is None: + from orcapod.contexts import get_default_context + + # The context declares its converter with the semantic-types protocol; it is + # the same converter object the logical types use. + converter: Any = get_default_context().type_converter + record_a = record_a or class_record(a, converter) + record_b = record_b or class_record(b, converter) + return record_a == record_b + + +def equivalent_hints(a: Any, b: Any) -> bool: + """Return True if two type hints denote the same types. + + Hints are equal, or differ only in structural classes that are one type + (``same_user_type``). Generic hints are compared by origin and arguments; union + members in any order. + + Args: + a: A type hint. + b: A type hint. + + Returns: + True if the hints denote the same types. + """ + import types as _types + + if a == b: + return True + if isinstance(a, type) and isinstance(b, type): + return same_user_type(a, b) + origin_a, origin_b = typing.get_origin(a), typing.get_origin(b) + if origin_a is None or origin_b is None: + return False + args_a, args_b = typing.get_args(a), typing.get_args(b) + unions = (typing.Union, _types.UnionType) + if origin_a in unions and origin_b in unions: + return all(any(equivalent_hints(x, y) for y in args_b) for x in args_a) and all( + any(equivalent_hints(x, y) for x in args_a) for y in args_b + ) + if origin_a != origin_b or len(args_a) != len(args_b): + return False + return all(equivalent_hints(x, y) for x, y in zip(args_a, args_b)) diff --git a/src/orcapod/logical_types/user_class_logical_type.py b/src/orcapod/logical_types/user_class_logical_type.py new file mode 100644 index 000000000..f03c7d3b7 --- /dev/null +++ b/src/orcapod/logical_types/user_class_logical_type.py @@ -0,0 +1,404 @@ +"""Shared logical type and factory for user classes (dataclasses and pydantic models). + +A user class's logical type name and Arrow extension name are its *identity string* +(``user_class_identity.identity_string``), and its Arrow extension metadata carries the +full identity record. Together they are the class's logical type identifier. + +Exact classes (ordinary dataclasses, pydantic models; ITL-972) are bound to the class +object: a value is written only if its class is that object, and stored values are read +by finding the class at its module path and comparing its record with the stored one. + +Write path (``create_for_python_type``): + Describes the class (name, kind, fields in declared order) and resolves each stored + field's Arrow type through the converter. + +Read path (``reconstruct_from_arrow``): + Reads the identity record from the metadata, finds the class for it + (``class_for_stored_record``) and returns that class's logical type. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from orcapod.errors import RecursiveTypeError, TypeIdentityError +import orcapod.logical_types.user_class_identity as uci +from orcapod.logical_types.base_logical_type import BaseLogicalType +from orcapod.logical_types.structural_class import class_for_record +from orcapod.logical_types.registry import ( + make_arrow_extension_type, + make_polars_extension_type, +) +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import polars as pl + import pyarrow as pa + + from orcapod.logical_types.protocols import TypeConverterProtocol +else: + pa = LazyModule("pyarrow") + pl = LazyModule("polars") + +logger = logging.getLogger(__name__) + + +# Extension type classes per (identity string, storage type, metadata). Two class objects +# with one identity (a class defined again, two structural classes of one type) must +# give Arrow types that compare equal, and pyarrow compares the type's Python class. +_EXTENSION_CLASSES: dict[tuple[str, str, str], tuple[type, type]] = {} + + +def _extension_classes( + logical_name: str, storage_type: pa.DataType, metadata_json: str +) -> tuple[type[pa.ExtensionType], type[pl.BaseExtension]]: + """Return the Arrow and Polars extension type classes of an identifier. + + Args: + logical_name: The identity string, used as the extension name. + storage_type: The Arrow storage struct. + metadata_json: The extension metadata as canonical JSON. + + Returns: + ``(arrow class, polars class)``, the same pair for the same arguments. + """ + key = (logical_name, str(storage_type), metadata_json) + found = _EXTENSION_CLASSES.get(key) + if found is None: + found = ( + make_arrow_extension_type( + logical_name, storage_type, metadata=metadata_json.encode("utf-8") + ), + make_polars_extension_type( + logical_name, storage_type, metadata=metadata_json + ), + ) + _EXTENSION_CLASSES[key] = found + return found + + +class UserClassLogicalType(BaseLogicalType): + """Logical type binding a user class to its Arrow extension type. + + The logical type name and Arrow extension name are the class's identity string; + the storage type is a ``pa.struct`` of the stored fields in declared order. + Subclasses set ``category``. + + Args: + logical_name: The identity string (e.g. ``"pkg.models.Point@76d961e4..."``). + python_type: The class. + storage_type: The Arrow ``pa.StructType`` of the stored fields. + field_annotations: ``(field_name, python_annotation)`` pairs matching the + fields in ``storage_type``. + identity_record: The identity record, embedded in the Arrow extension metadata. + ``None`` embeds only the category (for low-level tests). + """ + + category: str = "" + + #: A subclass of the class is a different class with its own identity, so the + #: registry must not match it to this logical type by ``issubclass``. + exact_python_type_only: bool = True + + def __init__( + self, + logical_name: str, + python_type: type, + storage_type: pa.StructType, + field_annotations: list[tuple[str, Any]], + identity_record: dict[str, Any] | None = None, + ) -> None: + self._logical_name = logical_name + self._python_type = python_type + self._storage_type = storage_type + self._field_annotations = field_annotations + self._identity_record = identity_record + + metadata: dict[str, Any] = {"category": self.category} + if identity_record is not None: + metadata["identity"] = identity_record + metadata_json = uci.canonical_json(metadata) + # ``storage_type`` never contains nested extension types (ET1 in design_issues/): + # the factory strips each field's top-level extension type before building it. + self._arrow_ext_class, self._polars_ext_class = _extension_classes( + logical_name, storage_type, metadata_json + ) + self._arrow_ext: pa.ExtensionType | None = None + self._polars_ext: pl.BaseExtension | None = None + + @property + def logical_type_name(self) -> str: + """The class's identity string.""" + return self._logical_name + + @property + def python_type(self) -> type: + """The class this logical type represents.""" + return self._python_type + + @property + def identity_record(self) -> dict[str, Any] | None: + """The identity record, or ``None`` for a logical type built without one.""" + return self._identity_record + + @property + def storage_type(self) -> pa.StructType: + """The Arrow struct type of the stored fields.""" + return self._storage_type + + def get_arrow_extension_type(self) -> pa.ExtensionType: + """Return the cached Arrow extension type (name = identity string).""" + if self._arrow_ext is None: + self._arrow_ext = self._arrow_ext_class() # pyright: ignore[reportCallIssue] # synthesized class takes no arguments + return self._arrow_ext + + def get_polars_extension_type(self) -> pl.BaseExtension: + """Return the cached Polars extension type.""" + if self._polars_ext is None: + self._polars_ext = self._polars_ext_class() # pyright: ignore[reportCallIssue] # synthesized class takes no arguments + return self._polars_ext + + def python_to_storage( + self, value: Any, converter: TypeConverterProtocol | None + ) -> dict[str, Any]: + """Convert an instance to an Arrow-compatible struct dict. + + Args: + value: An instance of ``python_type``. + converter: The active converter for per-field conversion. + + Returns: + A dict mapping field names to their Arrow storage values. + + Raises: + ValueError: If ``converter`` is ``None``. + TypeError: If ``value`` is not an instance of this type's class + (``accepts_instance``). + """ + if converter is None: + raise ValueError( + f"{type(self).__name__}.python_to_storage requires a converter — " + "pass a TypeConverterProtocol instance for field-level conversion." + ) + if not self.accepts_instance(value, converter): + raise TypeError( + f"Cannot store a value of class {uci.class_path(type(value))} as " + f"{uci.class_path(self._python_type)}: " + f"{self.instance_requirement}" + ) + return { + name: converter.python_to_storage(getattr(value, name), annotation) + for name, annotation in self._field_annotations + } + + #: What ``accepts_instance`` requires, for the error message. + instance_requirement: str = ( + "an ordinary dataclass or pydantic model is stored only from instances of " + "that exact class. Build an instance of the class, or annotate the value " + "with its own class." + ) + + def accepts_instance(self, value: Any, converter: TypeConverterProtocol) -> bool: + """Return True if ``value`` may be stored as this type. + + Args: + value: The value offered. + converter: The active converter. + + Returns: + For an exact class: whether ``type(value)`` is the class object. A subclass + instance is refused, because its own fields would be lost. + """ + return type(value) is self._python_type + + def storage_to_python( + self, storage_value: Any, converter: TypeConverterProtocol | None + ) -> Any: + """Build an instance from an Arrow struct dict. + + Args: + storage_value: A dict mapping field names to Arrow storage values, or + ``None`` for a null value (an optional field of this class that + holds ``None``). + converter: The active converter for per-field conversion. + + Returns: + An instance of ``python_type``, or ``None`` for a null value. + + Raises: + ValueError: If ``converter`` is ``None``. + """ + if converter is None: + raise ValueError( + f"{type(self).__name__}.storage_to_python requires a converter — " + "pass a TypeConverterProtocol instance for field-level conversion." + ) + if storage_value is None: + return None + kwargs = { + name: converter.storage_to_python(storage_value[name], annotation) + for name, annotation in self._field_annotations + } + return self._python_type(**kwargs) + + +class UserClassLogicalTypeFactory: + """Base factory for user-class logical types. + + Subclasses set ``kind``, ``category`` and ``logical_type_class``. + """ + + kind: str = "" + category: str = "" + logical_type_class: type[UserClassLogicalType] = UserClassLogicalType + + def supports_class(self, python_type: type) -> bool: + """Return True if this factory handles ``python_type``.""" + return uci.class_kind(python_type) == self.kind + + # ------------------------------------------------------------------ + # Write path + # ------------------------------------------------------------------ + + def create_for_python_type( + self, + python_type: type, + converter: TypeConverterProtocol, + ) -> UserClassLogicalType: + """Build the logical type of a user class (write path). + + Args: + python_type: The class. + converter: The active converter for field-type resolution. + + Returns: + A logical type whose name is the class's identity string. + + Raises: + ValueError: If an exact class is local or type hints cannot be resolved. + TypeError: If the class refers back to itself (directly or through other + classes), naming the field path, or a field type cannot be stored. + """ + record = uci.class_record(python_type, converter) + storage_type, field_annotations = self._storage(python_type, converter) + identity = uci.identity_string(record) + logger.debug("%s: built %r for %r", type(self).__name__, identity, python_type) + return self.logical_type_class( + identity, python_type, storage_type, field_annotations, record + ) + + def _storage( + self, python_type: type, converter: TypeConverterProtocol + ) -> tuple[pa.StructType, list[tuple[str, Any]]]: + """Return the storage struct and field annotations of a class. + + Registers the field types (their Arrow types are needed), but not + ``python_type`` itself. + + Args: + python_type: The class. + converter: The active converter for field-type resolution. + + Returns: + ``(storage_type, field_annotations)``, fields in declared order: the order + is part of the identity, so one identity always has one struct. + + Raises: + ValueError: If the type hints cannot be resolved. + RecursiveTypeError: If the class refers back to itself, naming the path. + """ + hints = uci.type_hints(python_type) + arrow_fields = [] + field_annotations: list[tuple[str, Any]] = [] + for field_name in uci.stored_field_names(python_type): + annotation = hints.get(field_name, Any) + try: + arrow_type = converter.register_python_class(annotation) + except TypeError as exc: + if isinstance(exc, RecursiveTypeError): + raise RecursiveTypeError( + f"{python_type.__qualname__}.{field_name} -> {exc}" + ) from exc + raise + # Struct fields cannot hold extension types (ET1; see design_issues/). + if isinstance(arrow_type, pa.ExtensionType): + arrow_type = arrow_type.storage_type + arrow_fields.append(pa.field(field_name, arrow_type)) + field_annotations.append((field_name, annotation)) + return pa.struct(arrow_fields), field_annotations + + # ------------------------------------------------------------------ + # Read path + # ------------------------------------------------------------------ + + def reconstruct_from_arrow( + self, + arrow_extension_name: str, + storage_type: pa.DataType, + metadata: dict[str, Any], + converter: TypeConverterProtocol, + ) -> UserClassLogicalType: + """Find the class of a stored column and return its logical type (read path). + + Args: + arrow_extension_name: The stored identity string. + storage_type: The stored struct type (already resolved). + metadata: The parsed extension metadata, with ``category`` and ``identity``. + converter: The active converter. + + Returns: + The logical type of the class the stored record describes. + + Raises: + ImportError: If an exact class cannot be found at its module path. + TypeIdentityError: If the column has no identity record (written before + Orcapod 0.3), the record is invalid, the class found is defined with + other fields, or its storage type differs from the stored one. + """ + record = metadata.get("identity") + if not isinstance(record, dict): + raise TypeIdentityError( + f"The stored column type {arrow_extension_name!r} was written by an " + f"Orcapod version before 0.3, which stored no description of the " + f"class. Orcapod 0.3 does not read such columns; re-running the " + f"pipeline computes the results into new tables." + ) + if ( + record.get("kind") != self.kind + or not isinstance(record.get("name"), str) + or uci.identity_string(record) != arrow_extension_name + ): + raise TypeIdentityError( + f"Stored column type {arrow_extension_name!r} has an invalid identity " + f"record for category {self.category!r}: {record!r}." + ) + try: + cls = class_for_record(record, converter) + except ImportError: + raise + except (TypeError, ValueError) as exc: + raise TypeIdentityError(str(exc)) from exc + _ = converter.register_python_class(cls) + lt = converter.get_logical_type_for_python_type(cls) + if not isinstance(lt, UserClassLogicalType): + raise TypeIdentityError( + f"{uci.class_path(cls)} is not registered as a user class." + ) + _check_storage(lt, storage_type, record) + return lt + + +def _check_storage( + lt: UserClassLogicalType, storage_type: pa.DataType, record: dict[str, Any] +) -> None: + """Raise ``TypeIdentityError`` if stored and registered storage differ beyond layout.""" + from orcapod.utils.arrow_utils import normalize_to_large_types + + if normalize_to_large_types( + lt.storage_type, keep_null=True + ) != normalize_to_large_types(storage_type, keep_null=True): + raise TypeIdentityError( + f"Stored values of {uci.describe_record(record)} have Arrow storage type " + f"{storage_type}, but the registered class stores {lt.storage_type}." + ) diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 87ffed444..16b85ecde 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, Literal from orcapod.channels import BroadcastChannel, Channel -from orcapod.errors import StoreError, StoreWriteError +from orcapod.errors import StoreError, StoreWriteError, TypeIdentityError from orcapod.pipeline.result import OrchestratorResult from orcapod.utils.store_errors import keep_store_error from orcapod.protocols.node_protocols import ( @@ -141,7 +141,7 @@ async def _run_async( effective_observer.on_run_start(run_id, pipeline_uri=pipeline_uri) topo_order: list[Any] = [] - store_error: StoreError | None = None + store_error: StoreError | TypeIdentityError | None = None try: topo_order = list(graph.topological_sort()) buf = self._buffer_size @@ -268,9 +268,17 @@ async def _run_async( # sync orchestrator does, rather than inside the TaskGroup's exception # group. store_error = _single_store_error(group, topo_order) - if store_error is None: - raise - raise store_error + if store_error is not None: + raise store_error + # A stored row that cannot be decoded as its class stops the run too, + # raised on its own as the sync orchestrator raises it (ITL-881). + identity_errors = [ + exc for exc in _leaves(group) if isinstance(exc, TypeIdentityError) + ] + if identity_errors: + store_error = identity_errors[0] + raise store_error + raise finally: # A failing on_run_end must not replace a failed store read or write. with keep_store_error(store_error, "Observer on_run_end"): diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index dfe238f5e..981706352 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -10,7 +10,7 @@ import uuid from typing import TYPE_CHECKING, Any, Literal -from orcapod.errors import StoreError +from orcapod.errors import StoreError, TypeIdentityError from orcapod.pipeline.result import OrchestratorResult from orcapod.utils import arrow_utils from orcapod.utils.store_errors import keep_store_error @@ -79,7 +79,7 @@ def run( effective_observer = observer if observer is not None else NoOpObserver() effective_observer.on_run_start(run_id, pipeline_uri=pipeline_uri) - store_error: StoreError | None = None + store_error: StoreError | TypeIdentityError | None = None current: Any = None try: topo_order = list(graph.topological_sort()) @@ -142,6 +142,11 @@ def run( exc.add_context(node_label=current.label) store_error = exc raise + except TypeIdentityError as exc: + # A stored row that cannot be decoded as its class stops the run as a + # store error does; on_run_end must not replace it (ITL-881). + store_error = exc + raise finally: # A failing on_run_end must not replace a failed store write. with keep_store_error(store_error, "Observer on_run_end"): diff --git a/src/orcapod/semantic_types/universal_converter.py b/src/orcapod/semantic_types/universal_converter.py index b26072f83..2ebd21582 100644 --- a/src/orcapod/semantic_types/universal_converter.py +++ b/src/orcapod/semantic_types/universal_converter.py @@ -430,9 +430,11 @@ def _register_python_class_impl( # Cycle detection (via the shared ContextVar-backed in_progress set) if annotation in in_progress: - raise TypeError( - f"Circular type dependency detected while synthesising " - f"LogicalType for {annotation!r}." + from orcapod.errors import RecursiveTypeError + + raise RecursiveTypeError( + f"{annotation.__qualname__} (a class that refers back to itself " + f"cannot be stored: Arrow has no recursive struct type)" ) # Factory dispatch via MRO walk diff --git a/src/orcapod/structural.py b/src/orcapod/structural.py new file mode 100644 index 000000000..06acd8fca --- /dev/null +++ b/src/orcapod/structural.py @@ -0,0 +1,147 @@ +"""``orcapod.dataclass``: dataclasses identified by name and fields (ITL-972). + +An ordinary ``@dataclass`` is identified by its exact class: a pod annotated with it +accepts that class object and no other. A class made with ``@orcapod.dataclass`` is +**structural**: Orcapod identifies it by its name and its fields (names, types and +order). Two such classes with the same name and fields are one type: a pod annotated +with one accepts a column of the other, their instances compare equal when their field +values are equal, and stored values can be read back without the class being defined. + +Orcapod never converts a value from one class to another. A function annotated with a +structural class receives the object that arrives, which may be an instance of another +class of the same type: a structural class promises its fields, not its methods. +``from_instance`` is the explicit conversion. + +See ``docs/concepts/your-own-data-types.md``. +""" + +from __future__ import annotations + +import dataclasses +import typing +from collections.abc import Callable +from typing import Any, TypeVar, overload + +import orcapod.logical_types.structural_class as _structural +import orcapod.logical_types.user_class_identity as _uci + +T = TypeVar("T") + +__all__ = ["dataclass"] + + +@overload +def dataclass(cls: type[T], /) -> type[T]: ... + + +@overload +def dataclass( + *, name: str | None = None, **options: Any +) -> Callable[[type[T]], type[T]]: ... + + +@typing.dataclass_transform() +def dataclass( + cls: type[T] | None = None, /, *, name: str | None = None, **options: Any +) -> type[T] | Callable[[type[T]], type[T]]: + """Define a dataclass that Orcapod identifies by its name and fields. + + Use it in place of ``dataclasses.dataclass``, bare or with keywords. The result is + a real dataclass (``dataclasses.fields``, ``asdict`` and ``replace`` work) with two + differences. ``==`` compares field values across classes of the same type: an + instance equals an instance of another ``orcapod.dataclass`` with the same name + and fields when their field values are equal (unless ``eq=False`` is passed or the + class defines ``__eq__`` itself). Equality and hashing always cover all stored + fields: ``field(compare=False)`` and ``field(hash=...)`` are not accepted. And the class has ``from_instance``, which builds + an instance from an instance of another dataclass with the same fields. + + Args: + cls: The class, when used as a bare decorator. + name: The name the type is identified by. Default: the class name. It must not + be empty, contain ``@``, ``;`` or whitespace, or start with ``orcapod.``. + **options: Keyword options of ``dataclasses.dataclass`` (``frozen``, + ``slots``, ``kw_only``, ``order``, ...). ``init=False`` is not allowed. + + Returns: + The dataclass, or a decorator that returns it. + + Raises: + TypeError: If the class is already a dataclass or a pydantic model, + ``init=False`` is passed, or a field sets ``compare=`` or ``hash=`` so + that equality or hashing would not cover exactly the stored fields. + ValueError: If the name is invalid. + + Example: + >>> import orcapod as op + >>> @op.dataclass + ... class Point: + ... x: int + ... y: float + """ + if options.get("init", True) is not True: + raise TypeError( + "orcapod.dataclass does not accept init=False: Orcapod builds instances " + "through the generated __init__." + ) + + def decorate(target: type[T]) -> type[T]: + """Make ``target`` a dataclass and mark it as a structural class.""" + if not isinstance(target, type): + raise TypeError(f"orcapod.dataclass decorates a class, got {target!r}.") + if "__dataclass_fields__" in vars(target): + raise TypeError( + f"{target.__qualname__} is already a dataclass. Use @orcapod.dataclass " + f"in place of @dataclasses.dataclass, not on top of it." + ) + if _uci.class_kind(target) == _uci.KIND_PYDANTIC: + raise TypeError( + f"{target.__qualname__} is a pydantic model. orcapod.dataclass makes " + f"dataclasses only." + ) + declared = _uci.validate_name(name if name is not None else target.__name__) + own_eq = "__eq__" in vars(target) + made = dataclasses.dataclass(target, **options) + _require_structural_comparison(made) + _structural.finish_structural_class( + made, declared, structural_equality=options.get("eq", True) and not own_eq + ) + return made + + if cls is None: + return decorate + return decorate(cls) + + +def _require_structural_comparison(cls: type) -> None: + """Raise unless equality and hashing of a new dataclass cover its stored fields. + + Two classes of one structural type must compare and hash alike, including a class + built from stored metadata, which knows the fields and nothing else. So a stored + field (``init=True``) always takes part, a field that is not stored never does, and + no field chooses its own ``hash=``. + + Args: + cls: The dataclass ``orcapod.dataclass`` has just created. + + Raises: + TypeError: Naming the first field that breaks the rule, and why. + """ + for field in dataclasses.fields(cls): + if field.hash is not None: + problem = f"sets hash={field.hash!r}" + elif field.init and not field.compare: + problem = "sets compare=False" + elif not field.init and field.compare: + problem = ( + "has init=False, so it is not stored, but takes part in comparison" + ) + else: + continue + raise TypeError( + f"Field {field.name!r} of the orcapod.dataclass {cls.__qualname__} " + f"{problem}. An orcapod.dataclass is identified by its fields, and two " + f"classes with the same name and fields must compare and hash alike, so " + f"equality and hashing cover exactly the stored fields. Remove the " + f"compare= or hash= argument (declare an init=False field with " + f"compare=False), or use an ordinary dataclass." + ) diff --git a/src/orcapod/utils/schema_utils.py b/src/orcapod/utils/schema_utils.py index 34e7cf9cd..b585895f9 100644 --- a/src/orcapod/utils/schema_utils.py +++ b/src/orcapod/utils/schema_utils.py @@ -4,7 +4,8 @@ import logging import sys from collections.abc import Callable, Collection, Mapping, Sequence -from typing import Any, get_args, get_origin, get_type_hints +from types import UnionType +from typing import Any, Union, get_args, get_origin, get_type_hints from orcapod.types import Schema, SchemaLike @@ -62,6 +63,42 @@ def _type_label(type_info: Any) -> str: return str(type_info) +def _union_members(hint: Any) -> tuple[Any, ...] | None: + """Return the members of a union hint, or ``None`` if ``hint`` is not a union.""" + if get_origin(hint) in (Union, UnionType): + return get_args(hint) + return None + + +def _accepts(incoming: Any, receiving: Any) -> bool: + """Return True if a value typed ``incoming`` can be passed where ``receiving`` is expected. + + ``incoming`` must be a subhint of ``receiving``, or the same type up to user-class + identity (a class and its re-definition are one type). A receiving union accepts + what one of its members accepts; an incoming union needs every member accepted. + + Args: + incoming: The type hint of the value offered. + receiving: The type hint of the parameter. + + Returns: + Whether the value can be passed. + """ + from beartype.door import is_subhint + + from orcapod.logical_types.user_class_identity import equivalent_hints + + if is_subhint(incoming, receiving) or equivalent_hints(incoming, receiving): + return True + incoming_members = _union_members(incoming) + if incoming_members is not None: + return all(_accepts(member, receiving) for member in incoming_members) + receiving_members = _union_members(receiving) + if receiving_members is not None: + return any(_accepts(incoming, member) for member in receiving_members) + return False + + def schema_incompatibility( incoming_types: SchemaLike, receiving_types: Schema ) -> str | None: @@ -79,12 +116,10 @@ def schema_incompatibility( ``None`` when the schemas are compatible, otherwise a one-line reason naming the first offending column or parameter. """ - from beartype.door import is_subhint - for key, type_info in incoming_types.items(): if key not in receiving_types: return f"column '{key}' is not a parameter of the function" - if not is_subhint(type_info, receiving_types[key]): + if not _accepts(type_info, receiving_types[key]): return ( f"column '{key}' has type {_type_label(type_info)}, " f"the function expects {_type_label(receiving_types[key])}" diff --git a/src/orcapod/utils/store_errors.py b/src/orcapod/utils/store_errors.py index 20d447d18..5ba8ad4d6 100644 --- a/src/orcapod/utils/store_errors.py +++ b/src/orcapod/utils/store_errors.py @@ -1,8 +1,9 @@ """Helpers that keep a ``StoreError`` intact on its way out of a run. -A failed store read or write always fails the run (ITL-878, ITL-884). User callbacks that run while -the error propagates (a post-run hook, an observer callback) may fail too; their -failure is logged and noted on the store error, never raised in its place. +A failed store read or write always fails the run (ITL-878, ITL-884), and so does a +stored row that cannot be decoded as its class (``TypeIdentityError``, ITL-881). User +callbacks that run while the error propagates (a post-run hook, an observer callback) +may fail too; their failure is logged and noted on the error, never raised in its place. """ from __future__ import annotations @@ -12,7 +13,7 @@ from contextlib import contextmanager from typing import TYPE_CHECKING -from orcapod.errors import StoreError +from orcapod.errors import StoreError, TypeIdentityError if TYPE_CHECKING: from orcapod.protocols.database_protocols import ArrowDatabaseProtocol @@ -21,12 +22,12 @@ def note_callback_failure( - store_error: StoreError, during: str, failure: Exception + store_error: StoreError | TypeIdentityError, during: str, failure: Exception ) -> None: """Log a callback failure and record it on the propagating store error. Args: - store_error: The store error on its way out of the run. + store_error: The store error (or identity error) on its way out of the run. during: What was running, for the log and the note, such as ``"Observer on_data_crash"``. failure: The exception the callback raised. @@ -45,7 +46,9 @@ def note_callback_failure( @contextmanager -def keep_store_error(store_error: StoreError | None, during: str) -> Iterator[None]: +def keep_store_error( + store_error: StoreError | TypeIdentityError | None, during: str +) -> Iterator[None]: """Run a callback so that it cannot replace a propagating store error. With ``store_error`` set, an ``Exception`` the block raises is logged and noted on @@ -54,7 +57,8 @@ def keep_store_error(store_error: StoreError | None, during: str) -> Iterator[No unchanged and its exceptions propagate. Args: - store_error: The store error on its way out, or ``None`` when none is. + store_error: The store error (or identity error) on its way out, or ``None`` + when none is. during: What the block runs, for the log and the note. Yields: diff --git a/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md new file mode 100644 index 000000000..54b3aaa6c --- /dev/null +++ b/superpowers/plans/2026-11-10-itl-881-user-class-identity-impl.md @@ -0,0 +1,189 @@ +# User class identity (v0.3) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Identify a dataclass or pydantic model by `{kind, name, fields}` (spec +`superpowers/specs/2026-11-10-user-class-identity-v0.3.md`), with a declared-name API, silent +rebinding of identical re-registrations, and a raise on stored/registered shape mismatch. + +**Architecture:** One new module, `orcapod/logical_types/user_class_identity.py`, owns the +naming policy, declarations, type tokens, the identity record and string (single-point +format), the process-wide `(kind, name)` table and the read-side class lookup. The two +factories share a new base (`user_class_logical_type.py`) that builds the identity string +and uses it as logical type name and Arrow extension name; the metadata carries the record. +`LogicalTypeRegistry` learns to rebind an identical identity to a new class object and stops +matching user classes by subclass. The public API (`register_type`, `set_type_naming`) lives +in `orcapod/user_types.py`. `TypeIdentityError` lives in `orcapod/errors.py`; the Delta read +path re-raises it. + +**Tech Stack:** Python 3.11–3.13, pyarrow, polars, pydantic v2, deltalake, IPython (dev, for +the notebook scenarios), pytest via `uv run`. + +**Changes after the plan was written (developer decisions, November 10, 2026):** +`set_type_naming` was dropped (fixed location rule plus `register_type(qualified_name=False)`), +the identity string uses the full 64-hex digest, and stored structs list fields in name +order. The spec is authoritative where this plan differs. + +--- + +## File map + +| File | Change | +|---|---| +| `src/orcapod/errors.py` | add `TypeIdentityError(ValueError)` | +| `src/orcapod/logical_types/user_class_identity.py` | **new**: kinds, naming policy, declarations, `type_token`, `identity_record`, `canonical_json`, `identity_string`, known-class table, `find_class`, `describe_fields`, S3 notice | +| `src/orcapod/logical_types/user_class_logical_type.py` | **new**: `UserClassLogicalType`, `UserClassLogicalTypeFactory` (shared write/read path) | +| `src/orcapod/logical_types/dataclass_logical_type_factory.py` | reduce to subclasses: kind, category, `supports_class`, stored fields | +| `src/orcapod/logical_types/pydantic_logical_type_factory.py` | same | +| `src/orcapod/logical_types/registry.py` | rebinding rule; exact-type match for user classes; `TypeIdentityError` from `_deserialize` | +| `src/orcapod/semantic_types/universal_converter.py` | cycle error names the class and says recursive classes cannot be stored | +| `src/orcapod/databases/delta_lake_databases.py` | re-raise `TypeIdentityError` where read errors are logged | +| `src/orcapod/logical_types/database_hooks.py` | add the column name to a `TypeIdentityError` raised while registering a discovered type | +| `src/orcapod/user_types.py` | **new**: `register_type`, `set_type_naming` | +| `src/orcapod/__init__.py` | re-export the two functions | +| tests | `tests/test_user_types/` (new): unit tests of identity, API, registry, read path; `test_workshop_scenarios.py` (subprocess + IPython `InteractiveShell`) | +| existing tests asserting `module.qualname` names | switch to the identity string | +| `tests/test_hashing/golden/cases.py` + `data/*.json` | add new fixture cases; regenerate (25 listed moves + additions) | +| docs | new `docs/concepts/your-own-data-types.md`; section in `docs/concepts/identity.md`; nav in `mkdocs.yml`; link from `function-pods.md` | +| `CHANGELOG.md`, `DESIGN_ISSUES.md` | entry each | + +## Key definitions (used by every task) + +```python +# user_class_identity.py +KIND_DATACLASS = "dataclass" +KIND_PYDANTIC = "pydantic" +_NAMING = {"main": "name", "module": "qualified"} # set_type_naming +_MAIN_MODULES = frozenset({"__main__", "__mp_main__"}) +_DECLARED: weakref.WeakKeyDictionary[type, str] # register_type +_KNOWN: dict[tuple[str, str], list[weakref.ref[type]]] # (kind, name) -> classes + +def validate_name(name: str) -> None: ... # non-empty, no '@', no whitespace, not 'orcapod.' +def class_kind(cls: type) -> str | None: ... +def resolve_name(cls: type) -> str: ... # declared > main policy > module policy +def declare(cls: type, name: str | None) -> None: ... # raises if already built under another name +def set_naming(main: str | None, module: str | None) -> None: ... +def type_token(annotation, converter) -> str: ... +def identity_record(kind, name, fields: dict[str, str]) -> dict: ... +def canonical_json(obj) -> str: ... # sort_keys, (",", ":") +def identity_string(record) -> str: ... # f"{name}@{sha256(canonical_json)[:16]}" (single point) +def describe_fields(fields: dict[str, str]) -> str: ... # "{x: int, y: float}" +def remember(kind, name, cls) -> None: ... # _KNOWN +def candidates(kind, name) -> list[type]: ... # _KNOWN + declared + __main__ walk + import +``` + +Type tokens: builtins bare (`int`); `NoneType` → `None`; other classes without logical type → +`canonical_type_name`; class with logical type → `lt.logical_type_name` (built-ins +`orcapod.path`, user classes their identity string, P1); `list[...]`/`set[...]`/`dict[...]` +→ `origin[args]` with origin by canonical name minus `builtins.`; unions → sorted members +joined by ` | `; `Literal` → `Literal[...]` with sorted `repr`s. + +--- + +### Task 1: Identity primitives (pure functions) + +**Files:** create `src/orcapod/logical_types/user_class_identity.py`, modify +`src/orcapod/errors.py`; test `tests/test_user_types/test_identity_record.py`. + +- [ ] Step 1: failing tests: `identity_string({"kind":"dataclass","name":"Point","fields":{"x":"int","y":"float"}}) == "Point@9117beaaf7ebe3ab"`; field-order independence; `type_token` for `int`, `int | None`, `Optional[int]`, `None | int`, `list[int]`, `typing.List[int]`, `dict[str, float]`, `Path` (`orcapod.path`), `datetime.datetime`, `Literal["b","a"]`; `validate_name` rejects `""`, `"a@b"`, `"a b"`, `"orcapod.x"`; `resolve_name` for a module class (`tests.…Point` qualified), a class with `__module__="__main__"` (`"Point"`), `"__mp_main__"` (same), after `set_naming(main="qualified")` (`"__main__.Point"`), after `set_naming(module="name")`. +- [ ] Step 2: run, expect ImportError (module missing). +- [ ] Step 3: implement the functions above. +- [ ] Step 4: run, pass. Commit `feat(types): identity record and string for user classes (ITL-881)`. + +### Task 2: Shared user-class logical type; factories use the identity + +**Files:** create `src/orcapod/logical_types/user_class_logical_type.py`; reduce both factory +modules to subclasses; test `tests/test_user_types/test_user_class_logical_type.py`. + +`UserClassLogicalType(identity, record, python_type, storage_type, field_annotations, category)`: +logical name = Arrow extension name = identity string; metadata +`canonical_json({"category": category, "identity": record}).encode()`; conversions as today. +Class attribute `exact_python_type_only = True`. + +`UserClassLogicalTypeFactory` (abstract: `kind`, `category`, `supports_class`, `stored_fields(cls, hints)`): +- `create_for_python_type(cls, converter)`: reject ``; name = `resolve_name(cls)`; + for each stored field: `arrow = converter.register_python_class(annotation)` (wrap a + circular `TypeError` as `TypeError(f"{cls.__qualname__}.{field}: recursive classes cannot be stored ({exc})")`); + token = `type_token(annotation, converter)`; record; identity; `remember(kind, name, cls)`; + S3 notice when the name came from the `__main__` policy (once per name per process). +- `reconstruct_from_arrow(ext_name, storage, metadata, converter)`: no `identity` key → + `TypeIdentityError` (pre-0.3 column); else for each candidate class of (kind, name): get its + logical type through `converter.register_python_class(cls)` + + `converter.get_logical_type(cls)`; return the first whose `logical_type_name == ext_name` + (and whose storage equals the stored storage after `normalize_to_large_types`, else + `TypeIdentityError` naming both struct types). No candidate → `ImportError` naming name, + kind, stored fields and `op.register_type(Point, name='Point')`. Candidates but no match → + `TypeIdentityError` naming class, kind, stored fields, registered fields. + +- [ ] Step 1: failing tests: logical name is the identity string; metadata contains the record; two classes with permuted fields share the identity; dataclass vs pydantic same name+fields differ; same name, `y: int` vs `y: float` differ; nested dataclass token is the inner identity string and changes when the inner fields change or the inner is declared under another name (P1); outer built before inner declared gives the same identity as inner-first (order); self-referential class → `TypeError` naming `Node.next`; `` → `ValueError`; subclass with an extra field gets its own identity (exact-type match). +- [ ] Step 2: run, fail. Step 3: implement. Step 4: update the existing factory/roundtrip tests that assert `module.qualname` (listed in the file map) to read the identity from the registry. Step 5: run `uv run pytest tests/test_logical_types tests/test_user_types -q`, pass. Commit. + +### Task 3: Registry rebinding and exact-type match + +**Files:** `src/orcapod/logical_types/registry.py`; test `tests/test_user_types/test_rebinding.py`. + +- `register_logical_type`: when the logical name is bound to a different instance whose + Arrow extension type has the same name, storage type and serialized metadata, and the + Python type differs → rebind: replace the logical-name and Arrow-name bindings, add the new + Python-type binding, keep the old Python-type binding. If the previous class's + `module.qualname` differs from the new one's → one `UserWarning` naming both. Anything + else keeps raising. +- `get_by_python_type`: skip the `issubclass` fallback for logical types with + `exact_python_type_only`. +- `_deserialize`: raise `TypeIdentityError` (subclass of `ValueError`) for storage/metadata mismatch. + +- [ ] Steps: failing tests (rebind same identity silently, `warnings` none; different module.qualname same identity → one `UserWarning`; one class object under two names → `ValueError`), implement, pass, commit. + +### Task 4: Public API + +**Files:** create `src/orcapod/user_types.py`; modify `src/orcapod/__init__.py`; test +`tests/test_user_types/test_register_type.py`. + +```python +def register_type(cls=None, /, *, name=None): # decorator, decorator factory, or call +def set_type_naming(*, main=None, module=None) -> None: +``` + +- [ ] Steps: failing tests through `op.register_type` / `op.set_type_naming` and a `DictSource` + `function_pod` pipeline: declared name gives the identity `Name@…`; `register_type(Point, name="Geo")` after first use under another name → `ValueError`; invalid names; non-dataclass → `TypeError`; returns the class. Implement, pass, commit. + +### Task 5: Read path and S2 guard + +**Files:** `delta_lake_databases.py`, `database_hooks.py`; test +`tests/test_user_types/test_read_path.py` (public interface: pipeline in a subprocess writes +to `DeltaTableDatabase`, a second subprocess reads with a different class). + +- [ ] Steps: failing tests: reader with matching class (declared in another module) reads values; reader without the class → `ImportError` naming `Point` and `register_type`; reader whose `Point` has different fields reading the stored table directly → `TypeIdentityError` naming column, both field sets; re-run of the pipeline with changed fields computes into new tables (calls == rows), old table still readable by the old class. Implement re-raise, pass, commit. + +### Task 6: Workshop scenarios W1–W12 as tests + +**File:** `tests/test_user_types/test_workshop_scenarios.py` (each scenario in a fresh +subprocess; notebook cells through `IPython.core.interactiveshell.InteractiveShell`, as E02; +children end with `os._exit(0)` after printing JSON, as the E-harness does). + +W1 re-run same fields (calls 0 on second run, no error); W2 changed fields (new tables: +calls 2, rows 2, no error); W3 pydantic re-run; W4 `importlib.reload`; W5 restart + re-run +(two processes, cache hit); W6 restart with changed fields (recompute, rows 2); W7 script as +`__main__` twice; W8 `-m` vs import with a declared name (hit); W11 notebook → module with a +declared name: identical pipeline hashes and table paths; W12 read notebook results from a +script defining the same `Point`. Plus: pydantic vs dataclass same name+fields are distinct +(different pod pipeline hashes). + +- [ ] Steps: write, run (must fail before Tasks 1–5 on `main` for W1–W4, W6, W11, W12; record which), pass after, commit. + +### Task 7: Golden values + +- [ ] Add fixture cases to `cases.py` (new names only; nothing moved): declared-name dataclass, nested dataclass, `Optional` field, container field, permuted-order twin pair. +- [ ] Run `uv run pytest tests/test_hashing/golden -q`: exactly the 25 spec entries fail. Record the list. +- [ ] Regenerate; `git diff --stat` and a script comparing old/new JSON: changed keys == the 25, plus only added keys. Commit `test(hashing): golden values for v0.3 user class identity (ITL-881)`. + +### Task 8: Docs, CHANGELOG, DESIGN_ISSUES + +- [ ] `docs/concepts/your-own-data-types.md` (runnable samples: define, declare, re-run, what changes identity), identity section in `docs/concepts/identity.md`, nav entry, link from `function-pods.md`. Run `uv run pytest tests/test_docs -q`. +- [ ] CHANGELOG entry with the hash-change statement from spec §e. +- [ ] DESIGN_ISSUES entry appended at the end (resolved, with Fix note). + +### Task 9: Verification and PR + +- [ ] ruff check/format, basedpyright exit code, full `uv run pytest tests/` on 3.11/3.12/3.13 (`uv run --python 3.1x`). +- [ ] Merge `origin/main` (after #286/#289 land), re-run E08 on the new main, regenerate golden only for the listed entries. +- [ ] PR body: intended hash changes list; mark ready via sensei:create-pr. diff --git a/superpowers/plans/2026-11-11-itl-881-field-order-identity.md b/superpowers/plans/2026-11-11-itl-881-field-order-identity.md new file mode 100644 index 000000000..cb1ef7795 --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-881-field-order-identity.md @@ -0,0 +1,99 @@ +# ITL-881 rebuild: field order is identity (PR #292) + +Continues `2026-11-10-itl-881-user-class-identity-impl.md`. Worker notes are at the end, so a +successor can pick this up. + +**Goal:** rebuild PR #292 on current `main` and change the identity of a user class so that +declared field order is part of it (developer, November 11, 2026, reversing "not order"). + +## State on arrival (November 11) + +- Worktree clean, nothing unpushed; head `b930f629`, conflicting with `main`. +- Merged `origin/main` as `ac719294` (pushed). Conflicts: CHANGELOG, DESIGN_ISSUES, mkdocs, + `delta_lake_databases.py`, `schema_utils.py`. Golden files merged cleanly and pass. +- Merge finding: main's registry MRO lookup (ITL-888) returned the base class's logical type + for a dataclass subclass, bringing back the subclass-stored-as-base bug. Fixed in the merge + commit; `test_subclass_has_its_own_identity_and_keeps_its_fields` caught it. +- Merge decision: main now wraps Delta read failures in `StoreReadError`. `TypeIdentityError` + still propagates unwrapped (the table is intact; the class differs). + +## Identity today (PR head) + +Record `{"kind", "name", "fields": {name: token}}`, canonical JSON with sorted keys, identity +string `Name@`. Fields are sorted by name in the record and in the stored struct, so +order never enters. + +## Change + +1. `identity_record`: `fields` becomes a list of `[name, token]` pairs in declared order + (`dataclasses.fields` order for `init=True` fields; `model_fields` order for pydantic). +2. Stored struct lists fields in declared order (as on `main` and v0.2.0). +3. `describe_fields` prints declared order. Read-path comparisons use the ordered list. +4. Docs, spec, CHANGELOG, DESIGN_ISSUES UCI1: "in any order" becomes "in declared order". +5. Golden: twins `GoldenTwinAB`/`GoldenTwinBA` must pin different values. They share the + declared name `golden.Twin`; with order in the identity that is two types under one name, + which is allowed (same as two field sets under one name). + +No label or version marker is introduced: the identity string is `Name@` and the +metadata is `{"category", "identity"}`. + +## Evidence (run on this branch, starfix 0.4.0) + +- starfix 0.4.0 hashes a plain `struct` and `struct` with the same values to the + same hash (order-insensitive). +- The Arrow extension name enters the hash: two extension types with different names over the + same storage hash differently. Through a `DictSource`, two classes differing only in name + have different row hash, content hash and pipeline hash, for a scalar column, a `list[...]` + column and a nested class. +- So with order in the identity string, value hashes separate on starfix 0.4.0. The PR stays + on `main`; `ordered_struct_fields` (starfix 0.5.0, PR #309) is not needed. +- Equality: a dataclass compares its fields as a tuple in declared order. Pydantic `__eq__` + compares type and `__dict__` (order-insensitive), but two models differing in order are + different types, and `model_dump`, `model_fields`, iteration and the JSON output all follow + declared order, so pod code observes it. + +## Tests first (must fail before the change) + +In `tests/test_user_types/`, through `DictSource` / function pods: + +- Same fields, other order: different type name, stored struct order follows the class, row + hash, content hash and pipeline hash differ (dataclass and pydantic). +- Subclass: own identity and all its fields, with base-first and subclass-first registration + run in two fresh processes; hash, pod uri and stored fields equal across the two orders + (dataclass and pydantic). +- Class redefined with an added field: new type, new tables, old results untouched. +- Re-run with unchanged order still rebinds silently. + +## Review findings still open + +1. CodeRabbit (minor, unanswered thread): testing a read candidate registers its nested + classes under their default names, so a later `register_type(Inner, name=...)` raises. + Fix: make the probe leave no registration behind when the candidate is rejected; add the + nested recovery test. +2. CodeRabbit nitpick (review body): assert that compaction rewrote the result table. +3. IPC read of non-large storage: answered, deferred to ITL-894 (strict xfail in place). + +## Verification + +Full suite on 3.11, 3.12, 3.13 one at a time; ruff; basedpyright; push; CI; CodeRabbit round. + +## Worker notes + +- Coordinator approved the plan (November 11): field order in identity, stay on `main`, + `TypeIdentityError` unwrapped (pinned in `tests/test_databases/test_delta_read_failures.py`), + defaults/methods/computed fields outside the identity (stated in the docs). +- Done and pushed: identity record with ordered fields; storage struct in declared order; + golden regenerated (vs `main`: 17 changed + 8 replaced = the 25 approved entries, rest + added; twins pin different values; ndarray-field fixture added); `probe_record` for read + candidates (nested classes stay undeclared); compaction test asserts the rewrite; docs, + spec, CHANGELOG, DESIGN_ISSUES UCI1. +- Remaining: full suite on 3.11/3.12/3.13, ruff, basedpyright, PR body, CI, CodeRabbit round. +- Out of scope, to report: the semantic hasher expands a namedtuple with fields sorted by + name (`semantic_hasher._expand_namedtuple`), so namedtuple field order does not enter its + hash, although a namedtuple compares as an ordered tuple. +- Second CodeRabbit round (plan approved by the coordinator): identity errors fatal under + `error_policy="continue"` in both orchestrators (own commit); `None` nested class on read; + stored record checked against its extension name; lazy candidate lookup; distinct scenario + values; test docstrings; spec example. Declined: stale read converter after rebinding (not + reproduced). Deferred: pydantic field alias fails on read (also on `main`). +- Full suite after these: 7794 passed, 96 skipped, 20 xfailed on 3.11, 3.12 and 3.13. diff --git a/superpowers/plans/2026-11-12-itl-972-user-class-kinds.md b/superpowers/plans/2026-11-12-itl-972-user-class-kinds.md new file mode 100644 index 000000000..baa8ab129 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-972-user-class-kinds.md @@ -0,0 +1,136 @@ +# User class kinds (ITL-972) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace PR #292's user-class identity with two kinds: exact classes (ordinary +dataclass, pydantic) and structural classes (`orcapod.dataclass`). + +**Architecture:** Keep PR #292's identity record, `name@digest` extension name and shared +`UserClassLogicalType`; make tokens structured JSON; add kind `structural` with its own +factory, a synthesized class on read, and one conversion helper at the pod-input boundary. +Remove `register_type`, declared names and equivalence for exact classes. + +**Tech Stack:** Python 3.11–3.13, pyarrow, polars, pydantic, deltalake, pytest, `uv run`. + +**Design:** `superpowers/specs/2026-11-12-itl-972-user-class-kinds.md`, approved with changes +by the developer on November 12, 2026 (its section "Answers to the open points"). The +decisions removed the pod-input conversion and the `P(other)` form from the tasks below and +added structural equality. + +--- + +## Successor notes (keep current) + +- Branch `eywalker/itl-972-user-classes-an-ordinary-dataclass-is`, created at the ITL-967 + tip c4b1e4a7. The merge of `origin/eywalker/itl-881-user-class-identity-impl` (c6837a15) + was tried and aborted; it is redone in Task 0. +- Conflicts found (6 files) and proposed resolutions: design, section "Merge conflicts". +- Dispatch: task `task_1a2dfeda545d`, dispatch `ctx_604897ad82ad`; questions go through + `orca orchestration ask`. +- One pytest process at a time, one Python version at a time. +- November 12: design sent; O1-O7 are with the developer. Approved meanwhile: the merge + (done, 89b4c61b, resolutions as in the design) and failing tests for the parts no + decision changes. +- Baseline after the merge (Python 3.12): 8626 passed, 101 skipped, 19 xfailed, 37 failed. + The 37: the 25 golden entries of `GoldenPoint`/`GoldenModel` (the ITL-967 golden files + are kept until the one regeneration) plus `test_no_unpinned_entries[semantic|schema| + datagram]` and `test_values_independent_of_process_and_hash_seed[0|4242]`; and 7 new + tests in `tests/test_user_types/test_exact_classes.py` that fail by design. + `tests/test_user_types/test_measured_case.py` does not import yet (`op.dataclass`). + +- November 12, later: developer's answers applied (no automatic conversion, `from_instance` + only, structural `__eq__`, subclass column accepted at a base parameter). Implemented: + `logical_types/user_class_identity.py` (kinds, names, structured tokens, record), + `logical_types/structural_class.py` (rebuild from metadata, equality, `from_instance`), + `logical_types/structural_logical_type_factory.py`, `orcapod/structural.py` (decorator). + ITL-967 tip 9743004d and `main` (#321, #310, #311) merged. Golden regenerated once: 25 + moved entries, listed in the design, section e. Checks: ruff, basedpyright exit 0, suite + green except `tests/test_design_issues.py`, which fails on `design_issues/ITL-935-LTI1.md` + and `ITL-942-WH1.md` (files of the ITL-967 branch, to be fixed there). +- `P.from_instance` is added at run time, so static type checkers do not see it. + +## Files + +| File | Change | +|---|---| +| `src/orcapod/logical_types/user_class_identity.py` | kinds (`dataclass`, `pydantic`, `structural`); names (module path; structural marker); structured tokens; record; token → annotation; remove `declare`, `_declared`, `default_name(qualified_name=)`, the `__main__` notice; `equivalent_hints` becomes exact for exact classes, by record for structural | +| `src/orcapod/logical_types/user_class_logical_type.py` | exact-class value check on write; read path by module lookup for exact classes | +| `src/orcapod/logical_types/structural_logical_type_factory.py` (new) | `StructuralLogicalType`, factory, category `orcapod.structural`, synthesis on read | +| `src/orcapod/structural.py` (new) | `dataclass` decorator | +| `src/orcapod/logical_types/structural_class.py` (new) | class from a stored record (synthesis), `structural_eq`, `from_instance` | +| `src/orcapod/user_types.py` | deleted | +| `src/orcapod/__init__.py` | export `dataclass`; drop `register_type` | +| `src/orcapod/utils/schema_utils.py` | `_accepts`: `is` for exact classes, record equality for structural | +| `src/orcapod/logical_types/registry.py`, `database_hooks.py`, `databases/delta_lake_databases.py` | merge resolutions; rebinding without the cross-module warning | +| `src/orcapod/contexts/logical_types.json` | register the structural factory | +| `tests/test_user_types/` | rewritten per kind (below) | +| `tests/test_hashing/golden/cases.py`, `data/*.json` | fixtures; one regeneration | +| `docs/concepts/your-own-data-types.md`, `identity.md`, `function-pods.md` | two kinds, when to use each | +| `changelog.d/ITL-972.md`, `design_issues/ITL-972-UCK1.md` | fragment and entry; `changelog.d/ITL-881.md` and `design_issues/ITL-881-UCI1.md` from the merge are folded into them and removed | +| `CLAUDE.md`, `.zed/rules` | a "User classes" section with the two-kind rules | + +## Tasks + +### Task 0: Merge and baseline +- [ ] `git merge --no-ff origin/eywalker/itl-881-user-class-identity-impl`; resolve as in the + design; golden JSON: ITL-967 side. +- [ ] `uv run pytest tests/ -x -q`; record failures (expected: golden entries of user + classes and nothing else); commit the merge, push. + +### Task 1: Regression test for the measured case (fails first) +- [ ] `tests/test_user_types/test_measured_case.py`: QOld/QNew as structural classes; pod A + `-> QOld`, pod B `p: QNew` calling `p.describe()`; in memory and through a Delta-backed + `PipelineJob`; assert `type(p) is QNew` on both. Same pair as ordinary classes: B refuses + A's output on both paths. Show it failing. + +### Task 2: Exact classes +- [ ] Tests: name is module path (`__main__.X`, `__mp_main__` → `__main__`); schema check + refuses a re-defined class, a subclass at a base parameter, accepts the same object; + write refuses a value of another class; read by import; changed fields on read → + `TypeIdentityError` with the column; subclass keeps its fields; re-run cell does not raise. +- [ ] Implement; delete `user_types.py` and `register_type` tests; pydantic follows. + +### Task 3: Structured tokens +- [ ] Tests: token of each annotation form; logical-type field uses the identifier text; + nested class embeds the record; token → annotation round trip; order is identity. +- [ ] Implement in `user_class_identity.py`. + +### Task 4: `orcapod.dataclass` +- [ ] Tests: real dataclass; options pass through (`frozen`, `slots`, `kw_only`, `order`); + `init=False`, already-dataclass, pydantic rejected; `name=` validation; two classes of + one name and fields share identifier and hashes; different name → different; exact-class + field rejected; local class allowed. +- [ ] Implement `structural.py` and the factory; register the category. + +### Task 5: Conversion +- [ ] Tests: `from_instance` from ordinary and structural instances; exact field-name set; + type disagreement; nested and container conversion; `P(other)` sugar and the first-field + rule; `__post_init__` runs; pydantic refused; `inspect.signature(P)` unchanged. +- [ ] Implement; then the pod-input helper in `data_function.py` (sync, async). +- [ ] Task 1's tests pass. + +### Task 6: Reading without the class +- [ ] Test (child process, `tests/test_user_types/child_harness.py`): write with a + structural class, read in a process that does not define it → synthesized class with the + stored fields; nested, container, optional, logical-type fields; then define the class + and a pod parameter receives it. +- [ ] Implement synthesis. + +### Task 7: Carried-over behaviour still holds +- [ ] `tests/test_pipeline/test_type_identity_failure.py`, Delta compaction, nested `None`, + field order, workshop scenarios rewritten for the two kinds; operators that compare + Python types (`MergeJoin`, `Join`) with two structural classes of one identity. + +### Task 8: Golden +- [ ] Add fixtures; list entries that fail before regeneration, by cause; regenerate once; + check the diff against the list; `tests/test_hashing/test_hash_label_rule.py` passes. + +### Task 9: Docs, changelog, design issue, agent rules +- [ ] Pages (doc samples run in `tests/test_docs`), `changelog.d/ITL-972.md`, + `design_issues/ITL-972-UCK1.md`, `CLAUDE.md` + `.zed/rules`. + +### Task 10: Checks and PR +- [ ] `uv run ruff format .`, `uv run --with ruff ruff check .`, + `uv run --with basedpyright basedpyright` (exit code), `uv run pytest tests/`. +- [ ] Merge `origin/main` if it moved; push; PR against `main` ("Stacked on the ITL-967 + pull request; supersedes #292"); CodeRabbit rounds; `worker_done` with the PR URL. diff --git a/superpowers/specs/2026-11-10-user-class-identity-v0.3.md b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md new file mode 100644 index 000000000..b8faf1868 --- /dev/null +++ b/superpowers/specs/2026-11-10-user-class-identity-v0.3.md @@ -0,0 +1,420 @@ +# Identity of a user class in v0.3 (ITL-881) + +**Status:** implemented on this branch (PR #292). Approved by the coordinator November 10, +2026; P1, the identity-string format (fold, full-length hash) and the dropped naming switch +decided by the developer the same day. +**Revised November 11, 2026 (developer):** declared field **order is part of the identity**, +for dataclasses and pydantic models. This reverses the November 10 ruling "not field order". +Criterion: take the type's own equality seriously (a dataclass compares its fields as an +ordered tuple; a pydantic model dumps, iterates and serializes in declared order); a finer +identity can cost a recompute but cannot be incorrect. The sections below state the revised +rule. +**Issue:** ITL-881 (v0.3, a public release; tag target November 16, 2026). +**Builds on:** the findings in PR #288 (`superpowers/specs/2026-11-10-user-class-identity.md`, +experiments E01–E08). "W1"–"W15" are the workshop scenarios from §3.4 of that document. + +Each decision is marked **CONFIRMED** (the developer decided it, November 10, 2026) or +**PENDING** (the default this spec plans with; each pending item is a small, local change). + +| ID | Decision | Status | +|---|---|---| +| D-name | Nominal identity with a declared name; `__main__` classes default to the name only | CONFIRMED: developer decision 1 | +| D-fields | Field names, field types and declared field order are part of the identity | CONFIRMED: developer decision 2, revised November 11, 2026 (order is identity) | +| D-computed | Computed fields are not part of the identity in v0.3 | CONFIRMED as deferred: developer decision 2 defers it; v0.3 keeps today's stored fields | +| D-kind | Dataclass and pydantic are distinct kinds | CONFIRMED: developer decision 3 | +| D-rebind | Same identity re-registered: rebind silently | CONFIRMED: developer answer to S1 | +| D-raise | Stored shape disagreeing with registered shape raises, in v0.3 | CONFIRMED: developer answer to S2 | +| D-doc | A page documenting identity | CONFIRMED: developer answer to S4 | +| P1 | A field typed by another user class carries that class's identity as that class was registered | CONFIRMED: developer decision, November 10, 2026 | +| D-cycle | A self-referential or cyclic class is rejected with a clear `TypeError` | CONFIRMED by the coordinator from this proposal: Arrow cannot store a recursive struct | +| D-format | Identity string = name + `@` + the full 64-hex SHA-256 of the record (the "fold") | CONFIRMED: developer, November 10, 2026; alternative B (fields only in metadata) not pursued, appendix A | +| D-naming | No process-wide naming switch: fixed location rule, per-class `register_type(name=...)` or `register_type(qualified_name=False)` | CONFIRMED: developer dropped `set_type_naming`; a project-wide choice would live in the data context, not in v0.3 | +| D-mutation | `register_type` never modifies the class; same name again is a no-op; another name raises | CONFIRMED: developer | +| P2 | Public API name `register_type` and argument `qualified_name` | CONFIRMED: developer, November 10, 2026 | +| D-hash | The 25 golden entries in §e move; nothing else | CONFIRMED by the coordinator: follows from decision 2; the PR lists every moved entry | +| D-legacy | Reading a pre-0.3 user-class column directly raises a clear error | CONFIRMED by the coordinator: a consequence of the identity check; nothing extra is built | +| D-notice | One-time notice when a `__main__` class is first registered (S3) | CONFIRMED in scope by the coordinator; last item before the cuts | + +## a. The identity definition + +A user class's identity is an **identity record** with three components: + +```json +{"fields": [["x", "int"], ["y", "float"]], "kind": "dataclass", "name": "Point"} +``` + +### Name component — CONFIRMED (decision 1) + +The name is a string chosen once per class, by the first rule that applies: + +1. **Declared name.** The user declared one (§b). Used verbatim. +2. **Per-class choice without a name:** `register_type(qualified_name=False)` gives + `cls.__qualname__`; `register_type(qualified_name=True)` gives `module.qualname`. +3. **`__main__` class** (`cls.__module__` is `"__main__"` or `"__mp_main__"`, the name a + `multiprocessing` spawn child gives the main module, W14): `cls.__qualname__` + (`"Point"`, or `"Outer.Point"` for a nested class). +4. **Module class:** `f"{module}.{qualname}"` (`"pkg.models.Point"`). The module is + passed through `canonical_module_name`, which leaves user modules unchanged. + +Rules 3 and 4 are fixed (D-naming): no process-wide switch changes them, so a class's +identity never depends on whether or when some setting was called. + +A name must be non-empty, must not contain `@` or whitespace, and must not start with +`orcapod.` (reserved for built-in types such as `orcapod.path`). A declared name is +NFC-normalized, so one name typed in two Unicode forms is one name (Python already +normalizes class and field identifiers). With `qualified_name=True`, a `__main__` class is +named `__main__.` in a `multiprocessing` spawn child too. A class whose +`__qualname__` contains `` stays rejected in v0.3, even with a declared name +(scope, §g). + +### Kind component — CONFIRMED (decision 3) + +`"dataclass"` (`dataclasses.is_dataclass`, which includes `pydantic.dataclasses.dataclass`) +or `"pydantic"` (`pydantic.BaseModel` subclass). A dataclass and a pydantic model with the +same name and fields have different identity records, so they are different types. + +**Subclasses.** Each class has its own identity; a subclass never borrows its parent's. +Today the registry matches an unregistered class to a registered *parent* by +`issubclass`, so `class P3(Point)` with an extra field `z` is stored as `Point` and `z` is +silently dropped (reproduced on `main`, November 10, 2026). v0.3 matches dataclass and +pydantic logical types by exact class only. + +### Field component — CONFIRMED (decision 2; nested classes: P1) + +`fields` lists each **stored field's name** with its **type token**, as `[name, token]` +pairs in declared order. It is a JSON array, so field order enters the identity. The stored fields are exactly +the ones stored today: dataclass fields with `init=True`, pydantic `model_fields`. The +Arrow storage struct lists the fields in the same declared order, so two classes with the +same identity always have the same storage type. + +Type token of an annotation (from `typing.get_type_hints`, which strips `Annotated` +extras): + +| Annotation | Token | Example | +|---|---|---| +| builtin scalar | its name | `int`, `float`, `str`, `bool`, `bytes` | +| other class without a logical type (stdlib) | `canonical_type_name(cls)` | `datetime.datetime`, `decimal.Decimal` | +| type with a built-in logical type | its logical type name | `orcapod.path`, `orcapod.uuid`, `orcapod.file` | +| generic container | `origin[arg, …]`, origin by its canonical name, args in order | `list[int]`, `dict[str, float]`, `tuple[int, str]` (`typing.List[int]` gives `list[int]`) | +| `Optional` / union | member tokens sorted, `None` last, joined by ` \| ` | `int \| None` for `Optional[int]`, `int \| None` and `None \| int` | +| `Literal` | `Literal[...]` with the members' `repr`s sorted | `Literal['a', 'b']` | +| nested dataclass / pydantic model (P1) | that class's identity string (below), as that class is registered | `Inner@3c1d…` (full digest), `geo.Inner@…` | + +**Nested classes (P1, CONFIRMED).** The token of a field typed by another user class is +that class's own registered identity string: its name component however *that* class was +registered (declared name, name only, or `module.qualname`), its kind, and its field +component, recursively through the fingerprint. It is never re-derived from +`module.qualname` and never reduced to the storage struct. So renaming the inner class, +declaring it under another name, or changing its fields changes the outer identity. + +**Registration order does not matter.** `register_type` only records the declared name +and checks it (§b); the logical type, and so the identity, is built at first use, and the +outer class's build registers the inner class first (implicitly, by the location rule, if +it was never declared). Declaring every class before first use therefore gives the same +identities in any order: outer decorated before inner, inner declared later in the same +cell, inner registered implicitly by the outer's first use. Declaring a class after it +was used under a different name raises (§c). An identical notebook re-run of the inner +class rebinds it (§c) and leaves the outer identity unchanged. + +**Cycles (D-cycle).** A class that refers to itself, directly (`next: Node | None`) or +through other classes, is rejected at first use with `RecursiveTypeError` (a `TypeError` subclass) naming the +path (`Node.next -> Node`, `CycleA.b -> CycleB.a -> CycleA`). Today's converter already +raises `TypeError: Circular type dependency detected…`; v0.3 keeps the rejection and names +the path. Encoding the +back-reference by name would give a well-defined identity, but Arrow cannot store a +recursive struct, so there is nothing to identify. + +Deliberately **not** part of the identity: defaults and default factories, methods and +validators, properties and pydantic `computed_field`s, `init=False` dataclass fields, +`ClassVar`s, private attributes, docstrings, pydantic field aliases and +constraints (`Field(gt=0)`, `Annotated` metadata), `frozen`/`slots`/config flags, base +classes (only the resulting fields count). Computed fields are deferred (D-computed); a +later decision to include them adds a key that is absent when empty, so it would move +only classes that have computed fields. + +A type with a logical type contributes its logical type *name*, so those names are part of +user-class identities: adding or renaming a built-in logical type (for example a new +`orcapod.ndarray` from PR #289, where today an `np.ndarray` field gives the token +`numpy.ndarray`) moves the identity of every class with such a field. Built-in logical +type names are already a stability commitment (`orcapod.path`); after #289 lands this +branch adds a golden fixture with an ndarray field. + +The token is built from the Python annotation, not the Arrow type, because the developer +named field *types* and because the Arrow type cannot tell `str` from `orcapod.path` +(both `large_string`). + +### Identity string + +``` +identity string = f"{name}@{fingerprint}" +fingerprint = sha256(canonical_json(record)).hexdigest() # full 64 hex digits +canonical_json = json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + # hashed as UTF-8 +``` + +`Point(x: int, y: float)` as a dataclass gives +`Point@76d961e40619a836535e3fdebdead7ea65aa9413955a889f3d5b51a7a94b65eb` (the SHA-256 of +`{"fields":[["x","int"],["y","float"]],"kind":"dataclass","name":"Point"}`); with `y: int`, +`Point@008496cf…`; as a pydantic model with `y: float`, `Point@8a0f5b42…`. SHA-256 of a +documented JSON string, not the Orcapod semantic hasher, so the identity can be reproduced +by hand and does not depend on a hasher version. The full digest is used (developer): +there is no truncation rule to explain, and the name is not meant for people to type. +The length lives in one constant (`FINGERPRINT_HEX_DIGITS`) and the format in one function +(`identity_string`). + +Where the long name shows: the Arrow field metadata of stored columns, `extension<...>` in +Arrow type reprs, and Polars extension dtypes. It does not appear in table paths (those +use pipeline hashes) or in Orcapod's error messages, which describe a class as +`Point (dataclass) {x: int, y: float}`. A prefix-matching lookup for abbreviated names was +considered and not built: no API takes a logical type name, and the read path compares +full strings from stored metadata. + +### Where each component enters + +The identity string replaces today's `f"{module}.{qualname}"` as the class's logical +type name. Every place that used the old string now uses the new one; there is no +second mechanism. + +| Place | What it carries | How | +|---|---|---| +| Logical type name | identity string | `DataclassLogicalType.logical_type_name` / pydantic equivalent | +| Arrow extension name (`ARROW:extension:name`) | identity string | same string; one Arrow/Polars extension registration per identity | +| Arrow extension metadata | category plus the full record: `{"category":"orcapod.dataclass","identity":{…}}`, canonical JSON | read path uses it to find the class and to print the stored shape | +| Arrow storage type | struct of the stored fields, in declared order | derived from the annotations | +| Schema hash, source `pipeline_hash`, system-tag column names | identity string | `TypeObjectHandler` hashes a registered class as `type:` (unchanged code) | +| Pod signature hash, node `pipeline_hash`, DB table paths | identity string | via the schema/signature hashes | +| Per-row data hash, `content_hash` | identity string + record + values | the extension column is hashed with its extension name and metadata (unchanged code) | +| `list[Point]`, `Point \| None`, nested fields | identity string | container logical names embed the element's logical name; nested fields via the token (P1) | + +**Same name, different fields → different pipeline identity.** `Point(x: int, y: int)` +and `Point(x: int, y: float)` have different fingerprints, so different logical type +names, different schema hashes, different pod and node pipeline hashes, and different +table paths. They never share a table. That removes the "same table, different shape" +failure (W6, W10: today 0 rows plus log warnings): the changed class computes into its own +tables, and the old tables stay as they were. In one process the two shapes have two +extension names, so Arrow's and Polars' global extension registries (keyed by name) hold +both without conflict; that is why the fields must be in the extension *name*, not only +in its metadata. + +## b. Public API — `register_type` (D-naming, D-mutation) + +```python +import orcapod as op + +@op.register_type # default name: __main__ → "Point", module → "pkg.models.Point" +@dataclass +class Point: ... + +@op.register_type(name="Point") # declared name, the same wherever the class is defined +@dataclass +class Point: ... + +@op.register_type(qualified_name=False) # a module class named by its class name: "Point" +@dataclass +class Point: ... + +op.register_type(Point, name="Point") # call form, for classes you do not own +``` + +- `register_type` records the name for the class object in a process-wide weak-keyed + table. It never modifies the class (no attributes, no dataclass parameters, no pydantic + config) and returns the same object. It checks the kind, the name, `name` and + `qualified_name` not both given, and that the class is not local. The logical type is built + at first use, so declarations may come in any order (§a). Classes never declared are + registered on first use under the location rule. +- Registering the same class again with the same name is a no-op. A different name raises + `ValueError` naming both, including a declaration after the class was already used + under its default name: one class object has one name per process, and a registration + is never changed later. +- The public name lives in one place (`orcapod/user_types.py`, re-exported from + `orcapod`). + +**Name collisions (same kind, same name, two different class objects):** + +| Case | What happens | What the user sees | +|---|---|---| +| Same fields, same `module.qualname` (cell re-run, `%autoreload`, W1/W3/W4) | Same identity: the identity is rebound to the new class object (§c) | nothing | +| Same fields, different `module.qualname` (two modules declare `"Point"`, or use `qualified_name=False`) | Same identity by definition: they are the same type; values read back are built as the most recently registered class | one `UserWarning` per rebinding naming both classes. Warranted: values come back as a different class object than may be expected | +| Different fields (W2 with changed fields, or two modules) | Different identities: two types that happen to share a display name; no clash | nothing at registration. Reading picks the class whose identity matches (§d) | + +## c. Rebinding rule (S1) — CONFIRMED (developer answer to S1) + +- **Same identity string, new class object → rebind silently.** The registry rebinds the + identity (logical name and extension name) to a logical type built for the new class. + Earlier class objects keep their mapping, so instances created before the re-run still + convert. Arrow's global registration for that extension name is reused (same name, + storage type and metadata by construction). This fixes W1, W3, W4. +- **Different identity under the same name → a different type.** Nothing is rebound; + the new class gets its own logical type and extension name (W2: the next run computes + into new tables). +- A pod defined before the class cell was re-run still accepts the re-defined class: + schema compatibility treats two user classes with the same identity string as the same + type (`user_class_identity.equivalent_hints`, used by `check_schema_compatibility`). +- **Still raises:** + 1. One class object registered under two names (§b), including a declaration after + the class was already used under its default name. + 2. An invalid declared name, or `name` together with `qualified_name` (§a, §b). + 3. A local class (``), as today. + 4. A shape disagreement on read (§d, S2). + 5. A self-referential or cyclic class (D-cycle). + +## d. Read path — CONFIRMED behaviour (S2), lookup mechanism proposed + +A stored column carries the extension name `Point@9117beaa…` (full digest) and the record +in its metadata. To build values, the reader resolves a class: + +1. **Registered identity:** the extension name is registered in the reading context → use it. +2. **Declared or registered by name:** classes recorded under `(kind, name)` in this + process (by `register_type` or by first use); take the one whose identity string + matches. +3. **`__main__` lookup:** walk the name as an attribute path from + `sys.modules["__main__"]` (`"Point"`, `"Outer.Point"`), accepted if its identity matches. + This makes W12 (read notebook results from a script that defines the same `Point`) + work without a declaration. +4. **Import** (dotted name): import by the longest module prefix, as today, accepted if + its identity matches. + +Outcomes: + +- **No candidate found:** `ImportError` (kept), naming the declared name, the kind, the + stored fields, and how to fix it: "define the class and register it with + `op.register_type(Point, name='Point')` before reading". +- **Candidates found, none with a matching identity (S2):** `TypeIdentityError` + (new, a `ValueError` subclass in `orcapod.errors`) naming the column, the class name and + kind, the stored fields and the registered fields, for example "Column 'q': Stored + values of Point (dataclass) {x: int, y: int} do not match the registered class: Point + (dataclass) {x: int, y: float}. …". `ExtensionAwareDatabase` reads add the column name + (`register_discovered_logical_types`); the `ImportError` gets it too. +- **Same identity, different Arrow storage** (possible only if two data contexts map one + annotation to different Arrow types): `TypeIdentityError` naming the column, the class + and both struct types. The existing check in the synthesized + `__arrow_ext_deserialize__` raises it. Storage that differs only in large versus + non-large layouts, at any depth, is accepted: Delta compaction rewrites a struct field + `large_string` as `string` (found in review; on `main` this read failed into 0 rows). +- **Candidates are tested without registering them.** A class found by name that is not + yet used in the process is described without fixing its name, so a failed read never + stops the user from then declaring the class under the stored name. When a candidate + has the stored fields under another name, the error says to declare it. +- **Extension metadata is compared byte for byte**, like the extension name. It is part + of the per-row data hash already (`ARROW:extension:*` keys are hashed), so any change to + it is a hash change and needs the same approval; it is not a separate format risk. +- These errors propagate. `DeltaTableDatabase` today catches every read error and logs + "Error reading Delta table"; it re-raises `TypeIdentityError` in all five read paths + (`get_all_records`, the record lookups, the schema read), so a run can no longer report + success with 0 rows. Other read errors keep their current handling (out of + scope). The misleading "data may have been deleted externally" message is untouched + unless the path is reached for this reason (§g lists it as out of scope). +- **Pre-0.3 column** (D-legacy): an extension name without `@` and metadata + without `identity`, read directly (pipelines never reach these tables, §e): raise + `TypeIdentityError` saying the column was written by Orcapod before 0.3 under + `pkg.models.Point`, that v0.3 identifies the class by name and fields, and that the + pipeline recomputes into new tables. Not converted, not dropped. + +## e. Intended hash changes — CONFIRMED list, measured + +Every pinned value that involves a dataclass or pydantic type moves, because the identity +string changes for every user class (fields are now in it, for everyone). Measured on +`origin/main` (c5ab1e05) with E08 (`--ref origin/main`): changing the identity string of the +golden fixtures `GoldenPoint` and `GoldenModel` moves exactly these 25 entries, and the +`unchanged` control moves 0. Every identity-string change moves the same set (E08's +`__main__` and module-less variants list identical entries), so the v0.3 string moves +these and only these. + +**Measured after implementation** (golden files regenerated, compared with `origin/main` +c5ab1e05): 17 values change and 8 keys are replaced, exactly the entries below; 71 keys +are added (the 8 replacements and the new fixtures); `arrow.json` and `pipeline.json` are +byte-identical to `main`. Before regenerating, the golden test failed on exactly these 25 +entries plus the unpinned additions. + +| File | Entries | +|---|---| +| `semantic.json` (2) | `type.dataclass`, `type.pydantic` | +| `schema.json` (5) | `multi_field.logical_mix`, `single_field.dataclass.arrow_type`, `single_field.dataclass.schema_hash`, `single_field.pydantic.arrow_type`, `single_field.pydantic.schema_hash` | +| `datagram.json` (10 change) | `{dataclass,pydantic}.content_hash`, `.pipeline_hash`, `.system_tag_columns`, `.row[{"k":"r1"}].data_hash`, `.row[{"k":"r2"}].data_hash` | +| `datagram.json` (8 replaced) | `{dataclass,pydantic}.row[{"k":"r1"/"r2"}]._tag_record_id::` and `._tag_source_id::`: the keys embed the moved pipeline hash, so these 8 keys disappear and 8 keys with the new hash appear | + +`arrow.json` and `pipeline.json` have no user-class entries and do not move. Golden +additions (new keys only): a declared-name dataclass and a pydantic model with the same +name and fields (distinct values), a nested dataclass with a container field (P1), +`list[GoldenPoint]`, an `Optional` field, and field-order twins (`GoldenTwinAB`, +`GoldenTwinBA`, both `golden.Twin`) that pin **different** type, schema, data and pipeline +hashes, and a dataclass with a NumPy array field. + +**For the CHANGELOG (stated plainly):** "Orcapod 0.3 identifies a dataclass or pydantic +model by its name, its kind and its fields. Every pipeline whose data includes such a +class gets new pipeline hashes and table paths, so the first run under 0.3 recomputes +those pod and operator results into new tables. Tables written by earlier versions are +left untouched and are not read by 0.3 pipelines. There is no migration tool. Classes +defined in a notebook or script (`__main__`) are now identified by their name only, so +results survive moving the class into a module if it keeps its name and is registered +with `op.register_type(name=...)`." Results that do not involve user classes are +unaffected. + +## f. Interactions with PR #286 and PR #289 + +- **PR #286** (polars types) touches `builtin_handlers.py`, `logical_types/__init__.py`, + `contexts/data/v0.1.json`, `docs/concepts/function-pods.md`, the golden `cases.py` and + `semantic.json`/`schema.json`/`datagram.json`. This work does not touch + `builtin_handlers.py` or `v0.1.json` (the identity reaches hashing through the + logical type name), adds a new docs page instead of editing `function-pods.md` beyond a + link, and edits the golden files only by regeneration. +- **PR #289** (`orcapod.ndarray`) adds `array_format.py` and tests; no file in common, but + a field typed `np.ndarray` contributes its logical type name to the identity (§a), so + after #289 lands the ndarray token is re-checked and pinned with a golden fixture. +- **Preferred order:** #286 and #289 merge first. This branch then merges `origin/main`, + re-runs E08 against the new main to confirm the moved set, and regenerates the golden + files; the PR's golden diff must then show only the §e entries plus the additions. If + this PR merges first, #286 regenerates its own additive entries after merging main; + its user-class-free entries do not move. + +## g. Scope cuts, in order, if time runs short + +None were needed: everything below is implemented. The order that applied: + +1. The one-time `__main__` notice (S3, D-notice). +2. The collision `UserWarning` for same identity from a different `module.qualname`. +3. The `__main__` lookup on read (§d step 3). + +(`set_type_naming` was dropped by decision, not cut.) Not cut: the identity record and string, rebinding, `register_type`, the S2 raise, the +read lookup by declared name, the docs page and the CHANGELOG entry. + +Out of scope for v0.3: computed fields; local classes with a declared name; Enum, +NamedTuple and TypedDict support (D6 of the findings); the `DeltaTableDatabase` row-count +and shutdown-hang findings (§9 of the findings). + +## Workshop scenarios after v0.3 (tested: `tests/test_user_types/test_workshop_scenarios.py`) + +On `origin/main` 15 of the 19 scenario tests fail; all pass on this branch. The four that +pass on `main` pin behaviour v0.3 keeps (W5, W7, W12 with the same class, and a module +class without a declared name being a different type from the notebook class). + +| # | Scenario | v0.3 | +|---|---|---| +| W1/W3 | Re-run the cell, same fields (dataclass / pydantic) | silent rebind; cache hit | +| W2 | Re-run with changed fields | new type; recompute into new tables; old tables untouched | +| W4 | `%autoreload` of a module class | silent rebind | +| W5/W9 | Restart / second notebook, identical class | cache hit | +| W6/W10 | Restart / second notebook, changed fields | clean recompute, new tables; no 0-row success | +| W7 | Script as `__main__` twice | cache hit | +| W8 | `python -m pkg.pipeline` then `import pkg.pipeline` | miss unless declared (`"Point"` vs `"pkg.pipeline.Point"`); hit with `@op.register_type(name="Point")` | +| W11 | Notebook → module | hit if the module class declares the same name (`"Point"`) | +| W12 | Read notebook results from a script | works if the script defines a matching `Point` (top level or declared); otherwise the `ImportError` of §d | +| W14 | spawn child (`__mp_main__`) | same identity as `__main__` | +| W15 | Local class | `ValueError`, as today | + +## Appendix A. Alternative B (fields only in the extension metadata) — not pursued + +The developer asked whether the fields must be folded into the name, or whether the name +could stay `Point` with kind and fields in `ARROW:extension:metadata` and a type hash +composed from name, kind and fields. The developer then confirmed the fold (November 10, +2026) and the investigation stopped. Evidence gathered before it stopped: + +- Arrow's and Polars' extension registries are keyed by name only. Orcapod's registry + swallows `ArrowKeyError` on a second registration under the same name + (`registry.py`), so a second shape under one name would be read through the first + shape's `__arrow_ext_deserialize__`, which raises on the storage mismatch. B would need + a parametric extension type that accepts every shape and dispatches on the metadata; + not built or tested. +- Whether every Polars round trip keeps extension metadata as it keeps the name was not + verified. +- B would change the semantic hasher's type handler; the fold needs no hashing change. diff --git a/superpowers/specs/2026-11-12-itl-972-user-class-kinds.md b/superpowers/specs/2026-11-12-itl-972-user-class-kinds.md new file mode 100644 index 000000000..2b2587002 --- /dev/null +++ b/superpowers/specs/2026-11-12-itl-972-user-class-kinds.md @@ -0,0 +1,447 @@ +# Two kinds of user class: exact and structural (ITL-972) + +**Status:** APPROVED with changes by the developer, November 12, 2026 (section "Answers +to the open points"). The sections below state the decided design. +**Issue:** ITL-972 (v0.3; tag target November 16, 2026; workshop November 17). +**Supersedes:** the identity model of PR #292 (ITL-881, +`superpowers/specs/2026-11-10-user-class-identity-v0.3.md`). PR #292 stays open and is not +merged; its branch is merged into this one and reworked. +**Base:** tip of `eywalker/itl-967-v03-hash-step-combine-the-hash-changing` (c4b1e4a7) with +`eywalker/itl-881-user-class-identity-impl` (c6837a15) merged in. + +## Decided by the developer (November 12, 2026) — not reopened + +| ID | Decision | +|---|---| +| D1 | An ordinary dataclass is named by its full module path plus class name. | +| D2 | An ordinary dataclass is specific to the class object. Only an Orcapod dataclass is independent of the class object. A notebook re-run of an ordinary class makes a new class that earlier pods refuse. | +| D3 | An Orcapod dataclass is identified by its name plus its fields and their types. The decorator optionally takes a name; the default is the class name. The logical type metadata holds name, fields and types, enough to rebuild the class. | +| D4 | The constructor of a structural class accepts an instance of any Orcapod dataclass or ordinary dataclass and yields a proper instance of the structural class. | +| D5 | Dataclasses only. Pydantic models are completely class based. A structural pydantic model is ITL-973. | +| D-order | Field order is part of identity (developer, November 11, 2026, for PR #292). Kept: I found no reason to change it. | + +## Answers to the open points (developer, November 12, 2026) + +The design was sent with seven open points. The developer changed three; the coordinator +read the rest as approved as recommended. + +| ID | Question | Answer | +|---|---|---| +| O1 | Do an ordinary class's fields enter its identity? | **Yes** (as recommended). | +| O2 | Is a subclass column accepted at a parameter annotated with its ordinary base class? | **Accepted** (changed). "why refuse this? Per subclass rule of Python shouldn't this be accepted?" Exact class means no equivalence between *different* classes of the same description; it does not suspend subclassing. The subclass keeps its own identity and all its fields in storage. | +| O3 | Call form of the explicit conversion | **`P.from_instance(other)` only** (changed). The `P(other)` form is dropped: no `__init__` wrapper. | +| O4 | What `from_instance` accepts | Exactly the same field names and equal declared types; nested structural fields converted recursively (as recommended). | +| O5 | Is an ordinary dataclass column accepted at a structural parameter? | **No** (as recommended). | +| O6 | May `op.dataclass` add its marker and methods to the class it defines? | **Yes** (as recommended). | +| O7 | Do instances of two structural classes of one identity compare equal? | **Yes** (changed). "the whole point should be that structural classes identifies equality based on structural content as long as the two classes are of same structural identity." | +| — | Automatic conversion at pod inputs (proposed in §c.3) | **Removed** (changed). "let's not do auto conversion either — we should just check for structural identity as the basis of noting that they are compatible but simply pass it in as is without incurring some magical conversion." | + +**Consequence, stated plainly.** Orcapod never converts a value from one class to +another. A function annotated with structural class `B` may receive an instance of +another class of the same identity (in memory: the producer's object) or the class bound +to the type or built from metadata (from a store). **A structural class promises its +fields, not its methods.** The test of the November 12 measurement therefore asserts +compatibility, field access and structural equality on both paths, not `type(p) is QNew`. + +## Terms + +- **Ordinary class:** a class made by `dataclasses.dataclass` (or + `pydantic.dataclasses.dataclass`) and not by `orcapod.dataclass`; kind `dataclass`. +- **Pydantic model:** a `pydantic.BaseModel` subclass; kind `pydantic`. +- **Structural class:** a class made by `orcapod.dataclass`; kind `structural`. +- **Exact class:** the first two. Orcapod treats two exact classes as one type only when they + are the same class object. + +## Shared mechanics (all three kinds) + +Carried over from PR #292, changed where marked. + +**Identity record.** Canonical JSON (`sort_keys`, no whitespace, UTF-8, arrays keep order): + +```json +{"fields": [["x", "int"], ["y", "float"]], "kind": "structural", "name": "Point"} +``` + +**Identifier (ITL-935 rule: extension name plus extension metadata).** + +- Extension name and logical type name: `f"{name}@{sha256(canonical_json(record)).hexdigest()}"`. + The digest stays in the name (developer, November 10, 2026, D-format of PR #292) for a + technical reason that still holds: Arrow's and Polars' extension registries are keyed by + name only, so two shapes of one name in one process (a class edited and its cell re-run) + need two names. The digest is plain hex, no hash label. +- Extension metadata: `{"category": , "identity": }`, canonical JSON. + Categories: `orcapod.dataclass` (ordinary), `orcapod.pydantic`, and new + `orcapod.structural`. +- The whole identifier enters the type hash through `logical_type_identifier(lt).text` + (ITL-967 code, unchanged). No user-class identifier has a `"version"` key: the record is + the version. A change to the record format is a change of every user-class identifier. +- `readable_extension_metadata()` is the type's own metadata only. + +**Type tokens (CHANGED from PR #292: structured, so a class can be rebuilt from them).** +A field's token is JSON, built from the annotation after `typing.get_type_hints`: + +| Annotation | Token | +|---|---| +| `int`, `float`, `str`, `bool`, `bytes`, `None` | `"int"`, … `"None"` | +| stdlib class without a logical type (`datetime.datetime`) | its `canonical_type_name`, a string | +| type with a registered logical type (`Path`, `np.ndarray`, a third-party type) | `{"logical": ""}`: name **and** metadata, per the ITL-935 rule (PR #292 used the bare name) | +| `list[T]`, `set[T]`, `tuple[...]`, `dict[K, V]` | `{"generic": "list", "args": [, …]}`, origin by canonical name | +| `Optional[T]`, unions | `{"union": [, …]}`, members sorted by their canonical JSON, `"None"` last | +| `Literal[...]` | `{"literal": [, …]}`, sorted | +| another user class | `{"class": }` | + +A nested class contributes its **whole record**, not its identity string, so the outer +metadata describes every level without any class being importable. + +**Stored fields:** dataclass fields with `init=True`; pydantic `model_fields`. Storage is a +struct of them in declared order. Not part of identity: defaults, methods, validators, +`init=False` fields, `ClassVar`s, dataclass options, base classes. + +**Carried over unchanged from PR #292:** a subclass has its own identity and keeps its +fields (the registry never matches a user class by `issubclass`; audit finding R); field +order is identity; `TypeIdentityError` is not a `StoreReadError` and fails the run under +every `error_policy`; a nested class stored as `None` reads back; storage that differs only +in large versus non-large layout at any depth is accepted (Delta compaction); +`RecursiveTypeError` for a class that refers to itself; `tests/test_user_types`. + +## a. Ordinary dataclass (exact class) + +### a.1 Name + +`f"{module}.{qualname}"`, the module through `canonical_module_name` (D1). + +- A class in `__main__` is named `__main__.Point`. `__mp_main__` (a `multiprocessing` spawn + child) is written `__main__`, so a worker process finds the same type. Consequence: a + notebook class and the same class moved to `pkg/models.py` are different types + (`__main__.Point` versus `pkg.models.Point`); two scripts that each define + `__main__.Point` with the same fields share the type, as in v0.2.0. +- A local class (`` in the qualified name) is rejected, as today: it cannot be + looked up by module path. + +### a.2 Fields in or out (O1) + +**Variant IN (DECIDED).** Record `{"kind": "dataclass", "name": "pkg.models.Point", +"fields": [...]}`. + +- Hash: a field added, removed, retyped or reordered changes the identifier, so the schema + hash, pipeline hash and table paths. The edited class computes into new tables; the old + tables stay untouched and are no longer read by the pipeline. +- Storage: extension name `pkg.models.Point@`; metadata holds the record. +- It is what the ITL-935 rule requires: "change a type's identifier when, and only when, an + existing value would be stored or hashed differently". A changed field changes the stored + struct. +- One mechanism for all three kinds; the read check "stored fields equal the class's fields" + is a comparison of two identifiers. + +**Variant OUT (v0.2.0 behaviour; not chosen).** Record `{"kind", "name"}` only; extension name +`pkg.models.Point`, metadata `{"category":"orcapod.dataclass"}`, both exactly as on the +ITL-967 branch. + +- Hash: the identifier does not move when fields change, so pipeline hash and table path + stay. An edited class meets its old rows in the same table with another struct. Orcapod + must then raise on write (Delta schema mismatch) and on read (`TypeIdentityError` naming + the column) and the user must clear or move the store by hand. In v0.2.0 this case ended + as a run that reported success with 0 rows (W6, W10 of the ITL-881 findings). +- Storage: one extension name for every shape of the class. Within one process (edit a + field, re-run the cell) Arrow's registry holds the first shape; the second needs + `pa.unregister_extension_type` and a replacement, after which tables of the first shape + that pass through Polars or IPC are decoded with the wrong storage type. This is new, + untested code. +- Golden: no existing dataclass or pydantic entry moves relative to the ITL-967 branch. +- It contradicts the ITL-935 rule quoted above. + + +### a.3 Rebinding and declared names: what remains of PR #292 + +**Removed, explicitly:** + +- `op.register_type`, `orcapod/user_types.py`, declared names, `qualified_name=`. +- Name-only naming of `__main__` classes and the one-time `__main__` notice. +- Equivalence of two ordinary classes with the same identity in the schema check + (`equivalent_hints` no longer applies to exact classes). +- The read lookup among classes "declared or seen under a name", and the `UserWarning` + for the same identity from another module path. + +**Kept, reduced to one rule:** when a class object with the same module path and name as an +earlier one is used (a cell re-run, `importlib.reload`, `%autoreload`), the registry binds +the identifier to the **newest** class, and the old class object keeps its own binding so +its instances still convert. Without this, re-running a cell raises "already bound" (the +W1 failure of v0.2.0). It gives no equivalence: the two classes remain different types for +every check, and an edited class has another identifier anyway. + +**What the rule means for data already in memory (found while implementing).** An Arrow +table carries the identifier, not a class object, so the class of an Arrow-backed column +is whatever class the identifier is bound to *now*. After a class is defined again and +used, a source built before reports the new class as its column type and decodes its +values as the new class; a pod built for the old class refuses that source. That is the +module-path rule applied to memory as it applies to a store: data of `pkg.Point` is read +as `pkg.Point` as currently defined. Only a pod's own output schema holds a class object +(the annotation), so `A -> Old` feeding `B(p: New)` is refused naming the column. + +### a.4 Enforcing the exact class + +| Place | Rule | +|---|---| +| Schema compatibility (`schema_utils._accepts`, used at every pod input) | Two different exact classes are never equivalent, whatever their name and fields. Python's subtype rule still holds (O2): a column of a subclass is accepted at a parameter annotated with its base class, as `is_subhint` decides today. A class defined again is not a subclass of the earlier one, so a pod built for the earlier class refuses it; the error names the column. | +| Pod input | The value is passed as it is. No conversion. | +| Writing a value (`python_to_storage`) | `type(value) is python_type`, else `TypeError` naming both classes. A column's type is one class with one stored struct; a subclass instance has other fields, which would be lost (audit finding R). A subclass value belongs in a column of the subclass, which a base-class parameter accepts (O2). Found while implementing: `FunctionPodStream.as_table` converted a pod's output values by the types of the values, not by the pod's declared output types, so in memory a subclass instance returned for a base-class column was stored as the base without its own fields. It now converts the data columns by their declared types and raises like the stored path. | +| Reading stored values | The class is found by the stored name: the class bound in the reading context, else `sys.modules["__main__"]` for a `__main__.` name, else import by the longest module prefix. Its record, computed now, must equal the stored one. Not found: `ImportError`. Found with other fields: `TypeIdentityError` with the stored and the current fields. The database layer adds the column name (`register_discovered_logical_types` knows the column; the factory does not receive it). Never a fallback to the struct. | +| Subclass | Its own name, so its own identity, fields and tables. Never stored as its base. | +| Nested class | The outer record embeds the inner record (`{"class": …}`), so a change to the inner class changes the outer identifier. An ordinary class may have fields of any kind. | + +**O2 has no storage or hashing problem.** A subclass column keeps the subclass's +identifier in the stream's schema, hashes and stored tables; the base-class parameter only +decides compatibility. The function receives subclass instances in memory and from a store +alike, because the stored column is read as the subclass. + +The measured case (QOld, QNew; pod A `-> QOld`, pod B `p: QNew`) as ordinary classes: two +types, B refuses A's output at graph build, on both paths. + +## b. `orcapod.dataclass` (structural) + +### b.1 Public API + +```python +import orcapod as op + +@op.dataclass +class Point: + x: int + y: float + +@op.dataclass(name="geo.Point", frozen=True) +class Point: ... +``` + +- Decorator only, bare or with keywords. It calls `dataclasses.dataclass` with the given + options and returns a **real dataclass**: `dataclasses.is_dataclass`, `fields`, `asdict`, + `replace`, pattern matching all work. It is declared with `typing.dataclass_transform`, + so type checkers and editors see a dataclass. +- Options: every `dataclasses.dataclass` keyword is passed through (`repr`, `eq`, `order`, + `unsafe_hash`, `frozen`, `match_args`, `kw_only`, `slots`, `weakref_slot`). `init=False` + is rejected (`TypeError`): Orcapod builds instances through `__init__`. +- `name=`: optional. Default `cls.__name__`. Rules as in PR #292: non-empty, no `@`, `;` or + whitespace, not starting with `orcapod.`, NFC-normalized. +- It raises `TypeError` for a class that is already a dataclass or a pydantic model: the + kind is fixed where the class is defined, and there is no call form for classes you do + not own (wrap them: define a structural class with the same fields and construct it). +- A local class is allowed: nothing is looked up by module path. +- What it adds to the class (O6): a marker attribute with the name, the classmethod + `from_instance` (§c), and `__eq__` (§b.6). `__init__` is the one `dataclasses` generates. +- Field types allowed: scalars, types with a logical type, containers and unions of these, + and other **structural** classes. A field typed by an ordinary dataclass or a pydantic + model raises `TypeError` at first use, naming the field: a structural class must be + rebuildable from its metadata, and an exact class is not. (The reverse is allowed.) + +### b.2 Identifier + +Record `{"kind": "structural", "name": , "fields": [[field, token], …]}`, fields in +declared order (D-order). Extension name `Point@`, category `orcapod.structural`. +The module path is not part of it. + +- Two structural classes with the same name, fields, types and order: **one type**. Same + hashes, same tables, accepted for each other at every check. +- Different name, same fields: different types (D3: the name is part of it). +- Same name, different fields or order: different types that share a display name. They + coexist in one process (two extension names). +- A structural class and an ordinary class with the same name and fields: different types + (kind differs). + +### b.3 Metadata layout and reconstruction + +```json +{"category": "orcapod.structural", + "identity": {"kind": "structural", "name": "Sample", + "fields": [["id", "str"], + ["where", {"class": {"kind": "structural", "name": "Point", + "fields": [["x", "int"], ["y", "float"]]}}], + ["tags", {"generic": "list", "args": ["str"]}], + ["note", {"union": ["str", "None"]}], + ["file", {"logical": "orcapod.path;7b2276657273696f6e223a317d"}]]}} +``` + +Rebuilding a Python annotation from a token: scalars by a fixed table; a stdlib canonical +name by import; `logical` through the reading context's registry (the extension name must +be registered and the metadata readable, else `LogicalTypeIdentifierError`); `generic`, +`union`, `literal` recursively; `class` with kind `structural` by the rule of §b.4. The +rebuilt class's record must equal the stored record, checked before it is used. + +### b.4 Reading + +1. The identifier is bound in the reading context: use the bound class. +2. A structural class with that name was defined in this process (a weak table filled by + the decorator) and its record equals the stored one: use the most recently defined one. +3. Otherwise **synthesize** it: `dataclasses.make_dataclass` from the record, passed through + `op.dataclass(name=)`, module `orcapod.synthesized`. It is a normal + structural class: attribute access, `asdict`, equality, and it converts to and from any + class of the same shape. Nothing is imported and nothing can fail for a missing module. + +If the user defines a matching class later, it becomes the bound class when first used +(same rebinding rule as §a.3). Which class a *read* returns therefore depends on what is +defined; what a *pod parameter* receives does not (§c.3). + +### b.5 Schema compatibility + +Two structural classes are accepted for each other when their records are equal, at the top +level and inside generics and unions. The value is then passed **as it is**: Orcapod checks +compatibility and converts nothing. Everything else about the check is as for any type. + +### b.6 Instance equality (O7) + +`op.dataclass` installs `__eq__`: two instances are equal when their classes are the same +class, or two structural classes with equal records, and their compared fields +(`field(compare=True)`) are equal. Fields typed by structural classes compare the same +way, also inside lists and other containers, because their own `__eq__` is used. An +instance of a class with another identity, or of any other type, is not equal +(`NotImplemented`). + +- Not installed when `eq=False` is passed or the class body defines `__eq__`. +- `__hash__` stays what `dataclasses` generates (`frozen=True` or `unsafe_hash=True`): the + hash of the field tuple, which does not depend on the class object, so equal instances of + two classes have equal hashes. A class that is not hashable stays unhashable. +- One policy for every class of a type (coordinator, November 12, 2026, review round 2): + equality and hashing cover exactly the stored fields. `op.dataclass` refuses + `field(compare=False)`, `field(hash=...)` and an `init=False` field that takes part in + comparison, with a `TypeError` naming the field. A class synthesized from metadata is + hashable by its field values (`unsafe_hash=True`), so a defined and a synthesized class + agree on `==` and `hash` by construction, and nothing about options enters the record. +- Ordering (`order=True`) is unchanged: `<` between two classes is not supported. +- A class synthesized from stored metadata has the same `__eq__` and `from_instance`. + +## c. Explicit conversion (D4) + +### c.1 Call form (O3) + +`P.from_instance(other)`, a classmethod, is the only conversion. `P(...)` is the plain +field constructor that `dataclasses` generates. Nothing in Orcapod calls `from_instance`. + +### c.2 What `from_instance` accepts (O4) + +- `other` is an instance of a dataclass: ordinary or structural. A pydantic model, a dict + or anything else raises `TypeError` (D5). +- `other`'s stored field names equal `P`'s exactly. A missing or an extra field raises + `TypeError` naming the fields: dropping a field would lose data silently, and filling a + default would invent it. Order is not compared (fields are copied by name). +- Declared field types must agree: equal annotations, or, for a field `P` types with a + structural class, a dataclass instance that this rule converts recursively (also inside + `list`, `set`, `tuple`, `dict` values and `Optional`). Anything else raises `TypeError` + naming the field and both types. Values are not otherwise validated or coerced. +- The result is a new instance built by `P`'s own `__init__` (so `__post_init__` runs); + `other` is not modified. + +### c.3 No automatic conversion + +Orcapod passes values as they are, at pod inputs and everywhere else. Writing a value to a +structural column requires an instance of a structural class with the column's identity +(else `TypeError` saying to build one with `from_instance`). + +The measured case as structural classes: B accepts A's output on both paths. In memory it +receives the `QOld` object; from a store the class bound to the type. On both paths `p.x` +works and `p == QNew(p.x, p.y)` is true; `p.describe()` is not promised. + +### c.4 Ordinary instance at a structural parameter (O5) + +Not accepted by the schema check: the column's type is the ordinary class, which is another +type with another identifier. The user converts explicitly in a pod +(`def to_point(p: OldPoint) -> Point: return Point.from_instance(p)`). + +## d. Pydantic + +Exact class, the rules of §a with kind `pydantic` and category `orcapod.pydantic`: name is +module path plus class name; fields in the identity; no equivalence between different +classes in the schema check (subclassing as in §a.4); `type(value) is cls` on write; read by import and identifier comparison; a subclass is its +own type. No structural kind, no `op.register_type`. A pydantic instance is not accepted by +`from_instance`, and a structural class cannot have a pydantic field. + +Kept from PR #292: the shared `UserClassLogicalType` base and factory, `model_fields` as +the stored fields in declared order, building values with the model's constructor (so +validators run on read), `None` for a null nested model, the `TypeIdentityError` handling. +`pydantic.dataclasses.dataclass` classes are ordinary dataclasses. + +## e. What moves + +Stored layouts (`pdb_v2`, `rdb_v2`, `tdb_v2`, `odb_v1`, `sdb_v1`), the data context key and +the released-manifest pin do not change: this is a change of logical type identifiers only. +The new category `orcapod.structural` is registered in `contexts/logical_types.json`. + +**Relative to the ITL-967 branch** (dataclass and pydantic identifiers there: extension name +`module.qualname`, metadata `{"category": "orcapod.dataclass"}`): + +| Cause | What moves | +|---|---| +| The record (fields) enters every exact class's identifier (O1) | Every pinned value that involves `GoldenPoint` or `GoldenModel`: in `semantic.json` `type.dataclass`, `type.pydantic`; in `schema.json` `multi_field.logical_mix` and `single_field.{dataclass,pydantic}.{arrow_type,schema_hash}`; in `datagram.json` `{dataclass,pydantic}.{content_hash,pipeline_hash,system_tag_columns}`, the two rows' `data_hash`, and the keys that embed the moved pipeline hash. That was 25 entries when measured for PR #292 on `main`; I re-measure against the ITL-967 tip before regenerating and list each entry in the PR. For users: every pipeline with a dataclass or pydantic column gets new pipeline hashes and table paths. | +| New fixtures | Added keys only: structural classes (default name, explicit name, nested, container and optional fields, field-order twins with different values, two classes of one identity with equal values), an ordinary subclass, an ordinary class with a structural field. | +| Stored extension names and metadata | Exact classes: `module.qualname` becomes `module.qualname@`, metadata gains `identity`. Structural: new. A column written by v0.2.0 or by the ITL-967 branch read directly raises `TypeIdentityError` (no identity record), as in PR #292. | + +**Measured (November 12, 2026)**, golden files regenerated, compared key by key with the +golden files of the ITL-967 tip a0a3aeac (which contains `main` up to #311). Before the +first regeneration the golden test failed on exactly the 25 entries below (plus the +unpinned additions). + +| File | Changed values | Replaced keys | Added keys | +|---|---|---|---| +| `semantic.json` | 2: `type.dataclass`, `type.pydantic` | 0 | 10: `type.{structural, structural_nested, structural_with_array, structural_twin_ab, structural_twin_ba, structural_same_identity, pydantic_optional, dataclass_subclass, dataclass_with_structural, list_dataclass}` | +| `schema.json` | 5: `multi_field.logical_mix`, `single_field.{dataclass,pydantic}.{arrow_type,schema_hash}` | 0 | 20: `single_field..{arrow_type,schema_hash}` | +| `datagram.json` | 10: `{dataclass,pydantic}.{content_hash, pipeline_hash, system_tag_columns}` and both rows' `data_hash` | 8: `{dataclass,pydantic}.row[r1/r2]._tag_record_id::` and `._tag_source_id::` (the key embeds the moved pipeline hash; 8 keys disappear, 8 appear) | 89: the 8 replacement keys and 9 new fixtures × 9 keys | +| `arrow.json` | 0 | 0 | 0 | +| `pipeline.json` | 0 | 0 | 0 | + +Cause of all 25 moved entries: the identity record (kind, module-path name, ordered fields) +enters the identifier of `GoldenPoint` and `GoldenModel`. `structural` and +`structural_same_identity` (two classes of one identity) pin equal type, schema, pipeline +and row data hashes and equal system-tag column names (their source and record ids differ, +because each fixture's source has its own id); the field-order twins pin different values. + +**Relative to PR #292** (never merged, so no stored data exists): every user-class +identifier moves, for three causes: `__main__` classes are named `__main__.X` instead of +`X`; tokens are structured, a nested class embeds its record, and a logical-type field uses +the whole identifier instead of the bare name; declared names no longer exist. PR #292's +golden fixtures that use `register_type` (`golden.Declared`, `golden.Inner`/`Outer`, +`golden.WithArray`, `golden.Twin`) are replaced by structural fixtures; none of PR #292's +golden values survives. + +## Merge conflicts (ITL-967 tip + PR #292 branch) + +Six files. Proposed resolutions: + +| File | Conflict | Resolution | +|---|---|---| +| `databases/delta_lake_databases.py` | import list | union of both (obvious) | +| `logical_types/database_hooks.py` | ITL-967 walks per column and adds the column to `LogicalTypeIdentifierError`; PR #292 walks the whole schema and searches the columns afterwards for `TypeIdentityError`/`ImportError` | ITL-967's per-column loop, catching all three errors and adding the column there; PR #292's column search is dropped | +| `logical_types/registry.py` (`_deserialize`) | metadata mismatch: ITL-967 raises `LogicalTypeIdentifierError` against `readable_metadata`; PR #292 raises `TypeIdentityError` on inequality | ITL-967's check. PR #292's other change in that function (large/non-large tolerance at any depth; storage mismatch as `TypeIdentityError`) merged without conflict and is kept | +| `golden/data/{datagram,schema,semantic}.json` | both sides regenerated | take the ITL-967 side, regenerate once after implementation | + +## f. Risks, and the order of cuts + +Risks: + +1. Four days to the tag for a new public API and a hash change. The identity mechanics are + PR #292's tested code; the new parts are the tokens, the synthesized class, the + constructor wrapper and the pod-input helper. +2. Operators that compare Python types across streams (`MergeJoin`, `Join` on shared keys) + must use the same structural equivalence as the schema check, or two classes of one + identity are refused there. I audit every Python-type comparison PR #292 touched. +3. `slots=True` makes `dataclasses` build a new class; the marker and methods are added to + that class. Tested. +4. Writing is stricter than v0.2.0: a value is stored only as its own class (no duck-typed + values, no subclass instance in a base-class column). Intended by D2 but can surprise. +5. With no conversion, a function annotated with a structural class can receive an instance + without that class's methods. Documented as the contract of a structural class. +6. A synthesized class has no methods. Reading without the class defined gives data, not + behaviour; documented. + +Cuts, first cut first (each leaves a working, documented state): + +1. `Literal` tokens in structural classes (rejected with a clear error). +2. Recursive conversion of nested structural fields in `from_instance` (the nested value + must already be of the declared class). + +(`P(other)` and the automatic conversion at pod inputs, first on this list in the proposal, +were removed by decision.) + +Not cut: the two kinds, no equivalence of exact classes, the structural identifier and +metadata, structural equality, reading without the class (synthesis), the regression test +for the measured case, docs, changelog, golden list by cause. + +Out of scope: structural pydantic (ITL-973); migration of v0.2 data; computed fields; +Enum, NamedTuple, TypedDict. diff --git a/tests/test_contexts/test_data_context_v1.py b/tests/test_contexts/test_data_context_v1.py index b635bf82b..730d171d0 100644 --- a/tests/test_contexts/test_data_context_v1.py +++ b/tests/test_contexts/test_data_context_v1.py @@ -211,6 +211,7 @@ def test_logical_type_registration_is_unchanged_by_the_move() -> None: "list", "orcapod.dataclass", "orcapod.pydantic", + "orcapod.structural", "set", ] assert sorted( diff --git a/tests/test_databases/test_delta_read_failures.py b/tests/test_databases/test_delta_read_failures.py index e40ca3514..728e490da 100644 --- a/tests/test_databases/test_delta_read_failures.py +++ b/tests/test_databases/test_delta_read_failures.py @@ -11,10 +11,13 @@ anything is written, naming the column and both types; legitimate appends (same types in another layout, new or missing columns, unchanged extension storage) still succeed. +- A readable table whose stored user-class shape differs from the registered class + raises ``TypeIdentityError``, not ``StoreReadError`` (ITL-881). """ from __future__ import annotations +import dataclasses import datetime import json from collections.abc import Callable @@ -27,7 +30,8 @@ from deltalake.exceptions import DeltaProtocolError from orcapod.databases import DeltaTableDatabase -from orcapod.errors import InputValidationError, StoreReadError +from orcapod.errors import InputValidationError, StoreReadError, TypeIdentityError +from orcapod.sources import DictSource PATH = ("results", "t") @@ -496,3 +500,63 @@ def test_nullability_only_difference_is_accepted(self, base): back = self.read_back(base) assert back.num_rows == 2 assert back.schema.field("x").type == loose + + +# --------------------------------------------------------------------------- +# A readable table whose class differs is not an unreadable table (ITL-881) +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class _StoredShape: + """A registered user class; the table below stores another shape under its name.""" + + x: int + y: int + + +@pytest.fixture +def other_shape(base: Path) -> DeltaTableDatabase: + """A readable table storing ``_StoredShape``'s type name over another struct.""" + value: Any = _StoredShape(1, 2) + source = DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) + ext = source.as_table().schema.field("p").type + wrong = pa.struct([pa.field("x", pa.float64()), pa.field("y", pa.int64())]) + field = pa.field("x", wrong).with_metadata( + { + b"ARROW:extension:name": ext.extension_name.encode(), + b"ARROW:extension:metadata": ext.__arrow_ext_serialize__(), + } + ) + table = pa.table( + { + "__record_id": pa.array([b"a"], pa.large_binary()), + "x": pa.array([{"x": 1.5, "y": 2}], type=wrong), + }, + schema=pa.schema([pa.field("__record_id", pa.large_binary()), field]), + ) + deltalake.write_deltalake(str(table_dir(base)), table) + return DeltaTableDatabase(base_path=base) + + +READS_WITHOUT_VALUE_FILTER = { + name: read + for name, read in READS.items() + if name != "get_records_with_column_value" +} + + +@pytest.mark.parametrize( + "read", + READS_WITHOUT_VALUE_FILTER.values(), + ids=list(READS_WITHOUT_VALUE_FILTER), +) +def test_class_mismatch_is_type_identity_error_not_store_read_error(other_shape, read): + """A stored shape that differs from the registered class raises ``TypeIdentityError``. + + ``StoreReadError`` means the table cannot be read. Here the table is intact and the + class differs, so the identity error is raised as is, never wrapped. + """ + with pytest.raises(TypeIdentityError) as info: + read(other_shape) + assert not isinstance(info.value, StoreReadError) diff --git a/tests/test_docs/_doc_samples.py b/tests/test_docs/_doc_samples.py index 334fcc93e..e9044d6de 100644 --- a/tests/test_docs/_doc_samples.py +++ b/tests/test_docs/_doc_samples.py @@ -27,6 +27,7 @@ import contextlib import io import json +import linecache import re import sys import textwrap @@ -216,8 +217,18 @@ def run_blocks(blocks: list[Block], name: str = "") -> list[dict[str, Any]] results.append(result) continue captured = io.StringIO() + filename = f"{name}:{block.lineno}" + # Register the block's source so ``inspect.getsource`` (used to hash function + # pods) reads the block itself. Without it, ``linecache`` falls back to the + # ``__main__`` module's loader, i.e. this runner's own file (seen on 3.13). + linecache.cache[filename] = ( + len(block.source), + None, + block.source.splitlines(keepends=True), + filename, + ) try: - code = compile(block.source, f"{name}:{block.lineno}", "exec") + code = compile(block.source, filename, "exec") with contextlib.redirect_stdout(captured): exec(code, namespace) # noqa: S102 - running our own documentation except BaseException: # noqa: BLE001 - report SystemExit etc. as failures too diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 582f98c7b..9384e25ac 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -36,6 +36,7 @@ import pyarrow as pa import pydantic +from orcapod import dataclass as structural_dataclass from orcapod import function_pod CaseFn = Callable[[], dict[str, Any]] @@ -198,11 +199,104 @@ class GoldenModel(pydantic.BaseModel): count: int +@structural_dataclass(name="golden.Declared") +class GoldenDeclared: + """Structural class (ITL-972) with an optional field.""" + + label: str + score: float | None + + +class GoldenDeclaredModel(pydantic.BaseModel): + """Pydantic model with the same fields as ``GoldenDeclared``: another type.""" + + label: str + score: float | None + + +@structural_dataclass(name="golden.Inner") +class GoldenInner: + """Structural class nested in ``GoldenOuter``; its record enters the outer one.""" + + value: int + + +@structural_dataclass(name="golden.Outer") +class GoldenOuter: + """Structural class with a nested structural field and a container field.""" + + inner: GoldenInner + tags: list[str] + + +@structural_dataclass(name="golden.WithArray") +class GoldenWithArray: + """Structural class with a NumPy array field; the array's identifier enters its identity.""" + + label: str + values: np.ndarray # pyright: ignore[reportMissingTypeArgument] # the bare class is the annotation Orcapod stores + + +@structural_dataclass(name="golden.Twin") +class GoldenTwinAB: + """Structural ``golden.Twin``; same fields as ``GoldenTwinBA`` in another order. + + Declared field order is part of the identity (ITL-881), so the twins are two types + and must pin different values. + """ + + a: int + b: str + + +@structural_dataclass(name="golden.Twin") +class GoldenTwinBA: + """Structural ``golden.Twin``; must pin values different from ``GoldenTwinAB``'s.""" + + b: str + a: int + + +@structural_dataclass(name="golden.Declared") +class GoldenDeclaredAgain: + """A second class of the type ``golden.Declared``: must pin ``GoldenDeclared``'s values.""" + + label: str + score: float | None + + +@dataclasses.dataclass +class GoldenChild(GoldenPoint): + """Ordinary subclass of ``GoldenPoint``: its own type, with its own field.""" + + z: int = 0 + + +@dataclasses.dataclass +class GoldenHolder: + """Ordinary dataclass with a structural field; the inner record enters its identity.""" + + inner: GoldenInner + note: str + + def register_logical_classes() -> None: """Register the dataclass and pydantic fixtures with the default type converter.""" tc = _ctx().type_converter tc.register_python_class(GoldenPoint) tc.register_python_class(GoldenModel) + for cls in ( + GoldenDeclared, + GoldenDeclaredModel, + GoldenOuter, + GoldenWithArray, + GoldenTwinAB, + GoldenTwinBA, + GoldenDeclaredAgain, + GoldenChild, + GoldenHolder, + ): + tc.register_python_class(cls) PATH_A = Path("/golden/data/a.txt") @@ -292,6 +386,16 @@ def _semantic_types() -> dict[str, Any]: "set_uuid": set[uuid.UUID], "dataclass": GoldenPoint, "pydantic": GoldenModel, + "structural": GoldenDeclared, + "pydantic_optional": GoldenDeclaredModel, + "structural_nested": GoldenOuter, + "structural_with_array": GoldenWithArray, + "structural_twin_ab": GoldenTwinAB, + "structural_twin_ba": GoldenTwinBA, + "structural_same_identity": GoldenDeclaredAgain, + "dataclass_subclass": GoldenChild, + "dataclass_with_structural": GoldenHolder, + "list_dataclass": list[GoldenPoint], } @@ -703,6 +807,42 @@ def _datagram_rows() -> dict[str, list[dict[str, Any]]]: ], "list_path": [{"k": "r1", "c": [PATH_A, PATH_B]}, {"k": "r2", "c": []}], "set_uuid": [{"k": "r1", "c": {UUID_A, UUID_B}}, {"k": "r2", "c": {UUID_A}}], + "structural": [ + {"k": "r1", "c": GoldenDeclared("a", 0.5)}, + {"k": "r2", "c": GoldenDeclared("b", None)}, + ], + "pydantic_optional": [ + {"k": "r1", "c": GoldenDeclaredModel(label="a", score=0.5)}, + {"k": "r2", "c": GoldenDeclaredModel(label="b", score=None)}, + ], + "structural_nested": [ + {"k": "r1", "c": GoldenOuter(GoldenInner(1), ["x", "y"])}, + {"k": "r2", "c": GoldenOuter(GoldenInner(2), [])}, + ], + "structural_with_array": [ + {"k": "r1", "c": GoldenWithArray("a", np.arange(3, dtype=np.float32))}, + {"k": "r2", "c": GoldenWithArray("b", np.zeros((2, 2), dtype=np.int64))}, + ], + "structural_twin_ab": [ + {"k": "r1", "c": GoldenTwinAB(1, "x")}, + {"k": "r2", "c": GoldenTwinAB(2, "y")}, + ], + "structural_twin_ba": [ + {"k": "r1", "c": GoldenTwinBA("x", 1)}, + {"k": "r2", "c": GoldenTwinBA("y", 2)}, + ], + "structural_same_identity": [ + {"k": "r1", "c": GoldenDeclaredAgain("a", 0.5)}, + {"k": "r2", "c": GoldenDeclaredAgain("b", None)}, + ], + "dataclass_subclass": [ + {"k": "r1", "c": GoldenChild(1, 0.5, 7)}, + {"k": "r2", "c": GoldenChild(2, 1.5)}, + ], + "dataclass_with_structural": [ + {"k": "r1", "c": GoldenHolder(GoldenInner(1), "a")}, + {"k": "r2", "c": GoldenHolder(GoldenInner(2), "b")}, + ], } diff --git a/tests/test_hashing/golden/data/datagram.json b/tests/test_hashing/golden/data/datagram.json index 2c1a781f4..961d8181f 100644 --- a/tests/test_hashing/golden/data/datagram.json +++ b/tests/test_hashing/golden/data/datagram.json @@ -89,17 +89,41 @@ "_tag_source_id::0709635887fd551e9d40e296d3e05d558ca07855425ab2b09aec9f22a03e2003", "_tag_record_id::0709635887fd551e9d40e296d3e05d558ca07855425ab2b09aec9f22a03e2003" ], - "dataclass.content_hash": "semantic_v1:2dcdc68366a74b0e3923b9579dbc3927d6cc51a99019f5cfbcb225d3fe16d28b", - "dataclass.pipeline_hash": "semantic_v1:a2b61bcf492e6d8d76054829b62a9ec36da3ae05bd3b07b060820ff50069267a", - "dataclass.row[{\"k\":\"r1\"}]._tag_record_id::a2b61bcf492e6d8d76054829b62a9ec36da3ae05bd3b07b060820ff50069267a": "26db93a5eab45b0485c2699148b0600d", - "dataclass.row[{\"k\":\"r1\"}]._tag_source_id::a2b61bcf492e6d8d76054829b62a9ec36da3ae05bd3b07b060820ff50069267a": "golden_dataclass", - "dataclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000027b15d8db313a05d7a3f0af1cfa788fd730bb3d47eacb201de61f4c24b9a2f0bd", - "dataclass.row[{\"k\":\"r2\"}]._tag_record_id::a2b61bcf492e6d8d76054829b62a9ec36da3ae05bd3b07b060820ff50069267a": "71d5c6ce2f7b5b5b9dfdc7a61a7df366", - "dataclass.row[{\"k\":\"r2\"}]._tag_source_id::a2b61bcf492e6d8d76054829b62a9ec36da3ae05bd3b07b060820ff50069267a": "golden_dataclass", - "dataclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002abd8873e34ca7ee3f967b31d5fc3e56013d151495ae59d9a53f856f81deacc85", + "dataclass.content_hash": "semantic_v1:20d7825b4db7962d90fa509235fa00babf43d944115f41679157da0acc7cf24b", + "dataclass.pipeline_hash": "semantic_v1:c6ec2b120f34f587b02ab1a4b18f72e5927992e87c32e2f052a3e40ca4d98d2a", + "dataclass.row[{\"k\":\"r1\"}]._tag_record_id::c6ec2b120f34f587b02ab1a4b18f72e5927992e87c32e2f052a3e40ca4d98d2a": "26db93a5eab45b0485c2699148b0600d", + "dataclass.row[{\"k\":\"r1\"}]._tag_source_id::c6ec2b120f34f587b02ab1a4b18f72e5927992e87c32e2f052a3e40ca4d98d2a": "golden_dataclass", + "dataclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002a189082cf31fa5967e7ba24c3329df2f21edc1b47d5ec4e3bdf47dc665c3ae82", + "dataclass.row[{\"k\":\"r2\"}]._tag_record_id::c6ec2b120f34f587b02ab1a4b18f72e5927992e87c32e2f052a3e40ca4d98d2a": "71d5c6ce2f7b5b5b9dfdc7a61a7df366", + "dataclass.row[{\"k\":\"r2\"}]._tag_source_id::c6ec2b120f34f587b02ab1a4b18f72e5927992e87c32e2f052a3e40ca4d98d2a": "golden_dataclass", + "dataclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:00000263cccf44a6f5d200ada2f518eaacee8b3f994759fc7598fd3091e8038ccf8d7c", "dataclass.system_tag_columns": [ - "_tag_source_id::a2b61bcf492e6d8d76054829b62a9ec36da3ae05bd3b07b060820ff50069267a", - "_tag_record_id::a2b61bcf492e6d8d76054829b62a9ec36da3ae05bd3b07b060820ff50069267a" + "_tag_source_id::c6ec2b120f34f587b02ab1a4b18f72e5927992e87c32e2f052a3e40ca4d98d2a", + "_tag_record_id::c6ec2b120f34f587b02ab1a4b18f72e5927992e87c32e2f052a3e40ca4d98d2a" + ], + "dataclass_subclass.content_hash": "semantic_v1:62dc85f3a332c2c4317a7f20d251446949a781d5491381bb7d80a0a745891580", + "dataclass_subclass.pipeline_hash": "semantic_v1:56ebb9f39b80e7141340169dfdc5e6a6813ab2474a6a26160af4038f4b5f817e", + "dataclass_subclass.row[{\"k\":\"r1\"}]._tag_record_id::56ebb9f39b80e7141340169dfdc5e6a6813ab2474a6a26160af4038f4b5f817e": "f40a52839301533a88ffebedc90242e2", + "dataclass_subclass.row[{\"k\":\"r1\"}]._tag_source_id::56ebb9f39b80e7141340169dfdc5e6a6813ab2474a6a26160af4038f4b5f817e": "golden_dataclass_subclass", + "dataclass_subclass.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000023c2f4c89eec462ec33e78c575e6c3d280131e5ed0111aa5a3c340ab0d0b019c8", + "dataclass_subclass.row[{\"k\":\"r2\"}]._tag_record_id::56ebb9f39b80e7141340169dfdc5e6a6813ab2474a6a26160af4038f4b5f817e": "a318bd0decd25b518bc98e4997106608", + "dataclass_subclass.row[{\"k\":\"r2\"}]._tag_source_id::56ebb9f39b80e7141340169dfdc5e6a6813ab2474a6a26160af4038f4b5f817e": "golden_dataclass_subclass", + "dataclass_subclass.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002664624c2e125a3bd413239428c57d31212414848687c7d4610a65fc948deaca4", + "dataclass_subclass.system_tag_columns": [ + "_tag_source_id::56ebb9f39b80e7141340169dfdc5e6a6813ab2474a6a26160af4038f4b5f817e", + "_tag_record_id::56ebb9f39b80e7141340169dfdc5e6a6813ab2474a6a26160af4038f4b5f817e" + ], + "dataclass_with_structural.content_hash": "semantic_v1:64df71db1af4d5af5bb4f9e497bc02fd58b39dd883cdd94936ec536213933288", + "dataclass_with_structural.pipeline_hash": "semantic_v1:18c4521babb1a868f9ccf1ca69c611c6e5ba58bc300df52a84badae13b923b49", + "dataclass_with_structural.row[{\"k\":\"r1\"}]._tag_record_id::18c4521babb1a868f9ccf1ca69c611c6e5ba58bc300df52a84badae13b923b49": "0b8c4f45949b5a7da318283c9b994348", + "dataclass_with_structural.row[{\"k\":\"r1\"}]._tag_source_id::18c4521babb1a868f9ccf1ca69c611c6e5ba58bc300df52a84badae13b923b49": "golden_dataclass_with_structural", + "dataclass_with_structural.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002b83c7531d38f9ba2787e702f7eb1c6739be651d06d995bd97a66c693361e6035", + "dataclass_with_structural.row[{\"k\":\"r2\"}]._tag_record_id::18c4521babb1a868f9ccf1ca69c611c6e5ba58bc300df52a84badae13b923b49": "d4274a0ebdbe5a95b918e4d94928a816", + "dataclass_with_structural.row[{\"k\":\"r2\"}]._tag_source_id::18c4521babb1a868f9ccf1ca69c611c6e5ba58bc300df52a84badae13b923b49": "golden_dataclass_with_structural", + "dataclass_with_structural.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:0000020f8a663edc2d1e735e95df2be4aa0ed4fbec9040ded7456b08449dd3c43ff50b", + "dataclass_with_structural.system_tag_columns": [ + "_tag_source_id::18c4521babb1a868f9ccf1ca69c611c6e5ba58bc300df52a84badae13b923b49", + "_tag_record_id::18c4521babb1a868f9ccf1ca69c611c6e5ba58bc300df52a84badae13b923b49" ], "dataframe.content_hash": "semantic_v1:965ad5065b7660d062b6d08b64d45424e52f2457c5b6403157a57b92fb14c633", "dataframe.pipeline_hash": "semantic_v1:09e21beb4bc98046eb825d92f94de9a313d22fc6d43d2309ea042381fc78ebb5", @@ -173,17 +197,29 @@ "_tag_source_id::53230401cc2b625854422548d42e634a004812f0e2ab7531fd7a4ddd226b8192", "_tag_record_id::53230401cc2b625854422548d42e634a004812f0e2ab7531fd7a4ddd226b8192" ], - "pydantic.content_hash": "semantic_v1:5b05dc5b24189afbebafbdfe2b853442350d8c8e837a45cc08fc30ba5b2b4ca2", - "pydantic.pipeline_hash": "semantic_v1:32facd7969e882139293fafe5677d896f37debab969e8c5b3abeaeb2ef8317fa", - "pydantic.row[{\"k\":\"r1\"}]._tag_record_id::32facd7969e882139293fafe5677d896f37debab969e8c5b3abeaeb2ef8317fa": "cb9bbf1a138d5c848e86f99bab7c0acb", - "pydantic.row[{\"k\":\"r1\"}]._tag_source_id::32facd7969e882139293fafe5677d896f37debab969e8c5b3abeaeb2ef8317fa": "golden_pydantic", - "pydantic.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002459c1d7ed12f91d7f292eafe1514160775ff885d92151dcf0da2ecf256dbbf1a", - "pydantic.row[{\"k\":\"r2\"}]._tag_record_id::32facd7969e882139293fafe5677d896f37debab969e8c5b3abeaeb2ef8317fa": "10c1dbaba95a5c5fa7c14661231224eb", - "pydantic.row[{\"k\":\"r2\"}]._tag_source_id::32facd7969e882139293fafe5677d896f37debab969e8c5b3abeaeb2ef8317fa": "golden_pydantic", - "pydantic.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002ce57007b0730b902132be534f05b27828f588bea325de4d54876721c6c5c7463", + "pydantic.content_hash": "semantic_v1:1e98599a90054ae7570a2c4b25bb4cd6d9ac5fe74f2f2ccc7c3c7b2db00f7dd2", + "pydantic.pipeline_hash": "semantic_v1:6a6cedf20d8149a5f8656ddf56240302b7a1e7b9a7da7a65301e1f73ae5ed166", + "pydantic.row[{\"k\":\"r1\"}]._tag_record_id::6a6cedf20d8149a5f8656ddf56240302b7a1e7b9a7da7a65301e1f73ae5ed166": "cb9bbf1a138d5c848e86f99bab7c0acb", + "pydantic.row[{\"k\":\"r1\"}]._tag_source_id::6a6cedf20d8149a5f8656ddf56240302b7a1e7b9a7da7a65301e1f73ae5ed166": "golden_pydantic", + "pydantic.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:00000232c57135bf6e7e881f80760cd79714e9940d944c6ad9626a291a762c48cf5f8d", + "pydantic.row[{\"k\":\"r2\"}]._tag_record_id::6a6cedf20d8149a5f8656ddf56240302b7a1e7b9a7da7a65301e1f73ae5ed166": "10c1dbaba95a5c5fa7c14661231224eb", + "pydantic.row[{\"k\":\"r2\"}]._tag_source_id::6a6cedf20d8149a5f8656ddf56240302b7a1e7b9a7da7a65301e1f73ae5ed166": "golden_pydantic", + "pydantic.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002186225047e9c6a0da98110c85ee5d63a0b7e4f0411b145e2dd3cc9d3d2739864", "pydantic.system_tag_columns": [ - "_tag_source_id::32facd7969e882139293fafe5677d896f37debab969e8c5b3abeaeb2ef8317fa", - "_tag_record_id::32facd7969e882139293fafe5677d896f37debab969e8c5b3abeaeb2ef8317fa" + "_tag_source_id::6a6cedf20d8149a5f8656ddf56240302b7a1e7b9a7da7a65301e1f73ae5ed166", + "_tag_record_id::6a6cedf20d8149a5f8656ddf56240302b7a1e7b9a7da7a65301e1f73ae5ed166" + ], + "pydantic_optional.content_hash": "semantic_v1:3f16874b1e29580075f7a2493a9f3274790df542484a53ba3b43e8faccb78e98", + "pydantic_optional.pipeline_hash": "semantic_v1:a02b28f92bf8b20e1baebaacf04668881ade1533ba4d30ce15a2ada6079e2757", + "pydantic_optional.row[{\"k\":\"r1\"}]._tag_record_id::a02b28f92bf8b20e1baebaacf04668881ade1533ba4d30ce15a2ada6079e2757": "c70b54e9e5e45706a31d732f0b9dd5a6", + "pydantic_optional.row[{\"k\":\"r1\"}]._tag_source_id::a02b28f92bf8b20e1baebaacf04668881ade1533ba4d30ce15a2ada6079e2757": "golden_pydantic_optional", + "pydantic_optional.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002122b76a3c80bb68e681bc77f0352deec2c135abf1cf59e6a3403b34220916d5a", + "pydantic_optional.row[{\"k\":\"r2\"}]._tag_record_id::a02b28f92bf8b20e1baebaacf04668881ade1533ba4d30ce15a2ada6079e2757": "089055e7f0cc533a8cd0fbc4ef2b98a2", + "pydantic_optional.row[{\"k\":\"r2\"}]._tag_source_id::a02b28f92bf8b20e1baebaacf04668881ade1533ba4d30ce15a2ada6079e2757": "golden_pydantic_optional", + "pydantic_optional.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002d7f6469d42cf4549895e782b93f2c4d9c2ed2386bafdb7702f7af0af12f6c6c0", + "pydantic_optional.system_tag_columns": [ + "_tag_source_id::a02b28f92bf8b20e1baebaacf04668881ade1533ba4d30ce15a2ada6079e2757", + "_tag_record_id::a02b28f92bf8b20e1baebaacf04668881ade1533ba4d30ce15a2ada6079e2757" ], "scalars.content_hash": "semantic_v1:41f441950183d3ca7a3fbb9bb1ada5c050e889afab8a2c5f00c6b4f490018d97", "scalars.pipeline_hash": "semantic_v1:a7cdef49c8d02a3910137a97c6deac6bc87493cadc102bfce49cb195725fae7a", @@ -209,6 +245,78 @@ "_tag_source_id::5b561315f1796dce0f0cd9a4a98df81a781ca0c0a2c71cefd349418aeaebce07", "_tag_record_id::5b561315f1796dce0f0cd9a4a98df81a781ca0c0a2c71cefd349418aeaebce07" ], + "structural.content_hash": "semantic_v1:c10515d11e1ba4f48cd49d48eadf9507f4d599a9055f1d949a37e51ff1c6a042", + "structural.pipeline_hash": "semantic_v1:857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120", + "structural.row[{\"k\":\"r1\"}]._tag_record_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120": "4d9ebef6bdad53b2a2e86517e90cf715", + "structural.row[{\"k\":\"r1\"}]._tag_source_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120": "golden_structural", + "structural.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000026f26f6153d35674741279d65fee4d3a1a2170b65a0a85f590b006235a166605a", + "structural.row[{\"k\":\"r2\"}]._tag_record_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120": "70d0c948fbc55d2f993116d1fcda5a1e", + "structural.row[{\"k\":\"r2\"}]._tag_source_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120": "golden_structural", + "structural.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:00000259dfbb9f5e924fc30c7478783124aacb8996340bb9cc66faa3b22aa93cc48841", + "structural.system_tag_columns": [ + "_tag_source_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120", + "_tag_record_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120" + ], + "structural_nested.content_hash": "semantic_v1:96b799976fc92ceb1a8565b0ceba06f94c8c0d196325888bcdd5518ce4a1e169", + "structural_nested.pipeline_hash": "semantic_v1:dd2e821e26fd11da77a896b9f023cce75f76392e5ce4925973175ea63656dc81", + "structural_nested.row[{\"k\":\"r1\"}]._tag_record_id::dd2e821e26fd11da77a896b9f023cce75f76392e5ce4925973175ea63656dc81": "352fad85ab315ee8a1c6a55b48f0e9f5", + "structural_nested.row[{\"k\":\"r1\"}]._tag_source_id::dd2e821e26fd11da77a896b9f023cce75f76392e5ce4925973175ea63656dc81": "golden_structural_nested", + "structural_nested.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000020b9676a7774b7540c60c035d3e9ffc369be1a0deaa085c5da2f803c2e35bfcf8", + "structural_nested.row[{\"k\":\"r2\"}]._tag_record_id::dd2e821e26fd11da77a896b9f023cce75f76392e5ce4925973175ea63656dc81": "79fcd56a6f0d5c838a0aae83792dc479", + "structural_nested.row[{\"k\":\"r2\"}]._tag_source_id::dd2e821e26fd11da77a896b9f023cce75f76392e5ce4925973175ea63656dc81": "golden_structural_nested", + "structural_nested.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:0000021b63f2aec8215175d6790b0d640399ad6baf2353962c5d89b8138178e1040db9", + "structural_nested.system_tag_columns": [ + "_tag_source_id::dd2e821e26fd11da77a896b9f023cce75f76392e5ce4925973175ea63656dc81", + "_tag_record_id::dd2e821e26fd11da77a896b9f023cce75f76392e5ce4925973175ea63656dc81" + ], + "structural_same_identity.content_hash": "semantic_v1:c13c20bc3193393acbf3383f9efe5efcdd0ab561261430e86afe947c4f72aae5", + "structural_same_identity.pipeline_hash": "semantic_v1:857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120", + "structural_same_identity.row[{\"k\":\"r1\"}]._tag_record_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120": "8844c5489e5153f19ac9a4294769691a", + "structural_same_identity.row[{\"k\":\"r1\"}]._tag_source_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120": "golden_structural_same_identity", + "structural_same_identity.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000026f26f6153d35674741279d65fee4d3a1a2170b65a0a85f590b006235a166605a", + "structural_same_identity.row[{\"k\":\"r2\"}]._tag_record_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120": "c72be70d2ac25159ac4f54aae6736db1", + "structural_same_identity.row[{\"k\":\"r2\"}]._tag_source_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120": "golden_structural_same_identity", + "structural_same_identity.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:00000259dfbb9f5e924fc30c7478783124aacb8996340bb9cc66faa3b22aa93cc48841", + "structural_same_identity.system_tag_columns": [ + "_tag_source_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120", + "_tag_record_id::857220ab510aa48c20a6c9b9fdde468ff50b90b435fec0c37360700488079120" + ], + "structural_twin_ab.content_hash": "semantic_v1:92a5912742f7b55bc5b89f19d204e59bfc01c2a0b4983813d6edfd28f6ea286b", + "structural_twin_ab.pipeline_hash": "semantic_v1:0e5ac28965fa4f5d9bb4b586a31125ca50922133e21ef585cba1cd29023d7058", + "structural_twin_ab.row[{\"k\":\"r1\"}]._tag_record_id::0e5ac28965fa4f5d9bb4b586a31125ca50922133e21ef585cba1cd29023d7058": "176b469f28095d978351d0fd8ea77d45", + "structural_twin_ab.row[{\"k\":\"r1\"}]._tag_source_id::0e5ac28965fa4f5d9bb4b586a31125ca50922133e21ef585cba1cd29023d7058": "golden_structural_twin_ab", + "structural_twin_ab.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002a2def349b317a21e8d9220ecdd5080cfdb53248e75314b46340ba6097ba15aed", + "structural_twin_ab.row[{\"k\":\"r2\"}]._tag_record_id::0e5ac28965fa4f5d9bb4b586a31125ca50922133e21ef585cba1cd29023d7058": "c71a1c63958d5cfba09eaa076cd220a0", + "structural_twin_ab.row[{\"k\":\"r2\"}]._tag_source_id::0e5ac28965fa4f5d9bb4b586a31125ca50922133e21ef585cba1cd29023d7058": "golden_structural_twin_ab", + "structural_twin_ab.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:000002a2bc57737de3b384b8863bb523bfb5c8dc902144c442e9ba35b57dab0d28a6cb", + "structural_twin_ab.system_tag_columns": [ + "_tag_source_id::0e5ac28965fa4f5d9bb4b586a31125ca50922133e21ef585cba1cd29023d7058", + "_tag_record_id::0e5ac28965fa4f5d9bb4b586a31125ca50922133e21ef585cba1cd29023d7058" + ], + "structural_twin_ba.content_hash": "semantic_v1:3c511aacd4d465ad18354117161e6ae603d5982bd9a9f96a349fe55713095e64", + "structural_twin_ba.pipeline_hash": "semantic_v1:97ea3b680054c24d2cbfd76a1c717c0636ea3983b9282b9fa5e3febe1f926f90", + "structural_twin_ba.row[{\"k\":\"r1\"}]._tag_record_id::97ea3b680054c24d2cbfd76a1c717c0636ea3983b9282b9fa5e3febe1f926f90": "fcff3ee4896d58ebbd9717c386920e21", + "structural_twin_ba.row[{\"k\":\"r1\"}]._tag_source_id::97ea3b680054c24d2cbfd76a1c717c0636ea3983b9282b9fa5e3febe1f926f90": "golden_structural_twin_ba", + "structural_twin_ba.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:0000027283380cb21f7d6b8933ef682a7d2ae46f81631a65bf4def0b1dbd3185a48103", + "structural_twin_ba.row[{\"k\":\"r2\"}]._tag_record_id::97ea3b680054c24d2cbfd76a1c717c0636ea3983b9282b9fa5e3febe1f926f90": "219c3acffb245126801466a1a12fc818", + "structural_twin_ba.row[{\"k\":\"r2\"}]._tag_source_id::97ea3b680054c24d2cbfd76a1c717c0636ea3983b9282b9fa5e3febe1f926f90": "golden_structural_twin_ba", + "structural_twin_ba.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:0000020072e16a9c63ed4aa57d46f97b10ddc715bbba9f89b870aa4aa51beaeac6486b", + "structural_twin_ba.system_tag_columns": [ + "_tag_source_id::97ea3b680054c24d2cbfd76a1c717c0636ea3983b9282b9fa5e3febe1f926f90", + "_tag_record_id::97ea3b680054c24d2cbfd76a1c717c0636ea3983b9282b9fa5e3febe1f926f90" + ], + "structural_with_array.content_hash": "semantic_v1:96c02330fab6538e9433c87fd7907af86b81dcad22c3e9dd86e2f103f0e7e9fe", + "structural_with_array.pipeline_hash": "semantic_v1:c6f8ad5ef278842a5930fccaf9402c580869c12c1698248654588dee1a392a6d", + "structural_with_array.row[{\"k\":\"r1\"}]._tag_record_id::c6f8ad5ef278842a5930fccaf9402c580869c12c1698248654588dee1a392a6d": "a4e54f2bca2456dba1104df3b2e723f4", + "structural_with_array.row[{\"k\":\"r1\"}]._tag_source_id::c6f8ad5ef278842a5930fccaf9402c580869c12c1698248654588dee1a392a6d": "golden_structural_with_array", + "structural_with_array.row[{\"k\":\"r1\"}].data_hash": "arrow_v1:000002ed150b217ea035721e3312ca6f2313a49080245956570929c5b861234d607922", + "structural_with_array.row[{\"k\":\"r2\"}]._tag_record_id::c6f8ad5ef278842a5930fccaf9402c580869c12c1698248654588dee1a392a6d": "469a902713ab5c6bb6194ce0c0f3cfa4", + "structural_with_array.row[{\"k\":\"r2\"}]._tag_source_id::c6f8ad5ef278842a5930fccaf9402c580869c12c1698248654588dee1a392a6d": "golden_structural_with_array", + "structural_with_array.row[{\"k\":\"r2\"}].data_hash": "arrow_v1:0000022e58db4daf8d19fb1f6fdf9707185895820f14c35ff4acdc67160b073a6ec7c1", + "structural_with_array.system_tag_columns": [ + "_tag_source_id::c6f8ad5ef278842a5930fccaf9402c580869c12c1698248654588dee1a392a6d", + "_tag_record_id::c6f8ad5ef278842a5930fccaf9402c580869c12c1698248654588dee1a392a6d" + ], "uuid.content_hash": "semantic_v1:1bb82177a816d0191693f063664b3948d59ce18d8dc7328e1d1943c029248ee0", "uuid.pipeline_hash": "semantic_v1:5e1457a9e2dc542fef444e111a5cb4aa6077142a63056f7e5576d3a9ee8df756", "uuid.row[{\"k\":\"r1\"}]._tag_record_id::5e1457a9e2dc542fef444e111a5cb4aa6077142a63056f7e5576d3a9ee8df756": "c17eccb33f355b1e9d6878f55b081026", diff --git a/tests/test_hashing/golden/data/schema.json b/tests/test_hashing/golden/data/schema.json index e10c8a238..6f65e5d19 100644 --- a/tests/test_hashing/golden/data/schema.json +++ b/tests/test_hashing/golden/data/schema.json @@ -50,7 +50,7 @@ "_tag_record_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a" ], "multi_field.empty": "semantic_v1:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - "multi_field.logical_mix": "semantic_v1:0cd1d093a26e831081b04a477446a35eea52ede4c7ddde1fe091aa26e597a276", + "multi_field.logical_mix": "semantic_v1:4dd97df0a99272b7335c4d248f934f30dd3548abc0122167885a29599698880b", "multi_field.nullable_type": "semantic_v1:47f158f3ebe0c03fa07f15dc733b9f292c08e51c77f5d37879c5d6ce2ae3f0af", "multi_field.optional_field": "semantic_v1:8c133b192e23bfa5a2efb138d04bfcfdc2fa2df603344e1b5104243bf2a58ccc", "multi_field.two_scalars": "semantic_v1:8c133b192e23bfa5a2efb138d04bfcfdc2fa2df603344e1b5104243bf2a58ccc", @@ -59,8 +59,12 @@ "single_field.bool.schema_hash": "semantic_v1:d65d43b0876cd839e9c53022a3e523e8236ca0df552bd3fb7a6e199e259e6839", "single_field.bytes.arrow_type": "large_binary", "single_field.bytes.schema_hash": "semantic_v1:bcee6639200d3ab936ee0adab5de5a5533a4038e745d92a872410b8556f7e584", - "single_field.dataclass.arrow_type": "extension[struct]", - "single_field.dataclass.schema_hash": "semantic_v1:a0439e4672f557bf22d46b560ac83ed4c0005470311747e8e8042f0e56f735ee", + "single_field.dataclass.arrow_type": "extension[struct]", + "single_field.dataclass.schema_hash": "semantic_v1:829a2c1821178b9acab2b41690f1fcce7f9fd9bdfe0ad15531d82f38f17dc224", + "single_field.dataclass_subclass.arrow_type": "extension[struct]", + "single_field.dataclass_subclass.schema_hash": "semantic_v1:e6be7d71b664e3d850d8fe6d210613fb703a6672863cc1ff20416e11c0737f3e", + "single_field.dataclass_with_structural.arrow_type": "extension[struct, note: large_string>]", + "single_field.dataclass_with_structural.schema_hash": "semantic_v1:688cc938c85561643eb299bb5016d514ed87ce194def5d14b31859f7cc0c24f8", "single_field.dataframe.arrow_type": "extension[large_binary]", "single_field.dataframe.schema_hash": "semantic_v1:1128a9ddc94abf13783c79b712514fd14f67b3f0c5c9ca5818051cf62ac2d3f4", "single_field.dict_str_int.arrow_type": "large_list>", @@ -69,6 +73,8 @@ "single_field.float.schema_hash": "semantic_v1:e2d6e3ddecce8198dadb9de73dda99562ae800ca7c052e0df0a4d4c0474fa4c8", "single_field.int.arrow_type": "int64", "single_field.int.schema_hash": "semantic_v1:18aa336ce4f14f14a95521bc51bbae9f604f0a25e26efb5c19534ce20deca4e2", + "single_field.list_dataclass.arrow_type": "extension[large_list>]", + "single_field.list_dataclass.schema_hash": "semantic_v1:c48d04a14d764d23030b3939562338bdf1eb31811c1a4d77416464f6e7bae578", "single_field.list_int.arrow_type": "large_list", "single_field.list_int.schema_hash": "semantic_v1:37ebe999e47b31805394d5c4136a6202f866a99c92b1f7ab8ff1fbc9fe62b6fe", "single_field.list_list_int.arrow_type": "large_list>", @@ -81,12 +87,26 @@ "single_field.optional_int.schema_hash": "semantic_v1:7613bd137a749aba9e9c6f54091681242a22e408f5e8bf7a5caa47a212cc3141", "single_field.path.arrow_type": "extension[large_string]", "single_field.path.schema_hash": "semantic_v1:cf9f5ab623eb77bc115912dbb37a930d3e3cf19e39719c88a670deccfb9a5cde", - "single_field.pydantic.arrow_type": "extension[struct]", - "single_field.pydantic.schema_hash": "semantic_v1:60e2af7a186208f36c45a13e55e24eaf3a969a28e479face57592a023b2b41c3", + "single_field.pydantic.arrow_type": "extension[struct]", + "single_field.pydantic.schema_hash": "semantic_v1:7e65c46de4b5daceba5a06df5f24c8153e09507d42cf943c20ac4b398fe48cd1", + "single_field.pydantic_optional.arrow_type": "extension[struct]", + "single_field.pydantic_optional.schema_hash": "semantic_v1:0c522a5f2a7d2753c5ebce3177674afaab4f3ebe161eede4a6b326223955020e", "single_field.set_uuid.arrow_type": "extension[large_list]", "single_field.set_uuid.schema_hash": "semantic_v1:6ba2432e14f1026834a8b86ada09574f9da55a25d2cb237cc75fefd5761d1734", "single_field.str.arrow_type": "large_string", "single_field.str.schema_hash": "semantic_v1:c80b4f1243de36db40227c83541e647049d48cb229900af7787ce014b32155d4", + "single_field.structural.arrow_type": "extension[struct]", + "single_field.structural.schema_hash": "semantic_v1:fd78015b6d1fd510d51d8cf600ea74b40875a39a103dc2da0699657d55db845b", + "single_field.structural_nested.arrow_type": "extension[struct, tags: large_list>]", + "single_field.structural_nested.schema_hash": "semantic_v1:857eccc742263460931aa9ebfde684fcf31f8d78ab5794afbe117c2a05993a97", + "single_field.structural_same_identity.arrow_type": "extension[struct]", + "single_field.structural_same_identity.schema_hash": "semantic_v1:fd78015b6d1fd510d51d8cf600ea74b40875a39a103dc2da0699657d55db845b", + "single_field.structural_twin_ab.arrow_type": "extension[struct]", + "single_field.structural_twin_ab.schema_hash": "semantic_v1:ae904346e1568bac2921bb6b23be6fd1b53c2dfb267272a25707c13609e96702", + "single_field.structural_twin_ba.arrow_type": "extension[struct]", + "single_field.structural_twin_ba.schema_hash": "semantic_v1:fec2e905d7d0db153e5b9e5d9d443d698821c51154f304c8dc91fdb9afb6ce4e", + "single_field.structural_with_array.arrow_type": "extension[struct, data: large_binary>>]", + "single_field.structural_with_array.schema_hash": "semantic_v1:ce619b2aeb64419745accb29e02d830666a21b3227ce9da5ff6a98388d332b6a", "single_field.tuple_int_str.arrow_type": "struct", "single_field.tuple_int_str.schema_hash": "semantic_v1:84de6b5743e160faf5668eb29c7045f4757f150011645c3cec992ecd80d388f0", "single_field.uuid.arrow_type": "extension[large_binary]", diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/semantic.json index ecdae4e49..a166e3323 100644 --- a/tests/test_hashing/golden/data/semantic.json +++ b/tests/test_hashing/golden/data/semantic.json @@ -11,20 +11,30 @@ "polars_expr.str_contains": "polars_expr_v1:4c4cc5f6fe5bed1fbff4f59706145e216bb2597c2710c557d5ab0c39de476feb", "type.bool": "semantic_v1:cd8f971312b872f0e9a3ef40aa58d6bb537258e72a9a142e0cf712f0a35020e3", "type.bytes": "semantic_v1:f65cea8d1f29ca5b96d67caf1ced2f630b509e32574a7b78b9071958818a1c59", - "type.dataclass": "semantic_v1:b05e30cefeca780b2ac0182af6fb9b590bbefd865711cd86561d5f0298e88e59", + "type.dataclass": "semantic_v1:86baa1ef8dbf2518c6dcf7d188f6e20474248e9d8bc08d22b7fa0a4ed98bf8a8", + "type.dataclass_subclass": "semantic_v1:a708010250b375859e12f0bf5df84f07aeac185c97dcc7a85431a75b92da7666", + "type.dataclass_with_structural": "semantic_v1:82c2bb964c817a345467bb892979c137fc0217fc1cd73a9194a44bcaff0add51", "type.dataframe": "semantic_v1:ae415a3d304bd815c2935070c65793dc1ff6275db207f1a9e06cb755793ab33e", "type.dict_str_int": "semantic_v1:2f26c0edf323df8be4143a1965626399313409728cc8081378346c850757f7dd", "type.float": "semantic_v1:0059c08fc7dfe19dc8d095b7e95d56e8d377bbe49ac30376a00105eca2e7c75f", "type.int": "semantic_v1:24be397c58c33686e96b4f85a9f443fd2483564ec7dee1c14b96da3e60ec16ac", + "type.list_dataclass": "semantic_v1:81c0dc4515f05b3a2db095578562cd3a140131ea8586c86eb1acf4754dac1e3d", "type.list_int": "semantic_v1:388fcfad62b0e03893e350cabe27e6235aa52e5355df844aefce870cf2bbec75", "type.list_list_int": "semantic_v1:6c345e63e39c385db19c6d4abb56394fb31ea8412024ab8b33b67d3fec3340ef", "type.list_path": "semantic_v1:3f3ce39052a75cde9537faacc70ac47772c4ce7e8a0b94c69b0bd037b123bc36", "type.ndarray": "semantic_v1:b8eb285c0293ba5612fd8424f14c9cc03b306f9e6bf5bbb36c18d09b5202aaec", "type.optional_int": "semantic_v1:aaf3880090849afb79550bdcd6865c455ded79bf372efb283b6a68b82c627736", "type.path": "semantic_v1:7f0da67eaf9d048c9d1791fbe223333cfcc8018c66878647c8b7ae8571c72bd3", - "type.pydantic": "semantic_v1:a4244acc08b6702dc0183844a18b17a5fcaea4daa48048fae409ae1b84464d5f", + "type.pydantic": "semantic_v1:58c6aad8ca32918a34da1dddb62c72392079b9326b021aa3a4c57de4ae0a4ea8", + "type.pydantic_optional": "semantic_v1:37902c428bb3aff581d9dbdba4c74abc1aa359ebb062a055c9df3957a90f0d63", "type.set_uuid": "semantic_v1:33d4c1747923a88b8ff5b7eec15d697fdcd11dce11ea0c8071ce76032f00e9ef", "type.str": "semantic_v1:309d607de6633b959ae5118171f079b2830f4b1340064fac065652e305b038ad", + "type.structural": "semantic_v1:e050877645fb628a5ae83ad6c88398a91f2437a599a65054127056b221077564", + "type.structural_nested": "semantic_v1:ff79388a011792bcdc895cd9672ae9002b548662e0768129978a78d385d8f534", + "type.structural_same_identity": "semantic_v1:e050877645fb628a5ae83ad6c88398a91f2437a599a65054127056b221077564", + "type.structural_twin_ab": "semantic_v1:12ab17179b2f707e3551c93481b7eb7974d79805b2fa6a4e37890014bc8ca62a", + "type.structural_twin_ba": "semantic_v1:8f0abe45554d4cd0c68d5c9497a819564f7268cc2631602a2e1db6471f0112c7", + "type.structural_with_array": "semantic_v1:845c46052e7e3ad39f229ac4690663aecf2a24e62983361268c2ec0a44752772", "type.tuple_int_str": "semantic_v1:df199a19b63c33f3b7a8d06276faabbc74082af71710d69223980e84f71434ed", "type.uuid": "semantic_v1:763acbee42cb7bd6c7734ad7ca12ab6cb879ace3991e075366b0536bd67556f0", "value.bool.false": "semantic_v1:fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa", diff --git a/tests/test_logical_types/test_cache_behavior.py b/tests/test_logical_types/test_cache_behavior.py index 90bac4471..92e168ceb 100644 --- a/tests/test_logical_types/test_cache_behavior.py +++ b/tests/test_logical_types/test_cache_behavior.py @@ -28,6 +28,18 @@ ) +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + from orcapod.types import Schema + + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) + assert lt is not None + return lt.logical_type_name + + # Module-level dataclass — local classes cannot be reconstructed from FQCN. @@ -74,18 +86,19 @@ def test_cache_populated_after_first_read(tmp_path): parquet_path = _write_parquet(tmp_path, write_converter) read_converter = _fresh_converter() - fqcn = f"{_CachePoint.__module__}.{_CachePoint.__qualname__}" + type_name = _identity(_CachePoint) # Before read: not registered assert ( - read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn) is None + read_converter._logical_type_registry.get_by_arrow_extension_name(type_name) + is None ) read_converter.load_logical_types(pq.read_table(parquet_path)) # After read: registered (cache populated) assert ( - read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn) + read_converter._logical_type_registry.get_by_arrow_extension_name(type_name) is not None ) diff --git a/tests/test_logical_types/test_dataclass_logical_type_factory.py b/tests/test_logical_types/test_dataclass_logical_type_factory.py index 70b098e8f..e6ea85c2d 100644 --- a/tests/test_logical_types/test_dataclass_logical_type_factory.py +++ b/tests/test_logical_types/test_dataclass_logical_type_factory.py @@ -9,6 +9,32 @@ import pytest +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + from orcapod.types import Schema + + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) + assert lt is not None + return lt.logical_type_name + + +def _metadata(cls: type) -> dict[str, object]: + """Return the parsed Arrow extension metadata of a user class (ITL-881).""" + import json + + from orcapod.contexts import get_default_context + from orcapod.types import Schema + + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) + assert lt is not None + return json.loads(lt.get_arrow_extension_type().__arrow_ext_serialize__()) + + # ── Helpers ───────────────────────────────────────────────────────────────── @@ -382,16 +408,18 @@ def test_factory_reconstruct_from_arrow(): ) storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - metadata = {"category": "orcapod.dataclass"} - fqcn = f"{_RoundTripPoint.__module__}.{_RoundTripPoint.__qualname__}" + metadata = _metadata(_RoundTripPoint) + type_name = _identity(_RoundTripPoint) factory = DataclassLogicalTypeFactory() converter = _make_full_converter() - lt = factory.reconstruct_from_arrow(fqcn, storage, metadata, converter=converter) + lt = factory.reconstruct_from_arrow( + type_name, storage, metadata, converter=converter + ) assert isinstance(lt, DataclassLogicalType) assert lt.python_type is _RoundTripPoint - assert lt.logical_type_name == fqcn + assert lt.logical_type_name == type_name def test_factory_reconstruct_from_arrow_invalid_fqcn(): @@ -404,11 +432,16 @@ def test_factory_reconstruct_from_arrow_invalid_fqcn(): factory = DataclassLogicalTypeFactory() converter = _make_full_converter() + import orcapod.logical_types.user_class_identity as uci + + record = uci.identity_record( + "dataclass", "nonexistent.module.NoSuchClass", [("x", "int")] + ) with pytest.raises(ImportError): factory.reconstruct_from_arrow( - "nonexistent.module.NoSuchClass", + uci.identity_string(record), storage, - {"category": "orcapod.dataclass"}, + {"category": "orcapod.dataclass", "identity": record}, converter, ) @@ -429,7 +462,7 @@ def test_reconstruct_from_arrow_registers_nested_types(): pa.field("label", pa.large_string()), ] ) - outer_fqcn = f"{_OuterForRegistrationTest.__module__}.{_OuterForRegistrationTest.__qualname__}" + outer_type_name = _identity(_OuterForRegistrationTest) factory = DataclassLogicalTypeFactory() converter = _make_full_converter() @@ -442,7 +475,7 @@ def test_reconstruct_from_arrow_registers_nested_types(): # reconstruct_from_arrow for Outer should trigger registration of Inner as a side effect factory.reconstruct_from_arrow( - outer_fqcn, outer_storage, {"category": "orcapod.dataclass"}, converter + outer_type_name, outer_storage, _metadata(_OuterForRegistrationTest), converter ) # Inner must now be registered @@ -499,12 +532,12 @@ def test_dataclass_with_uuid_round_trip(): assert reconstructed.label == "hello" -# ── _import_from_fqcn nested class tests ───────────────────────────────────── +# ── nested class lookup on read ─────────────────────────────────────────────── @dataclasses.dataclass class _OuterForNestedTest: - """Module-level outer class for testing nested-class FQCN import.""" + """Module-level outer class for testing the read-side lookup of a nested class.""" @dataclasses.dataclass class Inner: @@ -512,20 +545,27 @@ class Inner: y: str -def test_import_from_fqcn_nested_class(): - """_import_from_fqcn resolves module-level nested dataclasses via attribute walk.""" - from orcapod.logical_types.dataclass_logical_type_factory import _import_from_fqcn +def test_reconstruct_finds_class_nested_in_a_class(): + """The read path imports a class nested in another class by its dotted name.""" + from orcapod.logical_types.dataclass_logical_type_factory import ( + DataclassLogicalTypeFactory, + ) - # _OuterForNestedTest.Inner lives in this test module; its FQCN uses '.' for nesting - module = _OuterForNestedTest.__module__ - inner_qualname = ( - _OuterForNestedTest.Inner.__qualname__ - ) # e.g. "_OuterForNestedTest.Inner" + import orcapod.logical_types.user_class_identity as uci - fqcn = f"{module}.{inner_qualname}" - cls = _import_from_fqcn(fqcn) - assert cls is _OuterForNestedTest.Inner - assert dataclasses.is_dataclass(cls) + inner = _OuterForNestedTest.Inner + # Build the stored identity by hand, so that the class is not registered and the + # read can only find it by importing its dotted name. + name = f"{inner.__module__}.{inner.__qualname__}" + record = uci.identity_record("dataclass", name, [("x", "int"), ("y", "str")]) + storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.large_string())]) + lt = DataclassLogicalTypeFactory().reconstruct_from_arrow( + uci.identity_string(record), + storage, + {"category": "orcapod.dataclass", "identity": record}, + _make_full_converter(), + ) + assert lt.python_type is inner def test_python_to_storage_raises_when_converter_none(): @@ -674,3 +714,28 @@ def test_list_of_nested_dataclass_parquet_roundtrip(tmp_path): assert reconstructed.items[0].x == 1 assert reconstructed.items[1].y == 4 assert reconstructed.label == "test" + + +def test_reconstruct_rejects_a_record_that_does_not_match_the_extension_name(): + """A stored record must hash to the stored extension name (ITL-881).""" + from orcapod.errors import TypeIdentityError + from orcapod.logical_types.dataclass_logical_type_factory import ( + DataclassLogicalTypeFactory, + ) + + # This test is about the factory's read entry point: an inconsistent stored + # column cannot be produced through the public interface. + metadata = _metadata(_Flat) + stored = metadata["identity"] + assert isinstance(stored, dict) + record = {**stored, "fields": [["name", "str"], ["count", "str"]]} + storage = pa.struct( + [pa.field("name", pa.large_string()), pa.field("count", pa.int64())] + ) + with pytest.raises(TypeIdentityError, match="invalid identity record"): + DataclassLogicalTypeFactory().reconstruct_from_arrow( + _identity(_Flat), + storage, + {"category": metadata["category"], "identity": record}, + _make_full_converter(), + ) diff --git a/tests/test_logical_types/test_default_context_factories.py b/tests/test_logical_types/test_default_context_factories.py index 2c880b16b..48b5a8f8d 100644 --- a/tests/test_logical_types/test_default_context_factories.py +++ b/tests/test_logical_types/test_default_context_factories.py @@ -20,6 +20,18 @@ from orcapod.logical_types.registry import LogicalTypeRegistry +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + from orcapod.types import Schema + + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) + assert lt is not None + return lt.logical_type_name + + # ── Module-level dataclasses (local classes cannot be registered) ──────────── @@ -115,8 +127,8 @@ def test_default_context_dataclass_auto_registered_on_use(): converter = create_registry().get_context().type_converter arrow_type = converter.register_python_class(_SimplePoint) assert isinstance(arrow_type, pa.ExtensionType) - fqcn = f"{_SimplePoint.__module__}.{_SimplePoint.__qualname__}" - assert arrow_type.extension_name == fqcn + type_name = _identity(_SimplePoint) + assert arrow_type.extension_name == type_name def test_default_context_pydantic_auto_registered_on_use(): @@ -124,8 +136,8 @@ def test_default_context_pydantic_auto_registered_on_use(): converter = create_registry().get_context().type_converter arrow_type = converter.register_python_class(_SimpleModel) assert isinstance(arrow_type, pa.ExtensionType) - fqcn = f"{_SimpleModel.__module__}.{_SimpleModel.__qualname__}" - assert arrow_type.extension_name == fqcn + type_name = _identity(_SimpleModel) + assert arrow_type.extension_name == type_name # ── Parquet round-trip tests ───────────────────────────────────────────────── diff --git a/tests/test_logical_types/test_pydantic_logical_type_factory.py b/tests/test_logical_types/test_pydantic_logical_type_factory.py index f7f6d2cdc..2bad77625 100644 --- a/tests/test_logical_types/test_pydantic_logical_type_factory.py +++ b/tests/test_logical_types/test_pydantic_logical_type_factory.py @@ -11,6 +11,32 @@ from pydantic import BaseModel, PrivateAttr, computed_field, field_validator +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + from orcapod.types import Schema + + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) + assert lt is not None + return lt.logical_type_name + + +def _metadata(cls: type) -> dict[str, object]: + """Return the parsed Arrow extension metadata of a user class (ITL-881).""" + import json + + from orcapod.contexts import get_default_context + from orcapod.types import Schema + + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) + assert lt is not None + return json.loads(lt.get_arrow_extension_type().__arrow_ext_serialize__()) + + # ── Helpers ────────────────────────────────────────────────────────────────── @@ -410,16 +436,18 @@ def test_factory_reconstruct_from_arrow(): ) storage = pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())]) - metadata = {"category": "orcapod.pydantic"} - fqcn = f"{_RoundTripPoint.__module__}.{_RoundTripPoint.__qualname__}" + metadata = _metadata(_RoundTripPoint) + type_name = _identity(_RoundTripPoint) factory = PydanticLogicalTypeFactory() converter = _make_full_converter() - lt = factory.reconstruct_from_arrow(fqcn, storage, metadata, converter=converter) + lt = factory.reconstruct_from_arrow( + type_name, storage, metadata, converter=converter + ) assert isinstance(lt, PydanticLogicalType) assert lt.python_type is _RoundTripPoint - assert lt.logical_type_name == fqcn + assert lt.logical_type_name == type_name def test_factory_reconstruct_from_arrow_invalid_fqcn(): @@ -432,11 +460,16 @@ def test_factory_reconstruct_from_arrow_invalid_fqcn(): factory = PydanticLogicalTypeFactory() converter = _make_full_converter() + import orcapod.logical_types.user_class_identity as uci + + record = uci.identity_record( + "pydantic", "nonexistent.module.NoSuchClass", [("x", "int")] + ) with pytest.raises(ImportError): factory.reconstruct_from_arrow( - "nonexistent.module.NoSuchModel", + uci.identity_string(record), storage, - {"category": "orcapod.pydantic"}, + {"category": "orcapod.pydantic", "identity": record}, converter, ) @@ -454,7 +487,7 @@ def test_reconstruct_from_arrow_registers_nested_types(): pa.field("label", pa.large_string()), ] ) - outer_fqcn = f"{_OuterModel.__module__}.{_OuterModel.__qualname__}" + outer_type_name = _identity(_OuterModel) factory = PydanticLogicalTypeFactory() converter = _make_full_converter() @@ -463,7 +496,7 @@ def test_reconstruct_from_arrow_registers_nested_types(): assert converter._logical_type_registry.get_by_python_type(_InnerModel) is None factory.reconstruct_from_arrow( - outer_fqcn, outer_storage, {"category": "orcapod.pydantic"}, converter + outer_type_name, outer_storage, _metadata(_OuterModel), converter ) # Inner must now be registered as a side effect @@ -737,9 +770,9 @@ def test_literal_model_polars_dataframe_round_trip(): df = pl.from_arrow(table) # Polars column must carry the correct extension type - fqcn = f"{_LiteralRoundTripModel.__module__}.{_LiteralRoundTripModel.__qualname__}" + type_name = _identity(_LiteralRoundTripModel) assert isinstance(df.dtypes[0], pl.BaseExtension) - assert df.dtypes[0].ext_name() == fqcn + assert df.dtypes[0].ext_name() == type_name # Convert back to Arrow and reconstruct Python objects table_back = df.to_arrow() diff --git a/tests/test_logical_types/test_registry.py b/tests/test_logical_types/test_registry.py index 482699a5c..8263f7aeb 100644 --- a/tests/test_logical_types/test_registry.py +++ b/tests/test_logical_types/test_registry.py @@ -22,7 +22,6 @@ LogicalTypeRegistry, make_arrow_extension_type, make_polars_extension_type, - _canonical_storage, ) @@ -680,43 +679,6 @@ def test_registry_does_not_expose_ensure_methods(): assert not hasattr(registry, "ensure_extension_type") -# --------------------------------------------------------------------------- -# _canonical_storage helper -# --------------------------------------------------------------------------- - - -class TestCanonicalStorage: - """Unit tests for the _canonical_storage helper.""" - - def test_string_maps_to_large_string(self): - assert _canonical_storage(pa.string()) == pa.large_string() - - def test_utf8_maps_to_large_string(self): - # pa.utf8() is an alias for pa.string() - assert _canonical_storage(pa.utf8()) == pa.large_string() - - def test_string_view_maps_to_large_string(self): - assert _canonical_storage(pa.string_view()) == pa.large_string() - - def test_large_string_is_identity(self): - assert _canonical_storage(pa.large_string()) == pa.large_string() - - def test_binary_maps_to_large_binary(self): - assert _canonical_storage(pa.binary()) == pa.large_binary() - - def test_binary_view_maps_to_large_binary(self): - assert _canonical_storage(pa.binary_view()) == pa.large_binary() - - def test_large_binary_is_identity(self): - assert _canonical_storage(pa.large_binary()) == pa.large_binary() - - def test_int64_is_identity(self): - assert _canonical_storage(pa.int64()) == pa.int64() - - def test_fixed_binary_is_identity(self): - assert _canonical_storage(pa.binary(16)) == pa.binary(16) - - # --------------------------------------------------------------------------- # __arrow_ext_deserialize__ storage-family tolerance (ITL-602) # --------------------------------------------------------------------------- @@ -803,7 +765,7 @@ def test_non_canonical_storage_accepts_exact_match(self): """Extension registered with plain string (non-canonical) still reads its own data. Regression test for the Copilot review finding: the previous check - _canonical_storage(storage_type) != _storage would map string → large_string and + normalize_to_large_types(storage_type) != _storage would map string → large_string and then compare large_string != string, incorrectly raising for a type that should read back cleanly. """ diff --git a/tests/test_logical_types/test_roundtrips.py b/tests/test_logical_types/test_roundtrips.py index 33931b3d5..fcf0328c1 100644 --- a/tests/test_logical_types/test_roundtrips.py +++ b/tests/test_logical_types/test_roundtrips.py @@ -34,6 +34,18 @@ from orcapod.semantic_types.universal_converter import UniversalTypeConverter +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + from orcapod.types import Schema + + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) + assert lt is not None + return lt.logical_type_name + + # ── Module-level dataclasses ────────────────────────────────────────────────── # DataclassLogicalTypeFactory rejects local (in-function) classes because they # have no stable fully-qualified class name for reconstruction from Arrow schema. @@ -338,12 +350,12 @@ def test_simple_dataclass_round_trip( tmp_path, ) - fqcn = f"{_PointA.__module__}.{_PointA.__qualname__}" + type_name = _identity(_PointA) field = result.schema.field("point") assert hasattr(field.type, "extension_name"), ( f"Expected extension type on field 'point', got {field.type!r}" ) - assert field.type.extension_name == fqcn + assert field.type.extension_name == type_name rows = read_converter.arrow_table_to_python_dicts(result) assert len(rows) == 1 @@ -370,13 +382,13 @@ def test_two_dataclasses_same_shape_distinct_extension_names( tmp_path, ) - fqcn_a = f"{_PointA.__module__}.{_PointA.__qualname__}" - fqcn_b = f"{_PointB.__module__}.{_PointB.__qualname__}" + type_name_a = _identity(_PointA) + type_name_b = _identity(_PointB) field = result.schema.field("point") assert hasattr(field.type, "extension_name") - assert field.type.extension_name == fqcn_a - assert field.type.extension_name != fqcn_b # distinct from _PointB + assert field.type.extension_name == type_name_a + assert field.type.extension_name != type_name_b # distinct from _PointB rows = read_converter.arrow_table_to_python_dicts(result) reconstructed = rows[0]["point"] @@ -401,15 +413,19 @@ def test_nested_dataclass_round_trip( tmp_path, ) - fqcn_outer = f"{_Outer.__module__}.{_Outer.__qualname__}" - fqcn_inner = f"{_Inner.__module__}.{_Inner.__qualname__}" + type_name_outer = _identity(_Outer) + type_name_inner = _identity(_Inner) assert ( - read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn_outer) + read_converter._logical_type_registry.get_by_arrow_extension_name( + type_name_outer + ) is not None ), "_Outer should be registered after read" assert ( - read_converter._logical_type_registry.get_by_arrow_extension_name(fqcn_inner) + read_converter._logical_type_registry.get_by_arrow_extension_name( + type_name_inner + ) is not None ), "_Inner should be registered transitively after read" @@ -444,7 +460,7 @@ def test_delta_polars_read_delta(tmp_path: Path) -> None: import polars as pl delta_path = str(tmp_path / "polars_delta") - fqcn = f"{_PointA.__module__}.{_PointA.__qualname__}" + type_name = _identity(_PointA) # Write — registers _PointA in PyArrow + Polars global registries. write_converter = _fresh_converter() @@ -463,8 +479,8 @@ def test_delta_polars_read_delta(tmp_path: Path) -> None: assert col_dtype.is_extension(), ( f"Expected a Polars extension type on column 'point', got {col_dtype!r}" ) - assert col_dtype.ext_name() == fqcn, ( - f"Expected extension name {fqcn!r}, got {col_dtype.ext_name()!r}" + assert col_dtype.ext_name() == type_name, ( + f"Expected extension name {type_name!r}, got {col_dtype.ext_name()!r}" # pyright: ignore[reportAttributeAccessIssue] # polars stubs lack ext_name ) diff --git a/tests/test_logical_types/test_schema_compatibility.py b/tests/test_logical_types/test_schema_compatibility.py index 860519348..6289f6b09 100644 --- a/tests/test_logical_types/test_schema_compatibility.py +++ b/tests/test_logical_types/test_schema_compatibility.py @@ -28,6 +28,17 @@ from orcapod.utils.schema_utils import check_schema_compatibility +def _identity(cls: type) -> str: + """Return the identity string (logical type name) of a user class (ITL-881).""" + from orcapod.contexts import get_default_context + + converter = get_default_context().type_converter + converter.ensure_types_registered_for_schemas(Schema({"value": cls})) + lt = converter.get_logical_type(cls) + assert lt is not None + return lt.logical_type_name + + # Module-level dataclasses — DataclassLogicalTypeFactory rejects local classes # because they have no stable fully-qualified class name for reconstruction. @@ -64,10 +75,10 @@ def test_arrow_schema_distinct_extension_names_for_same_shape(): assert isinstance(type_a, pa.ExtensionType) assert isinstance(type_b, pa.ExtensionType) - fqcn_a = f"{_PointA.__module__}.{_PointA.__qualname__}" - fqcn_b = f"{_PointB.__module__}.{_PointB.__qualname__}" - assert type_a.extension_name == fqcn_a - assert type_b.extension_name == fqcn_b + type_name_a = _identity(_PointA) + type_name_b = _identity(_PointB) + assert type_a.extension_name == type_name_a + assert type_b.extension_name == type_name_b assert type_a.extension_name != type_b.extension_name diff --git a/tests/test_logical_types/test_type_utils.py b/tests/test_logical_types/test_type_utils.py index e8e52864a..c83c61dc5 100644 --- a/tests/test_logical_types/test_type_utils.py +++ b/tests/test_logical_types/test_type_utils.py @@ -167,15 +167,3 @@ def _patched(name: str, *args, **kwargs): with pytest.raises(ModuleNotFoundError, match="'path'"): _walk_fqcn("pathlib.Path") - - -# ── _import_from_fqcn tests ────────────────────────────────────────────────── - - -def test_import_from_fqcn_raises_for_non_dataclass(): - """_import_from_fqcn raises ImportError when FQCN resolves to a non-dataclass.""" - from orcapod.logical_types.dataclass_logical_type_factory import _import_from_fqcn - - # pathlib.Path is importable via _walk_fqcn but is not a dataclass - with pytest.raises(ImportError): - _import_from_fqcn("pathlib.Path") diff --git a/tests/test_pipeline/test_type_identity_failure.py b/tests/test_pipeline/test_type_identity_failure.py new file mode 100644 index 000000000..1a7092c1c --- /dev/null +++ b/tests/test_pipeline/test_type_identity_failure.py @@ -0,0 +1,200 @@ +"""A stored row that cannot be decoded as its class fails the run (ITL-881). + +``TypeIdentityError`` means a stored table is intact but holds another shape than the +registered class. It is not a ``StoreReadError``, yet it must be as fatal: under +``error_policy="continue"`` a row whose stored result cannot be decoded must never be +skipped as an ordinary row failure, or stored results would silently look dropped. Both +orchestrators must raise it. +""" + +from __future__ import annotations + +import dataclasses +import shutil +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import deltalake +import pyarrow as pa +import pytest + +from orcapod import function_pod +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import StoreError, TypeIdentityError +from orcapod.pipeline import PipelineJob +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + +Orchestrator = SyncPipelineOrchestrator | AsyncPipelineOrchestrator + +_CALLS: list[int] = [] + + +@dataclasses.dataclass +class Pair: + """The value the pod stores.""" + + x: int + y: int + + +@function_pod(output_keys="q") +def pair(x: int) -> Pair: + """Record the call and return ``Pair(x, x)``.""" + _CALLS.append(x) + return Pair(x, x) + + +ORCHESTRATORS: dict[str, Callable[[], Orchestrator]] = { + "sync": lambda: SyncPipelineOrchestrator(error_policy="continue"), + "async": lambda: AsyncPipelineOrchestrator(error_policy="continue"), +} + + +@pytest.fixture(autouse=True) +def _reset_calls(): + """Start and end every test with no recorded calls.""" + _CALLS.clear() + yield + _CALLS.clear() + + +@pytest.fixture(params=list(ORCHESTRATORS)) +def make_orchestrator(request: pytest.FixtureRequest) -> Callable[[], Orchestrator]: + """Return a factory for the parametrized orchestrator (``continue`` policy).""" + return ORCHESTRATORS[request.param] + + +def build_job(store_path: Path) -> PipelineJob: + """Build a job that stores one ``Pair`` per row of a three-row source.""" + source = ArrowTableSource( + pa.table({"k": ["a", "b", "c"], "x": [1, 2, 3]}), + tag_columns=["k"], + infer_nullable=True, + ) + job = PipelineJob(name="itl881", store=DeltaTableDatabase(base_path=store_path)) + with job: + _ = pair.pod(source, label="pair") + return job + + +def table_dirs(store_path: Path) -> tuple[Path, Path]: + """The (result table, pipeline table) directories the job wrote.""" + dirs = {p.parent for p in store_path.rglob("*.parquet")} + result = [d for d in dirs if "_result" in d.parts] + pipeline = [d for d in dirs if "_result" not in d.parts] + assert len(result) == 1 and len(pipeline) == 1, sorted(dirs) + return result[0], pipeline[0] + + +def store_another_shape(result_table: Path) -> None: + """Rewrite column ``q`` with another struct under the same extension name. + + Stands in for a table written by other means: the table stays readable, but its + stored shape no longer matches the registered ``Pair``. + """ + stored = deltalake.DeltaTable(str(result_table)).to_pyarrow_table() + index = stored.schema.get_field_index("q") + ext = stored.schema.field(index).type + assert isinstance(ext, pa.ExtensionType) + wrong = pa.struct([pa.field("x", pa.float64()), pa.field("y", pa.int64())]) + values: list[Any] = [{"x": 0.5, "y": 1}] * stored.num_rows + field = pa.field("q", wrong).with_metadata( + { + b"ARROW:extension:name": ext.extension_name.encode(), + b"ARROW:extension:metadata": ext.__arrow_ext_serialize__(), + } + ) + columns = [ + pa.array(values, type=wrong) if i == index else stored.column(i) + for i in range(stored.num_columns) + ] + fields = [field if i == index else f for i, f in enumerate(stored.schema)] + deltalake.write_deltalake( + str(result_table), + pa.Table.from_arrays(columns, schema=pa.schema(fields)), + mode="overwrite", + schema_mode="overwrite", + ) + + +@pytest.fixture +def stored(tmp_path: Path, make_orchestrator: Callable[[], Orchestrator]) -> Path: + """A store in which the job has computed and stored all three rows.""" + store_path = tmp_path / "store" + build_job(store_path).run(orchestrator=make_orchestrator()) + assert len(_CALLS) == 3 + _CALLS.clear() + return store_path + + +def test_other_stored_shape_fails_the_run( + stored: Path, make_orchestrator: Callable[[], Orchestrator] +) -> None: + """A result table holding another shape fails the run before any function call.""" + result_table, _ = table_dirs(stored) + store_another_shape(result_table) + + with pytest.raises(TypeIdentityError) as info: + build_job(stored).run(orchestrator=make_orchestrator()) + + assert not isinstance(info.value, StoreError) + assert _CALLS == [] + + +def test_other_stored_shape_fails_the_run_in_the_per_row_lookup( + stored: Path, make_orchestrator: Callable[[], Orchestrator] +) -> None: + """The per-row lookup's identity error fails the run under ``continue``.""" + # With the pipeline table gone, the first read of the result table is the per-row + # cache lookup, which runs inside the error_policy="continue" handler. + result_table, pipeline_table = table_dirs(stored) + store_another_shape(result_table) + shutil.rmtree(pipeline_table) + + with pytest.raises(TypeIdentityError) as info: + build_job(stored).run(orchestrator=make_orchestrator()) + + assert not isinstance(info.value, StoreError) + assert _CALLS == [] + + +class _CrashingObserver(NoOpObserver): + """An observer whose callbacks raise while the pod's failure propagates.""" + + def on_data_crash( + self, node_label: str, tag: Any, data: Any, error: Exception + ) -> None: + """Raise from the row-crash callback.""" + raise RuntimeError("observer failed in on_data_crash") + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Raise from the node-end callback of the pod's node.""" + if node_label == "pair": + raise RuntimeError("observer failed in on_node_end") + + def on_run_end(self, run_id: str) -> None: + """Raise from the run-end callback.""" + raise RuntimeError("observer failed in on_run_end") + + +def test_failing_observer_does_not_replace_the_identity_error( + stored: Path, make_orchestrator: Callable[[], Orchestrator] +) -> None: + """Callbacks that raise while the identity error propagates are noted, not raised.""" + result_table, pipeline_table = table_dirs(stored) + store_another_shape(result_table) + shutil.rmtree(pipeline_table) + + with pytest.raises(TypeIdentityError) as info: + build_job(stored).run( + orchestrator=make_orchestrator(), observer=_CrashingObserver() + ) + + notes = getattr(info.value, "__notes__", []) + assert any("observer failed in on_data_crash" in note for note in notes), notes + assert any("observer failed in on_run_end" in note for note in notes), notes + assert _CALLS == [] diff --git a/tests/test_user_types/__init__.py b/tests/test_user_types/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_user_types/child_harness.py b/tests/test_user_types/child_harness.py new file mode 100644 index 000000000..f63083d58 --- /dev/null +++ b/tests/test_user_types/child_harness.py @@ -0,0 +1,174 @@ +"""Run workshop scenarios in fresh Python processes (ITL-881 tests). + +A user class's identity depends on process-wide state (where the class is defined, what +is registered, Arrow's global extension registry), so the scenarios run each "kernel" or +"script" in its own interpreter. A child reports by printing one ``JSON:`` line +through ``report``. + +``WSJOB`` is a module the scenarios write next to their scripts: it runs one function pod +over a ``DictSource`` in a ``PipelineJob`` stored in a ``DeltaTableDatabase``. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +JSON_MARKER = "JSON:" +HARNESS_DIR = Path(__file__).parent + + +def write_files(root: Path, files: Mapping[str, str]) -> None: + """Write ``files`` (relative path to dedented content) under ``root``.""" + for rel, content in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(textwrap.dedent(content).lstrip(), encoding="utf-8") + + +def run_child( + argv: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str] | None = None, + timeout: float = 300.0, +) -> Any: + """Run ``python `` in ``cwd`` and return the payload it reported. + + Args: + argv: Arguments after the interpreter, e.g. ``["-c", code]``. + cwd: Working directory; also put on ``PYTHONPATH`` with this harness. + env: Extra environment variables. + timeout: Seconds before the child is killed. + + Returns: + The reported payload. + + Raises: + AssertionError: If the child exits non-zero or reports nothing. + """ + child_env = dict(os.environ) + child_env["PYTHONPATH"] = os.pathsep.join([str(cwd), str(HARNESS_DIR)]) + child_env.update(env or {}) + proc = subprocess.run( + [sys.executable, *argv], + cwd=cwd, + env=child_env, + capture_output=True, + text=True, + check=False, + timeout=timeout, + ) + if proc.returncode != 0: + raise AssertionError( + f"child failed (exit {proc.returncode}):\n{proc.stderr[-4000:]}" + ) + for line in proc.stdout.splitlines(): + if line.startswith(JSON_MARKER): + return json.loads(line[len(JSON_MARKER) :]) + raise AssertionError( + f"child reported nothing (exit {proc.returncode}):\n{proc.stderr[-4000:]}" + ) + + +def report(payload: Any) -> None: + """Print ``payload`` as the JSON line the parent reads, then exit at once. + + ``os._exit`` skips interpreter shutdown: a process that read Delta tables sometimes + hangs in Arrow's thread-pool destructor at shutdown (seen in the ITL-881 + experiments; out of scope here). + """ + print(JSON_MARKER + json.dumps(payload, default=str)) + sys.stdout.flush() + sys.stderr.flush() + os._exit(0) + + +def error_text(exc: BaseException) -> str: + """Return ``": "`` for an exception.""" + first = str(exc).splitlines()[0] if str(exc) else "" + return f"{type(exc).__name__}: {first}" + + +#: Job module written next to the scenario scripts. +WSJOB = ''' +import os +from pathlib import Path + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import DictSource + + +def table_paths(root): + """Return the sorted relative paths of the Delta tables under ``root``.""" + return sorted( + str(p.parent.relative_to(root)) for p in Path(root).rglob("_delta_log") if p.is_dir() + ) + + +def stored_fields(root, column): + """Return the field names of each stored struct column ``column``, as written.""" + import deltalake + import pyarrow as pa + + found = [] + for rel in table_paths(root): + schema = pa.schema(deltalake.DeltaTable(str(Path(root) / rel)).schema().to_arrow()) + if column in schema.names: + stored = schema.field(column).type + stored = getattr(stored, "storage_type", stored) + found.append([field.name for field in stored]) + return sorted(found) + + +def make_pod(point_cls): + """Return ``(pod, calls)`` for ``shift(p: point_cls) -> point_cls``.""" + calls = [] + + def shift(p): + """Record the call and return ``p`` unchanged.""" + calls.append(p) + return p + + shift.__annotations__ = {"p": point_cls, "return": point_cls} + return function_pod(output_keys="q")(shift), calls + + +def run_job(point_cls, pod_and_calls=None, rows=None): + """Run the shift pod over two rows; report calls, rows and identity.""" + pod, calls = pod_and_calls or make_pod(point_cls) + if rows is None: + names = list(getattr(point_cls, "model_fields", None) or point_cls.__dataclass_fields__) + # A distinct value per field, so that swapped fields would be noticed. + rows = [{n: 10 * i + j for j, n in enumerate(names)} for i in (1, 2)] + data = [{"k": f"r{i}", "p": point_cls(**row)} for i, row in enumerate(rows, 1)] + before = len(calls) + db_root = os.environ["WS_DB"] + job = PipelineJob(name="ws", store=DeltaTableDatabase(Path(db_root))) + with job: + pod.pod(DictSource(data=data, tag_columns=["k"], source_id="s"), label="shift") + job.run() + node = job.nodes["shift"] + values = [d.as_dict()["q"] for _, d in node.iter_data()] + return { + "calls": len(calls) - before, + "rows": node.as_table().num_rows, + "q_class_is_point": all(type(v) is point_cls for v in values), + "values": sorted(repr(v) for v in values), + "stored_fields": stored_fields(db_root, "q"), + "pod_pipeline_hash": pod.pod.pipeline_hash().to_string(), + "pod_uri": list(pod.pod.uri), + "node_identity_path": list(node.node_identity_path), + "table_paths": table_paths(db_root), + "extension_name": getattr( + node.as_table().schema.field("q").type, "extension_name", None + ), + } +''' diff --git a/tests/test_user_types/test_exact_classes.py b/tests/test_user_types/test_exact_classes.py new file mode 100644 index 000000000..5140fa719 --- /dev/null +++ b/tests/test_user_types/test_exact_classes.py @@ -0,0 +1,269 @@ +"""An ordinary dataclass or pydantic model is identified by its exact class (ITL-972). + +Two class objects are one type only when they are the same object. A class defined again +under the same module path and name with the same fields (a re-run notebook cell) is a new +class, and a pod built for the earlier one refuses it. There is no way to declare a name +that would make two classes interchangeable. +""" + +from __future__ import annotations + +import dataclasses +import importlib +from pathlib import Path +from typing import Any + +import pytest + +import orcapod as op +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import DictSource + +DATACLASS_SOURCE = """ +import dataclasses + + +@dataclasses.dataclass +class Reading: + '''A value defined twice by ``_define`` (same module path, name and fields).''' + x: int + y: int +""" + +PYDANTIC_SOURCE = """ +import pydantic + + +class Reading(pydantic.BaseModel): + '''A model defined twice by ``_define`` (same module path, name and fields).''' + x: int + y: int +""" + +KINDS = {"dataclass": DATACLASS_SOURCE, "pydantic": PYDANTIC_SOURCE} + + +def _define(source: str) -> type: + """Execute ``source`` as a cell of this module and return its ``Reading`` class. + + Each call makes a new class object with this module's path and the name ``Reading``, + as re-running a notebook cell does. + """ + namespace: dict[str, Any] = {"__name__": __name__} + exec(source, namespace) + return namespace["Reading"] + + +def _numbers() -> DictSource: + """Return a one-row source holding ``1`` in data column ``x``.""" + return DictSource(data=[{"k": "a", "x": 1}], tag_columns=["k"], source_id="n") + + +def _source(value: Any) -> DictSource: + """Return a one-row source holding ``value`` in data column ``p``.""" + return DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"], source_id="s") + + +def _sum_pod(cls: type) -> Any: + """Return a pod ``total(p: cls) -> int`` that adds the two fields.""" + + def total(p): + """Return ``p.x + p.y``.""" + return p.x + p.y + + total.__annotations__ = {"p": cls, "return": int} + return function_pod(output_keys="t")(total) + + +def _make_pod(cls: type) -> Any: + """Return a pod ``make(x: int) -> cls`` that builds an instance.""" + + def make(x): + """Return ``cls(x, x + 1)``.""" + return cls(x=x, y=x + 1) + + make.__annotations__ = {"x": int, "return": cls} + return function_pod(output_keys="q")(make) + + +def _receive_pod(cls: type) -> Any: + """Return a pod ``receive(q: cls) -> int`` that adds the two fields.""" + + def receive(q): + """Return ``q.x + q.y``.""" + return q.x + q.y + + receive.__annotations__ = {"q": cls, "return": int} + return function_pod(output_keys="t")(receive) + + +# --------------------------------------------------------------------------- +# No declared names +# --------------------------------------------------------------------------- + + +def test_register_type_is_not_part_of_the_public_api() -> None: + """``op.register_type`` and ``orcapod.user_types`` no longer exist.""" + assert not hasattr(op, "register_type") + assert "register_type" not in op.__all__ + with pytest.raises(ModuleNotFoundError): + _ = importlib.import_module("orcapod.user_types") + + +# --------------------------------------------------------------------------- +# The schema check uses the class object +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", sorted(KINDS)) +def test_pod_accepts_its_own_class(kind: str) -> None: + """A pod annotated with a class accepts a column of that same class object.""" + cls = _define(KINDS[kind]) + result = _sum_pod(cls).pod(_source(cls(x=1, y=2))) + assert [data.as_dict()["t"] for _, data in result.iter_data()] == [3] + + +@pytest.mark.parametrize("kind", sorted(KINDS)) +def test_pod_refuses_a_class_defined_again(kind: str) -> None: + """A pod built for a class refuses the class defined again with the same fields.""" + first = _define(KINDS[kind]) + pod = _sum_pod(first) + second = _define(KINDS[kind]) + assert second is not first + + with pytest.raises(ValueError, match="column 'p'"): + _ = pod.pod(_source(second(x=1, y=2))) + + +@pytest.mark.parametrize("kind", sorted(KINDS)) +def test_class_defined_again_can_be_used_on_its_own(kind: str) -> None: + """Defining a class again does not stop either class from being used.""" + first = _define(KINDS[kind]) + before = _sum_pod(first).pod(_source(first(x=1, y=2))) + second = _define(KINDS[kind]) + after = _sum_pod(second).pod(_source(second(x=3, y=4))) + + assert [data.as_dict()["t"] for _, data in after.iter_data()] == [7] + assert [data.as_dict()["t"] for _, data in before.iter_data()] == [3] + + +# --------------------------------------------------------------------------- +# The case measured on November 12, 2026: the same answer in memory and from a store +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", sorted(KINDS)) +def test_chain_of_two_classes_is_refused_in_memory(kind: str) -> None: + """Pod A returns the first class; pod B, annotated with the second, refuses it.""" + first = _define(KINDS[kind]) + second = _define(KINDS[kind]) + produced = _make_pod(first).pod(_numbers()) + + with pytest.raises(ValueError, match="column 'q'"): + _ = _receive_pod(second).pod(produced) + + +@pytest.mark.parametrize("kind", sorted(KINDS)) +def test_chain_of_two_classes_is_refused_through_a_store( + kind: str, tmp_path: Path +) -> None: + """The same chain in a Delta-backed job is refused as well, before anything runs.""" + first = _define(KINDS[kind]) + second = _define(KINDS[kind]) + job = PipelineJob(name="exact", store=DeltaTableDatabase(tmp_path / "db")) + + with pytest.raises(ValueError, match="column 'q'"): + with job: + produced = _make_pod(first).pod(_numbers(), label="make") + _ = _receive_pod(second).pod(produced, label="receive") + + +# --------------------------------------------------------------------------- +# Subclasses: Python's subtype rule at a pod input, own identity in storage +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Base: + """An ordinary dataclass.""" + + x: int + + +@dataclasses.dataclass +class Child(Base): + """A subclass with a field of its own.""" + + z: int = 0 + + +@function_pod(output_keys="seen") +def describe_base(p: Base) -> str: + """Return the class name of the argument and its ``x``.""" + return f"{type(p).__name__}:{p.x}" + + +@function_pod(output_keys="q") +def child_as_base(x: int) -> Base: + """Return a ``Child`` although the pod is annotated ``-> Base``.""" + return Child(x, 9) + + +def test_subclass_column_is_accepted_at_a_base_class_parameter() -> None: + """A pod annotated with the base class accepts a column of the subclass.""" + source = _source(Child(1, 7)) + assert source.output_schema()[1]["p"] is Child + assert source.as_table().schema.field("p").type.storage_type.names == ["x", "z"] + + result = describe_base.pod(source) + assert [data.as_dict()["seen"] for _, data in result.iter_data()] == ["Child:1"] + + +def test_base_class_column_is_refused_at_a_subclass_parameter() -> None: + """The subtype rule has one direction: a base column is not a subclass column.""" + with pytest.raises(ValueError, match="column 'p'"): + _ = _sum_pod(Child).pod(_source(Base(1))) + + +def test_subclass_and_base_are_different_types() -> None: + """A subclass has its own identifier and pipeline hash.""" + base, child = _source(Base(1)), _source(Child(1)) + assert ( + base.as_table().schema.field("p").type.extension_name + != child.as_table().schema.field("p").type.extension_name + ) + assert base.pipeline_hash() != child.pipeline_hash() + + +def test_a_value_is_stored_only_as_its_own_class() -> None: + """A subclass instance returned for a base-class column is not stored as the base.""" + with pytest.raises(TypeError, match="Child"): + _ = child_as_base.pod(_numbers()).as_table() + + +def test_in_memory_iteration_hands_over_the_returned_object() -> None: + """Before any conversion, a pod's own ``iter_data`` gives the function's return value. + + Known behaviour: the datagram holds what the function returned, so iterating the pod's + output in memory shows the ``Child`` object. It is refused as soon as the output + becomes Arrow (``as_table``, an operator, a store); see the test above. + """ + values = [ + data.as_dict()["q"] for _, data in child_as_base.pod(_numbers()).iter_data() + ] + assert [type(value) for value in values] == [Child] + + +def test_a_subclass_is_returned_by_declaring_it() -> None: + """Declaring ``-> Child`` keeps all fields, and a ``Base`` parameter accepts the column.""" + + @function_pod(output_keys="p") + def make_child(x: int) -> Child: + """Return a ``Child``.""" + return Child(x, 9) + + produced = make_child.pod(_numbers()) + assert produced.as_table().column("p").to_pylist() == [{"x": 1, "z": 9}] + seen = describe_base.pod(produced) + assert [data.as_dict()["seen"] for _, data in seen.iter_data()] == ["Child:1"] diff --git a/tests/test_user_types/test_field_order.py b/tests/test_user_types/test_field_order.py new file mode 100644 index 000000000..5e3dbe9ff --- /dev/null +++ b/tests/test_user_types/test_field_order.py @@ -0,0 +1,197 @@ +"""Declared field order is part of a user class's identity (ITL-881, ITL-972). + +A dataclass compares its fields as a tuple in declared order, and a pydantic model +dumps, iterates and serializes in declared order, so pod code can observe the order. +Two classes with the same fields in another order are therefore different types. + +Identity is observed through the public interface: the Arrow type of a ``DictSource`` +column, the row's ``content_hash`` and the source's ``content_hash`` and +``pipeline_hash``. Structural classes are compared under names ``itl972.order.``; exact +classes (ordinary dataclass, pydantic) by defining one module path and name twice. +""" + +from __future__ import annotations + +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod import function_pod +from orcapod.sources import DictSource + + +@op.dataclass(name="itl972.order.Data") +class DataXY: + """Structural class ``itl972.order.Data`` with fields ``x``, ``y``.""" + + x: int + y: int + + +@op.dataclass(name="itl972.order.Data") +class DataYX: + """Structural class ``itl972.order.Data`` with fields ``y``, ``x``.""" + + y: int + x: int + + +def _exact(kind: str, order: str) -> type: + """Return an exact class ``Ordered`` of this module with fields in ``order``. + + Each call makes a new class object with the same module path and name, so two + classes that differ only in field order can be compared. + """ + fields = "".join(f" {name}: int\n" for name in order) + head = { + "dataclass": "import dataclasses\n@dataclasses.dataclass\nclass Ordered:\n", + "pydantic": "import pydantic\nclass Ordered(pydantic.BaseModel):\n", + }[kind] + namespace: dict[str, Any] = {"__name__": __name__} + exec(head + fields, namespace) + return namespace["Ordered"] + + +ModelXY, ModelYX = _exact("pydantic", "xy"), _exact("pydantic", "yx") +ExactXY, ExactYX = _exact("dataclass", "xy"), _exact("dataclass", "yx") + + +@op.dataclass(name="itl972.order.Outer") +class OuterOfXY: + """Structural class ``itl972.order.Outer``; nests ``DataXY``.""" + + inner: DataXY + + +@op.dataclass(name="itl972.order.Outer") +class OuterOfYX: + """Structural class ``itl972.order.Outer``; nests ``DataYX``.""" + + inner: DataYX + + +PAIRS: dict[str, tuple[Any, Any]] = { + "dataclass": (DataXY(x=1, y=2), DataYX(x=1, y=2)), + "pydantic": (ModelXY(x=1, y=2), ModelYX(x=1, y=2)), + "exact_dataclass": (ExactXY(x=1, y=2), ExactYX(x=1, y=2)), + "list": ([DataXY(x=1, y=2)], [DataYX(x=1, y=2)]), + "nested": (OuterOfXY(DataXY(x=1, y=2)), OuterOfYX(DataYX(x=1, y=2))), +} + + +def _source(value: Any) -> DictSource: + """Return a one-row ``DictSource`` holding ``value`` in data column ``p``.""" + return DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"], source_id="s") + + +def _column_type(value: Any) -> pa.ExtensionType: + """Return the Arrow extension type of the column holding ``value``.""" + column_type = _source(value).as_table().schema.field("p").type + assert isinstance(column_type, pa.ExtensionType) + return column_type + + +def _type_name(value: Any) -> str: + """Return the Arrow extension name of the column holding ``value``.""" + return _column_type(value).extension_name + + +def _row_hash(value: Any) -> str: + """Return the content hash of the row holding ``value``.""" + _, data = next(iter(_source(value).iter_data())) + return data.content_hash().to_string() + + +@pytest.fixture(params=list(PAIRS)) +def pair(request: pytest.FixtureRequest) -> tuple[Any, Any]: + """Two values that differ only in the declared field order of their class.""" + return PAIRS[request.param] + + +def test_other_field_order_is_another_type(pair: tuple[Any, Any]) -> None: + """The same fields in another declared order give another extension name.""" + first, second = pair + assert _type_name(first) != _type_name(second) + + +def test_other_field_order_has_other_hashes(pair: tuple[Any, Any]) -> None: + """Another field order changes the row, content and pipeline hashes.""" + first, second = pair + assert _row_hash(first) != _row_hash(second) + assert _source(first).content_hash() != _source(second).content_hash() + assert _source(first).pipeline_hash() != _source(second).pipeline_hash() + + +@pytest.mark.parametrize( + ("value", "names"), + [ + (DataXY(x=1, y=2), ["x", "y"]), + (DataYX(x=1, y=2), ["y", "x"]), + (ModelXY(x=1, y=2), ["x", "y"]), + (ModelYX(x=1, y=2), ["y", "x"]), + ], +) +def test_stored_struct_lists_fields_in_declared_order( + value: Any, names: list[str] +) -> None: + """The stored struct lists its fields in the class's declared order.""" + storage = _column_type(value).storage_type + assert isinstance(storage, pa.StructType) + assert storage.names == names + + +def test_values_of_both_orders_read_back_as_their_own_class( + pair: tuple[Any, Any], +) -> None: + """A value of either order reads back equal and as its own class.""" + for value in pair: + _, data = next(iter(_source(value).iter_data())) + read = data.as_dict()["p"] + assert read == value + assert type(read) is type(value) + + +def test_pods_over_the_two_orders_have_different_pipeline_hashes() -> None: + """Pods over the two orders differ in pipeline hash and output type.""" + + @function_pod(output_keys="q") + def keep_xy(p: DataXY) -> DataXY: + """Return the ``DataXY`` value unchanged.""" + return p + + @function_pod(output_keys="q") + def keep_yx(p: DataYX) -> DataYX: + """Return the ``DataYX`` value unchanged.""" + return p + + xy = keep_xy.pod(_source(DataXY(x=1, y=2))) + yx = keep_yx.pod(_source(DataYX(x=1, y=2))) + assert xy.pipeline_hash() != yx.pipeline_hash() + assert ( + xy.as_table().schema.field("q").type.extension_name + != yx.as_table().schema.field("q").type.extension_name + ) + + +@op.dataclass(name="itl972.order.Same") +class SameFirst: + """First structural class ``itl972.order.Same``.""" + + a: int + b: str + + +@op.dataclass(name="itl972.order.Same") +class SameSecond: + """Second structural class ``itl972.order.Same``, same fields in the same order.""" + + a: int + b: str + + +def test_same_fields_in_the_same_order_are_one_type() -> None: + """Two classes with one name and the same ordered fields are one type.""" + assert _type_name(SameFirst(1, "x")) == _type_name(SameSecond(1, "x")) + assert _row_hash(SameFirst(1, "x")) == _row_hash(SameSecond(1, "x")) diff --git a/tests/test_user_types/test_identity_record.py b/tests/test_user_types/test_identity_record.py new file mode 100644 index 000000000..3ab191271 --- /dev/null +++ b/tests/test_user_types/test_identity_record.py @@ -0,0 +1,212 @@ +"""Identity record, identity string, type tokens and names of user classes (ITL-881, ITL-972). + +These tests target ``orcapod.logical_types.user_class_identity`` directly: they pin the +documented identity format, which is that module's own interface. +""" + +from __future__ import annotations + +import dataclasses +import datetime +import typing +from pathlib import Path +from typing import Literal, Optional + +import pytest + +from orcapod.contexts import get_default_context +from orcapod.logical_types import user_class_identity as uci +from orcapod.logical_types.structural_class import annotation_from_token + + +@pytest.fixture +def converter(): + """Return the default data context's type converter.""" + return get_default_context().type_converter + + +def _make_class(module: str, qualname: str = "Point") -> type: + """Return a one-field dataclass with the given module and qualified name.""" + cls = dataclasses.make_dataclass(qualname, [("x", int)]) + cls.__module__ = module + cls.__qualname__ = qualname + return cls + + +class TestIdentityString: + """The identity record, its canonical JSON and the identity string.""" + + def test_documented_example(self) -> None: + """The documented example record gives the documented identity string.""" + record = uci.identity_record( + "dataclass", "Point", [("x", "int"), ("y", "float")] + ) + assert uci.identity_string(record) == ( + "Point@76d961e40619a836535e3fdebdead7ea65aa9413955a889f3d5b51a7a94b65eb" + ) + + def test_canonical_json_is_compact_and_keeps_field_order(self) -> None: + """Canonical JSON is compact, sorts keys and keeps the field order.""" + record = uci.identity_record("pydantic", "P", [("b", "int"), ("a", "str")]) + assert ( + uci.canonical_json(record) + == '{"fields":[["b","int"],["a","str"]],"kind":"pydantic","name":"P"}' + ) + + def test_field_order_is_part_of_the_identity(self) -> None: + """Records differing only in field order have different identity strings.""" + a = uci.identity_record("dataclass", "P", [("x", "int"), ("y", "str")]) + b = uci.identity_record("dataclass", "P", [("y", "str"), ("x", "int")]) + assert uci.identity_string(a) != uci.identity_string(b) + + @pytest.mark.parametrize( + "other", + [ + ("pydantic", "Point", [("x", "int"), ("y", "float")]), + ("dataclass", "Point2", [("x", "int"), ("y", "float")]), + ("dataclass", "Point", [("x", "int"), ("y", "int")]), + ("dataclass", "Point", [("x", "int"), ("z", "float")]), + ("dataclass", "Point", [("x", "int")]), + ("dataclass", "Point", [("y", "float"), ("x", "int")]), + ], + ) + def test_each_component_changes_the_identity( + self, other: tuple[str, str, list[tuple[str, str]]] + ) -> None: + """Changing the kind, the name or any field changes the identity string.""" + base = uci.identity_record("dataclass", "Point", [("x", "int"), ("y", "float")]) + assert uci.identity_string(base) != uci.identity_string( + uci.identity_record(*other) + ) + + def test_describe_fields_keeps_the_given_order(self) -> None: + """``describe_fields`` renders the fields in the order given.""" + assert ( + uci.describe_record( + uci.identity_record("dataclass", "P", [("y", "float"), ("x", "int")]) + ) + == "P (dataclass) {y: float, x: int}" + ) + + +class TestTypeTokens: + """Type tokens of field annotations.""" + + @pytest.mark.parametrize( + ("annotation", "token"), + [ + (int, "int"), + (str, "str"), + (bool, "bool"), + (bytes, "bytes"), + (float, "float"), + (datetime.datetime, "datetime.datetime"), + (int | None, {"union": ["int", "None"]}), + (Optional[int], {"union": ["int", "None"]}), + (None | int, {"union": ["int", "None"]}), + (str | int, {"union": ["int", "str"]}), + (list[int], {"generic": "list", "args": ["int"]}), + (typing.List[int], {"generic": "list", "args": ["int"]}), # noqa: UP006 - the typing alias is the case under test + (set[str], {"generic": "set", "args": ["str"]}), + (dict[str, float], {"generic": "dict", "args": ["str", "float"]}), + ( + list[int | None], + {"generic": "list", "args": [{"union": ["int", "None"]}]}, + ), + (Literal["b", "a"], {"literal": ["'a'", "'b'"]}), + (tuple[int, ...], {"generic": "tuple", "args": ["int", "..."]}), + ], + ) + def test_token(self, converter, annotation: object, token: object) -> None: + """Each annotation maps to its canonical type token.""" + assert uci.type_token(annotation, converter) == token + + def test_logical_type_token_is_the_whole_identifier(self, converter) -> None: + """A type with a logical type contributes its name and its metadata.""" + from orcapod.logical_types.identifier import logical_type_identifier + + lt = converter.get_logical_type_for_python_type(Path) + assert uci.type_token(Path, converter) == { + "logical": logical_type_identifier(lt).text + } + assert uci.type_token(Path, converter)["logical"].startswith("orcapod.path;") + + def test_nested_class_token_is_its_record(self, converter) -> None: + """A field typed by a user class embeds that class's whole record.""" + inner = _make_class("pkg.models", "Inner") + assert uci.type_token(inner, converter) == { + "class": { + "kind": "dataclass", + "name": "pkg.models.Inner", + "fields": [["x", "int"]], + } + } + + @pytest.mark.parametrize( + "annotation", + [ + int, + datetime.datetime, + Path, + int | None, + list[int], + dict[str, float], + list[Path | None], + tuple[int, str], + tuple[int, ...], + Literal["a", "b"], + ], + ) + def test_annotation_is_rebuilt_from_its_token( + self, converter, annotation: object + ) -> None: + """A token gives back an annotation with the same token.""" + token = uci.type_token(annotation, converter) + rebuilt = annotation_from_token(token, converter) + assert rebuilt == annotation + assert uci.type_token(rebuilt, converter) == token + + +class TestNames: + """Name validation and the names of exact classes.""" + + @pytest.mark.parametrize( + "name", ["", "a@b", "a b", "a;b", "orcapod.thing", "tab\tname", 3] + ) + def test_invalid_names_rejected(self, name: object) -> None: + """Empty, non-string, reserved, ``@``, ``;`` and whitespace names are rejected.""" + with pytest.raises((ValueError, TypeError)): + uci.validate_name(name) # pyright: ignore[reportArgumentType] + + @pytest.mark.parametrize("name", ["Point", "geo.Point", "my-types.Point_2"]) + def test_valid_names_accepted(self, name: str) -> None: + """Dotted names with letters, digits, ``-`` and ``_`` are accepted.""" + assert uci.validate_name(name) == name + + def test_names_are_nfc_normalized(self) -> None: + """Composed and decomposed spellings of a name are one NFC name.""" + composed, decomposed = "Caf\u00e9", "Cafe\u0301" + assert uci.validate_name(decomposed) == uci.validate_name(composed) == composed + + def test_exact_class_is_named_by_module_path(self) -> None: + """An exact class is named by its module path and class name.""" + assert uci.class_name(_make_class("pkg.models")) == "pkg.models.Point" + + @pytest.mark.parametrize("module", ["__main__", "__mp_main__"]) + def test_main_class_is_named_under_main(self, module: str) -> None: + """A class in ``__main__`` or ``__mp_main__`` is named ``__main__.``.""" + assert uci.class_name(_make_class(module)) == "__main__.Point" + assert ( + uci.class_name(_make_class(module, "Outer.Point")) == "__main__.Outer.Point" + ) + + def test_structural_class_is_named_by_its_declared_name(self) -> None: + """A structural class is named by its declared name, not its module.""" + import orcapod as op + + @op.dataclass(name="geo.Point") + class Point: + x: int + + assert uci.class_kind(Point) == "structural" + assert uci.class_name(Point) == "geo.Point" diff --git a/tests/test_user_types/test_measured_case.py b/tests/test_user_types/test_measured_case.py new file mode 100644 index 000000000..c35ff24ab --- /dev/null +++ b/tests/test_user_types/test_measured_case.py @@ -0,0 +1,104 @@ +"""Two structural classes of one type behave the same in memory and from a store (ITL-972). + +The case measured on November 12, 2026: two classes of one identity, the second with a +method the first lacks. Pod A returns the first; pod B is annotated with the second. +Orcapod checks that the two classes have the same structural identity and passes the +value as it is, without converting it: in memory B receives pod A's object, from a store +the class bound to the type. A structural class promises its fields, not its methods, so +on both paths B can read the fields and the value equals the instance B would build. +""" + +from __future__ import annotations + +from pathlib import Path + +import orcapod as op +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import DictSource + + +@op.dataclass(name="itl972.measured.Q") +class QOld: + """The first definition: no ``describe``.""" + + x: int + y: int + + +@op.dataclass(name="itl972.measured.Q") +class QNew: + """The second definition: same name and fields, plus ``describe``.""" + + x: int + y: int + + def describe(self) -> str: + """Return a text only this class can produce; a ``QOld`` has no such method.""" + return f"QNew({self.x}, {self.y})" + + +@function_pod(output_keys="q") +def make(x: int) -> QOld: + """Return a ``QOld``.""" + return QOld(x, x + 1) + + +@function_pod(output_keys="seen") +def look(q: QNew) -> str: + """Read the fields of the argument and compare it with a ``QNew`` built here.""" + return f"{q.x},{q.y},{q == QNew(q.x, q.y)}" + + +def _source() -> DictSource: + """Return a two-row source of ``x`` values.""" + return DictSource( + data=[{"k": "a", "x": 1}, {"k": "b", "x": 2}], + tag_columns=["k"], + source_id="s", + ) + + +EXPECTED = ["1,2,True", "2,3,True"] + + +def _job(tmp_path: Path) -> PipelineJob: + """Return the two-pod job over a Delta store in ``tmp_path``, already run.""" + job = PipelineJob(name="measured", store=DeltaTableDatabase(tmp_path / "db")) + with job: + _ = look.pod(make.pod(_source(), label="make"), label="look") + job.run() + return job + + +def _seen(stream: object) -> list[str]: + """Return the sorted ``seen`` values of a stream or node.""" + return sorted(data.as_dict()["seen"] for _, data in stream.iter_data()) # pyright: ignore[reportAttributeAccessIssue] # a stream or a job node + + +def test_in_memory_chain_is_compatible() -> None: + """In memory, pod B accepts pod A's output, reads its fields and finds it equal.""" + assert _seen(look.pod(make.pod(_source()))) == EXPECTED + + +def test_stored_chain_is_compatible(tmp_path: Path) -> None: + """Through a Delta-backed job the result is the same.""" + assert _seen(_job(tmp_path).nodes["look"]) == EXPECTED + + +def test_second_run_reuses_stored_rows_with_the_same_result(tmp_path: Path) -> None: + """A second run reads the stored rows and gives the same result.""" + assert _seen(_job(tmp_path).nodes["look"]) == EXPECTED + assert _seen(_job(tmp_path).nodes["look"]) == EXPECTED + + +def test_values_are_passed_as_they_are() -> None: + """Orcapod does not convert: in memory pod B receives pod A's own object.""" + + @function_pod(output_keys="cls") + def class_name(q: QNew) -> str: + """Return the class name of the argument.""" + return type(q).__name__ + + stream = class_name.pod(make.pod(_source())) + assert {data.as_dict()["cls"] for _, data in stream.iter_data()} == {"QOld"} diff --git a/tests/test_user_types/test_read_path.py b/tests/test_user_types/test_read_path.py new file mode 100644 index 000000000..0a74e3474 --- /dev/null +++ b/tests/test_user_types/test_read_path.py @@ -0,0 +1,145 @@ +"""Stored and registered shapes of a user class must agree on read (ITL-881, S2). + +A stored column whose extension name matches a registered class but whose Arrow storage +differs (possible only if two data contexts map a field differently, or a table was +written by other means) must raise ``TypeIdentityError`` from ``DeltaTableDatabase`` +instead of being logged and read as "no records". +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import Any + +import deltalake +import pyarrow as pa +import pytest + +from orcapod.databases import DeltaTableDatabase +from orcapod.errors import TypeIdentityError +from orcapod.sources import DictSource + + +@dataclasses.dataclass +class StoredPoint: + """An ordinary dataclass whose stored shape the tests change.""" + + x: int + y: int + + +def _extension_field_metadata() -> dict[bytes | str, bytes | str]: + """Return Arrow field metadata marking a column as a ``StoredPoint`` column.""" + value: Any = StoredPoint(1, 2) + source = DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) + ext = source.as_table().schema.field("p").type + return { + b"ARROW:extension:name": ext.extension_name.encode(), + b"ARROW:extension:metadata": ext.__arrow_ext_serialize__(), + } + + +def test_storage_mismatch_raises_instead_of_reading_nothing(tmp_path: Path) -> None: + """A stored column whose storage differs from its class raises on read.""" + wrong = pa.struct([pa.field("x", pa.float64()), pa.field("y", pa.int64())]) + field = pa.field("q", wrong).with_metadata(_extension_field_metadata()) + table = pa.table( + {"q": pa.array([{"x": 1.5, "y": 2}], type=wrong)}, + schema=pa.schema([field]), + ) + deltalake.write_deltalake(str(tmp_path / "points"), table) + + with pytest.raises(TypeIdentityError, match="expected storage_type"): + DeltaTableDatabase(tmp_path).get_all_records(("points",)) + + +@dataclasses.dataclass +class Labelled: + """An ordinary dataclass with a string field.""" + + label: str + n: int + + +def _run_labelled(db_root: Path) -> tuple[int, list[Labelled]]: + """Run an echo pod over two ``Labelled`` rows; return the call count and values.""" + from orcapod import PipelineJob, function_pod + + calls: list[Labelled] = [] + + @function_pod(output_keys="q") + def echo(p: Labelled) -> Labelled: + """Record the call and return ``p`` unchanged.""" + calls.append(p) + return p + + rows: list[Any] = [{"k": f"r{i}", "p": Labelled(f"x{i}", i)} for i in (1, 2)] + job = PipelineJob(name="compact", store=DeltaTableDatabase(db_root)) + with job: + echo.pod(DictSource(data=rows, tag_columns=["k"], source_id="s"), label="echo") + # One commit per row, so the result table has one file per row to compact. + job.run(commit_every_rows=1) + values = [data.as_dict()["q"] for _, data in job.nodes["echo"].iter_data()] + return len(calls), sorted(values, key=lambda v: v.n) + + +def test_stored_values_read_after_delta_compaction(tmp_path: Path) -> None: + """Delta compaction rewrites large_string struct fields as string; still readable.""" + expected = [Labelled("x1", 1), Labelled("x2", 2)] + assert _run_labelled(tmp_path) == (2, expected) + rewritten: list[str] = [] + for log in tmp_path.rglob("_delta_log"): + table = deltalake.DeltaTable(str(log.parent)) + before = set(table.file_uris()) + metrics = table.optimize.compact() + if "q" in [field.name for field in table.schema().fields]: + # The result table must really be rewritten, or the cached read below + # would pass on the files the first run wrote. + assert metrics["numFilesRemoved"] == len(before) >= 2 + assert metrics["numFilesAdded"] == 1 + after = set(deltalake.DeltaTable(str(log.parent)).file_uris()) + assert len(after) == 1 and not after & before + rewritten.append(str(log.parent)) + assert len(rewritten) == 1 + assert _run_labelled(tmp_path) == (0, expected) + + +@pytest.mark.xfail( + strict=True, + raises=pa.ArrowInvalid, + reason=( + "ITL-894: _deserialize accepts a non-large storage variant but returns the " + "large storage type, so IPC reinterprets 32-bit offsets as 64-bit. Parquet " + "and Delta reads convert the data and are correct." + ), +) +def test_ipc_read_of_non_large_nested_storage_is_valid() -> None: + """A user-class column with non-large nested storage reads correctly over IPC.""" + import io + + narrow = pa.struct([pa.field("label", pa.string()), pa.field("n", pa.int64())]) + value: Any = Labelled("x1", 1) + ext = ( + DictSource(data=[{"k": "a", "p": value}], tag_columns=["k"]) + .as_table() + .schema.field("p") + .type + ) + metadata: dict[bytes | str, bytes | str] = { + b"ARROW:extension:name": ext.extension_name.encode(), + b"ARROW:extension:metadata": ext.__arrow_ext_serialize__(), + } + table = pa.table( + {"p": pa.array([{"label": "hello", "n": 1}], type=narrow)}, + schema=pa.schema([pa.field("p", narrow).with_metadata(metadata)]), + ) + sink = io.BytesIO() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + read = pa.ipc.open_stream(sink.getvalue()).read_all() + read.validate(full=True) + read_type = read.schema.field("p").type + assert read_type.extension_name == ext.extension_name + assert read_type.__arrow_ext_serialize__() == ext.__arrow_ext_serialize__() + assert read.column("p").to_pylist() == [{"label": "hello", "n": 1}] diff --git a/tests/test_user_types/test_rebind_and_compatibility.py b/tests/test_user_types/test_rebind_and_compatibility.py new file mode 100644 index 000000000..6ed646933 --- /dev/null +++ b/tests/test_user_types/test_rebind_and_compatibility.py @@ -0,0 +1,187 @@ +"""Rebound user classes in containers, in unions, and foreign storage (ITL-881). + +Two classes declared under one name with the same fields have one identity; the second +definition rebinds it, as a re-run notebook cell does. These tests cover what follows a +rebind: the schema of a list of the class, a pod parameter typed as a union with the +class, and storage that differs from the registered class. Every test declares names under +``itl972.rebind.`` that no other test uses, because declarations are process-wide. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod import function_pod +from orcapod.errors import TypeIdentityError +from orcapod.sources import ArrowTableSource, DictSource +from tests.test_user_types.child_harness import run_child + + +def _source(key: str, value: Any) -> DictSource: + """Return a one-row ``DictSource`` holding ``value`` in data column ``key``.""" + return DictSource(data=[{"k": "a", key: value}], tag_columns=["k"]) + + +def _read(source: Any, key: str) -> Any: + """Return the value of data column ``key`` in the source's only row.""" + return [data.as_dict()[key] for _, data in source.iter_data()][0] + + +# --------------------------------------------------------------------------- +# A list of a rebound class +# --------------------------------------------------------------------------- + + +@op.dataclass(name="itl972.rebind.Point", frozen=True) +class PointV1: + """First definition under ``itl972.rebind.Point``.""" + + x: int + + +@op.dataclass(name="itl972.rebind.Point", frozen=True) +class PointV2: + """Second definition under ``itl972.rebind.Point``, with the same fields.""" + + x: int + + +def test_list_schema_names_the_rebound_class() -> None: + """After a rebind, a list column's schema and values use the new class.""" + first = _source("ps", [PointV1(1)]) + stored = first.as_table() + assert first.output_schema()[1]["ps"] == list[PointV1] + + second = _source("ps", [PointV2(1)]) + assert second.output_schema()[1]["ps"] == list[PointV2] + assert _read(second, "ps") == [PointV2(1)] + + # A table written before the rebind reads back as the new class as well. + reread = ArrowTableSource(stored.select(["k", "ps"]), tag_columns=["k"]) + assert reread.output_schema()[1]["ps"] == list[PointV2] + assert type(_read(reread, "ps")[0]) is PointV2 + + +@op.dataclass(name="itl972.rebind.Cell", frozen=True) +class CellV1: + """First definition under ``itl972.rebind.Cell``.""" + + x: int + + +@op.dataclass(name="itl972.rebind.Cell", frozen=True) +class CellV2: + """Second definition under ``itl972.rebind.Cell``, with the same fields.""" + + x: int + + +def test_nested_list_schema_names_the_rebound_class() -> None: + """A list of lists follows the rebind of its innermost class.""" + first = _source("grid", [[CellV1(1)]]) + assert first.output_schema()[1]["grid"] == list[list[CellV1]] + second = _source("grid", [[CellV2(1)]]) + assert second.output_schema()[1]["grid"] == list[list[CellV2]] + assert _read(second, "grid") == [[CellV2(1)]] + + +# --------------------------------------------------------------------------- +# An identity-equivalent class offered to a union parameter +# --------------------------------------------------------------------------- + + +@op.dataclass(name="itl972.rebind.Reading") +class ReadingOld: + """First definition under ``itl972.rebind.Reading``.""" + + x: int + + +@op.dataclass(name="itl972.rebind.Reading") +class ReadingNew: + """Second definition under ``itl972.rebind.Reading``, with the same fields.""" + + x: int + + +@function_pod(output_keys="p") +def make_old(n: int) -> ReadingOld: + """Return a ``ReadingOld`` holding ``n``.""" + return ReadingOld(n) + + +@function_pod(output_keys="y") +def read_optional(p: ReadingNew | None) -> int: + """Return the reading's ``x``, or ``-1`` for no reading.""" + return -1 if p is None else p.x + + +@function_pod(output_keys="y") +def read_int_or_str(p: int | str) -> int: + """Return ``0``; only the parameter type matters.""" + return 0 + + +def test_old_class_output_feeds_a_union_parameter_of_the_new_class() -> None: + """A pod output typed as the old class is accepted by ``New | None``.""" + source = DictSource(data=[{"k": "a", "n": 3}], tag_columns=["k"]) + out = read_optional.pod(make_old.pod(source)) + assert _read(out, "y") == 3 + + +def test_union_parameter_still_rejects_another_type() -> None: + """A union parameter with no member matching the column is still refused.""" + source = DictSource(data=[{"k": "a", "n": 3}], tag_columns=["k"]) + with pytest.raises(ValueError, match="column 'p' has type"): + read_int_or_str.pod(make_old.pod(source)) + + +# --------------------------------------------------------------------------- +# Storage that differs from the registered class +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class Named: + """An ordinary dataclass with a string field.""" + + label: str + n: int + + +def _struct(label_type: pa.DataType) -> pa.DataType: + """Return ``Named``'s storage struct with ``label`` of ``label_type``.""" + return pa.struct([pa.field("label", label_type), pa.field("n", pa.int64())]) + + +def test_null_storage_is_not_the_registered_string_storage() -> None: + """A ``null`` field stored where the class has a string raises ``TypeIdentityError``.""" + ext = _source("p", Named("x", 1)).as_table().schema.field("p").type + metadata = ext.__arrow_ext_serialize__() + # Non-public entry point: pyarrow's extension hook. No public path produces this + # input (Delta refuses a null column; Parquet does not rebuild the extension type). + deserialize = type(ext).__arrow_ext_deserialize__ + # A layout variant of the same type is still accepted (Delta compaction). + assert deserialize(_struct(pa.string()), metadata).storage_type == ext.storage_type + with pytest.raises(TypeIdentityError, match="expected storage_type"): + deserialize(_struct(pa.null()), metadata) + + +# --------------------------------------------------------------------------- +# The child-process harness +# --------------------------------------------------------------------------- + + +def test_run_child_rejects_a_report_from_a_failed_child(tmp_path: Path) -> None: + """A child that reports and then exits non-zero is a failure, not a result.""" + code = "print('JSON:1', flush=True); raise SystemExit(3)" + with pytest.raises(AssertionError, match="exit 3"): + run_child(["-c", code], cwd=tmp_path) + ok = "print('JSON:1', flush=True)" + assert run_child(["-c", ok], cwd=tmp_path) == 1 diff --git a/tests/test_user_types/test_structural_dataclass.py b/tests/test_user_types/test_structural_dataclass.py new file mode 100644 index 000000000..86a6beb40 --- /dev/null +++ b/tests/test_user_types/test_structural_dataclass.py @@ -0,0 +1,750 @@ +"""``orcapod.dataclass``: a dataclass identified by its name and fields (ITL-972). + +Covers the decorator (a real dataclass, its options, what it refuses), the identity +(name plus ordered fields; the class object and its module do not matter), equality +across classes of one type, explicit conversion with ``from_instance``, and reading +stored values in a process that does not define the class. +""" + +from __future__ import annotations + +import dataclasses +import inspect +from pathlib import Path +from typing import Any + +import pydantic +import pytest + +import orcapod as op +from orcapod import function_pod +from orcapod.sources import DictSource +from tests.test_user_types.child_harness import WSJOB, run_child, write_files + + +def _source(value: Any, key: str = "p") -> DictSource: + """Return a one-row source holding ``value`` in data column ``key``.""" + return DictSource(data=[{"k": "a", key: value}], tag_columns=["k"], source_id="s") + + +def _extension_name(source: DictSource, key: str = "p") -> str: + """Return the Arrow extension name of data column ``key``.""" + return source.as_table().schema.field(key).type.extension_name + + +def _convert(cls: type, other: Any) -> Any: + """Return ``cls.from_instance(other)``. + + ``from_instance`` is added by the decorator, which static type checkers cannot see. + """ + return cls.from_instance(other) # pyright: ignore[reportAttributeAccessIssue] # added by op.dataclass at run time + + +def _read(source: Any, key: str = "p") -> Any: + """Return the value of data column ``key`` in the stream's only row.""" + return [data.as_dict()[key] for _, data in source.iter_data()][0] + + +# --------------------------------------------------------------------------- +# The decorator +# --------------------------------------------------------------------------- + + +@op.dataclass +class Plain: + """Bare decorator: named ``Plain``.""" + + x: int + y: float = 0.5 + + +@op.dataclass(name="itl972.Frozen", frozen=True, slots=True, order=True) +class Frozen: + """Decorator with a name and dataclass options.""" + + x: int + y: int + + +def test_result_is_a_real_dataclass() -> None: + """The class works with the ``dataclasses`` module like any dataclass.""" + assert dataclasses.is_dataclass(Plain) + assert [f.name for f in dataclasses.fields(Plain)] == ["x", "y"] + p = Plain(1) + assert p == Plain(1, 0.5) + assert dataclasses.asdict(p) == {"x": 1, "y": 0.5} + assert dataclasses.replace(p, x=2) == Plain(2, 0.5) + assert repr(p) == "Plain(x=1, y=0.5)" + assert Plain(x=3, y=1.0).y == 1.0 + + +def test_constructor_signature_is_the_dataclass_signature() -> None: + """``inspect.signature`` shows the fields, as for a plain dataclass.""" + assert list(inspect.signature(Plain).parameters) == ["x", "y"] + + +def test_dataclass_options_are_passed_through() -> None: + """``frozen``, ``slots`` and ``order`` behave as in ``dataclasses.dataclass``.""" + f = Frozen(1, 2) + with pytest.raises(dataclasses.FrozenInstanceError): + f.x = 5 # pyright: ignore[reportAttributeAccessIssue] # the assignment must fail + assert not hasattr(f, "__dict__") + assert Frozen(1, 2) < Frozen(1, 3) + assert hash(f) == hash(Frozen(1, 2)) + + +def test_init_false_is_refused() -> None: + """``init=False`` is not accepted.""" + with pytest.raises(TypeError, match="init=False"): + + @op.dataclass(init=False) + class NoInit: + x: int + + +def test_field_level_compare_and_hash_choices_are_refused() -> None: + """Equality and hashing cover exactly the stored fields, for every class of a type.""" + with pytest.raises(TypeError, match="'payload'.*compare=False"): + + @op.dataclass + class Uncompared: + x: int + payload: list[int] = dataclasses.field(default_factory=list, compare=False) + + with pytest.raises(TypeError, match="'payload'.*hash=False"): + + @op.dataclass(frozen=True) + class Unhashed: + x: int + payload: tuple[int, ...] = dataclasses.field(default=(), hash=False) + + with pytest.raises(TypeError, match="'cached'.*init=False"): + + @op.dataclass + class Derived: + x: int + cached: int = dataclasses.field(init=False, default=0) + + @op.dataclass(name="itl972.DerivedOk") + class DerivedOk: + x: int + cached: int = dataclasses.field(init=False, default=0, compare=False) + + assert DerivedOk(1) == DerivedOk(1) + + +def test_an_existing_dataclass_is_refused() -> None: + """The decorator defines the dataclass; it is not applied on top of one.""" + with pytest.raises(TypeError, match="already a dataclass"): + + @op.dataclass + @dataclasses.dataclass + class Twice: + x: int + + +def test_a_pydantic_model_is_refused() -> None: + """A pydantic model cannot be made structural.""" + with pytest.raises(TypeError, match="pydantic"): + + @op.dataclass + class Model(pydantic.BaseModel): + x: int + + +@pytest.mark.parametrize("name", ["", "a b", "a@b", "a;b", "orcapod.thing"]) +def test_invalid_names_are_refused(name: str) -> None: + """A name must not be empty, contain ``@``, ``;`` or whitespace, or be reserved.""" + with pytest.raises(ValueError, match="name"): + + @op.dataclass(name=name) + class Named: + x: int + + +# --------------------------------------------------------------------------- +# Identity: name plus ordered fields +# --------------------------------------------------------------------------- + + +@op.dataclass(name="itl972.Pair") +class PairA: + """``itl972.Pair`` with fields ``x``, ``y``.""" + + x: int + y: int + + +@op.dataclass(name="itl972.Pair") +class PairB: + """A second class of the same name and fields.""" + + x: int + y: int + + +@op.dataclass(name="itl972.Pair") +class PairReordered: + """The same name with the fields in another order: another type.""" + + y: int + x: int + + +@op.dataclass(name="itl972.OtherPair") +class PairRenamed: + """The same fields under another name: another type.""" + + x: int + y: int + + +@dataclasses.dataclass +class OrdinaryPair: + """An ordinary dataclass with the same fields: another type.""" + + x: int + y: int + + +def test_default_name_is_the_class_name() -> None: + """Without ``name=`` the type is named by the class name, not the module path.""" + assert _extension_name(_source(Plain(1))).split("@")[0] == "Plain" + + +def test_same_name_and_fields_is_one_type() -> None: + """Two classes with one name and the same fields share identifier and hashes.""" + a, b = _source(PairA(1, 2)), _source(PairB(1, 2)) + assert _extension_name(a) == _extension_name(b) + assert a.pipeline_hash() == b.pipeline_hash() + assert a.as_table().schema.field("p").type == b.as_table().schema.field("p").type + + +@pytest.mark.parametrize( + "other", [PairReordered(2, 1), PairRenamed(1, 2), OrdinaryPair(1, 2)] +) +def test_other_order_name_or_kind_is_another_type(other: Any) -> None: + """Field order, the name and the kind of class are each part of the identity.""" + a, b = _source(PairA(1, 2)), _source(other) + assert _extension_name(a) != _extension_name(b) + assert a.pipeline_hash() != b.pipeline_hash() + + +def test_a_local_class_is_allowed() -> None: + """A structural class may be defined inside a function.""" + + @op.dataclass(name="itl972.Local") + class Local: + x: int + + assert _read(_source(Local(3))) == Local(3) + + +def test_a_field_typed_by_an_exact_class_is_refused() -> None: + """Every class inside a structural class must be structural.""" + + @op.dataclass(name="itl972.HoldsOrdinary") + class HoldsOrdinary: + inner: OrdinaryPair + + with pytest.raises(TypeError, match="inner"): + _ = _source(HoldsOrdinary(OrdinaryPair(1, 2))) + + +def test_pod_accepts_another_class_of_the_same_type() -> None: + """A pod annotated with one class accepts a column of the other, fields intact.""" + + @function_pod(output_keys="seen") + def look(p: PairB) -> str: + """Return the fields of the argument and whether it equals a ``PairB``.""" + return f"{p.x},{p.y},{p == PairB(p.x, p.y)}" + + assert _read(look.pod(_source(PairA(1, 2))), "seen") == "1,2,True" + + +# --------------------------------------------------------------------------- +# Equality across classes of one type +# --------------------------------------------------------------------------- + + +def test_instances_of_two_classes_of_one_type_are_equal() -> None: + """Equal field values make instances of two classes of one type equal.""" + assert PairA(1, 2) == PairB(1, 2) + assert PairB(1, 2) == PairA(1, 2) + assert PairA(1, 2) != PairB(1, 3) + assert not (PairA(1, 2) != PairB(1, 2)) + + +@pytest.mark.parametrize( + "other", [PairReordered(2, 1), PairRenamed(1, 2), OrdinaryPair(1, 2), (1, 2)] +) +def test_instances_of_another_type_are_not_equal(other: Any) -> None: + """Another order, name or kind of class is another type: never equal.""" + assert PairA(1, 2) != other + assert other != PairA(1, 2) + + +def test_nested_structural_fields_compare_the_same_way() -> None: + """Equality reaches through fields, lists and optionals of structural classes.""" + a = Segment(PairA(1, 2), [PairA(3, 4)], PairA(5, 6)) + b = SegmentB(PairB(1, 2), [PairB(3, 4)], PairB(5, 6)) + assert a == b + assert a != SegmentB(PairB(1, 2), [PairB(3, 9)], PairB(5, 6)) + + +def test_hash_agrees_with_equality() -> None: + """Hashable structural instances that are equal have equal hashes.""" + + @op.dataclass(name="itl972.Key", frozen=True) + class KeyA: + x: int + + @op.dataclass(name="itl972.Key", frozen=True) + class KeyB: + x: int + + assert KeyA(1) == KeyB(1) + assert hash(KeyA(1)) == hash(KeyB(1)) + lookup: dict[Any, str] = {KeyA(1): "a"} + assert lookup[KeyB(1)] == "a" + + +def test_eq_false_and_a_class_eq_are_left_alone() -> None: + """``eq=False`` and an ``__eq__`` written in the class are not replaced.""" + + @op.dataclass(name="itl972.NoEq", eq=False) + class NoEq: + x: int + + @op.dataclass(name="itl972.OwnEq") + class OwnEq: + x: int + + def __eq__(self, other: object) -> bool: + """Everything is equal.""" + return True + + assert NoEq(1) != NoEq(1) + assert OwnEq(1) == "anything" + + +@pytest.mark.parametrize( + "other", [PairReordered(2, 1), PairRenamed(1, 2), OrdinaryPair(1, 2)] +) +def test_pod_refuses_another_type(other: Any) -> None: + """A pod annotated with a structural class refuses a column of another type.""" + + @function_pod(output_keys="seen") + def look(p: PairA) -> str: + """Return the class name of the argument.""" + return type(p).__name__ + + with pytest.raises(ValueError, match="column 'p'"): + _ = look.pod(_source(other)) + + +# --------------------------------------------------------------------------- +# Conversion +# --------------------------------------------------------------------------- + + +@op.dataclass(name="itl972.Segment") +class Segment: + """A class with structural fields, alone, in a list and optional.""" + + start: PairA + stops: list[PairA] + end: PairA | None = None + + +@op.dataclass(name="itl972.Segment") +class SegmentB: + """Another class of the same type, built over ``PairB``.""" + + start: PairB + stops: list[PairB] + end: PairB | None = None + + +def test_from_instance_of_an_ordinary_dataclass() -> None: + """``from_instance`` builds the class from an ordinary dataclass instance.""" + built = _convert(PairA, OrdinaryPair(1, 2)) + assert type(built) is PairA + assert built == PairA(1, 2) + + +def test_from_instance_of_another_structural_class() -> None: + """``from_instance`` builds the class from another structural class's instance.""" + built = _convert(PairA, PairReordered(y=2, x=1)) + assert type(built) is PairA + assert built == PairA(1, 2) + + +def test_from_instance_converts_nested_fields() -> None: + """Fields typed by a structural class are converted, also in lists and optionals.""" + source = SegmentB(PairB(1, 2), [PairB(3, 4)], PairB(5, 6)) + built = _convert(Segment, source) + assert type(built.start) is PairA + assert [type(stop) for stop in built.stops] == [PairA] + assert type(built.end) is PairA + assert built == Segment(PairA(1, 2), [PairA(3, 4)], PairA(5, 6)) + assert _convert(Segment, SegmentB(PairB(1, 2), [])).end is None + + +@op.dataclass(name="itl972.Named") +class Named: + """A structural class with one text field.""" + + label: str + + +@op.dataclass(name="itl972.Either") +class Either: + """A structural class with a union of two structural classes.""" + + value: PairA | Named + + +@dataclasses.dataclass +class OtherEither: + """An ordinary dataclass with the same field as ``Either``.""" + + value: PairA | Named + + +@dataclasses.dataclass +class OrdinaryNamed: + """An ordinary dataclass with the field of ``Named``.""" + + label: str + + +def test_from_instance_keeps_the_union_member_a_value_already_is() -> None: + """A union field keeps a value of one member and converts to the member that fits.""" + kept = _convert(Either, OtherEither(Named("n"))) + assert type(kept.value) is Named + converted = _convert(Either, OtherEither(OrdinaryNamed("n"))) # pyright: ignore[reportArgumentType] # dataclasses do not check field types + assert type(converted.value) is Named + assert converted.value == Named("n") + + +@op.dataclass(name="itl972.IntRecord") +class IntRecord: + """A structural class whose ``x`` is an ``int``.""" + + x: int + + +@op.dataclass(name="itl972.TextRecord") +class TextRecord: + """A structural class whose ``x`` is a ``str``: same field name, other type.""" + + x: str + + +@op.dataclass(name="itl972.Records") +class Records: + """A structural class with unions of the two record classes.""" + + one: IntRecord | TextRecord + many: list[IntRecord] | list[TextRecord] + + +@dataclasses.dataclass +class OrdinaryText: + """An ordinary dataclass with the field of ``TextRecord``.""" + + x: str + + +@dataclasses.dataclass +class OrdinaryRecords: + """An ordinary dataclass with the field names of ``Records``.""" + + one: IntRecord | TextRecord + many: list[IntRecord] | list[TextRecord] + + +def test_from_instance_chooses_the_union_member_by_field_types() -> None: + """A union member is chosen by declared field types, not by field names alone.""" + source: Any = OrdinaryRecords(OrdinaryText("a"), [OrdinaryText("b")]) # pyright: ignore[reportArgumentType] # dataclasses do not check field types + built = _convert(Records, source) + assert built.one == TextRecord("a") + assert type(built.one) is TextRecord + assert built.many == [TextRecord("b")] + assert [type(item) for item in built.many] == [TextRecord] + + +@dataclasses.dataclass +class OrdinaryFlag: + """An ordinary dataclass that fits neither record class.""" + + flag: bool + + +def test_from_instance_refuses_a_union_value_no_member_can_take() -> None: + """A value that fits no member of a union is an error, never left unconverted.""" + alone: Any = OrdinaryRecords(OrdinaryFlag(True), []) # pyright: ignore[reportArgumentType] # dataclasses do not check field types + with pytest.raises(TypeError, match="OrdinaryFlag"): + _ = _convert(Records, alone) + in_list: Any = OrdinaryRecords(TextRecord("a"), [OrdinaryFlag(True)]) # pyright: ignore[reportArgumentType] # dataclasses do not check field types + with pytest.raises(TypeError, match="list"): + _ = _convert(Records, in_list) + + +def test_from_instance_leaves_a_union_value_no_member_claims() -> None: + """A value of a non-class member of a union is passed as it is.""" + + @dataclasses.dataclass + class Source: + value: int | IntRecord + + converted = _convert(IntOrRecord, Source(3)) + assert converted.value == 3 + + +@op.dataclass(name="itl972.IntOrRecord") +class IntOrRecord: + """A structural class whose field is an ``int`` or a structural class.""" + + value: int | IntRecord + + +def test_from_instance_refuses_other_fields() -> None: + """A missing or an extra field is an error naming the fields.""" + + @dataclasses.dataclass + class Wider: + x: int + y: int + z: int + + @dataclasses.dataclass + class Narrower: + x: int + + with pytest.raises(TypeError, match="z"): + _ = _convert(PairA, Wider(1, 2, 3)) + with pytest.raises(TypeError, match="y"): + _ = _convert(PairA, Narrower(1)) + + +def test_from_instance_refuses_another_field_type() -> None: + """A field declared with another type is an error naming the field.""" + + @dataclasses.dataclass + class Texts: + x: str + y: int + + with pytest.raises(TypeError, match="'x'"): + _ = _convert(PairA, Texts("1", 2)) + + +def test_from_instance_refuses_what_is_not_a_dataclass_instance() -> None: + """A pydantic model, a dict or a class is refused.""" + + class Model(pydantic.BaseModel): + x: int + y: int + + for other in (Model(x=1, y=2), {"x": 1, "y": 2}, OrdinaryPair): + with pytest.raises(TypeError, match="dataclass"): + _ = _convert(PairA, other) + + +def test_constructor_is_the_plain_field_constructor() -> None: + """``P(other)`` does not convert: the constructor takes field values only.""" + built = PairA(OrdinaryPair(1, 2), 3) # pyright: ignore[reportArgumentType] # dataclasses do not check field types + assert built.x == OrdinaryPair(1, 2) + with pytest.raises(TypeError): + _ = PairA(OrdinaryPair(1, 2)) # pyright: ignore[reportCallIssue, reportArgumentType] # y is missing + + +def test_conversion_runs_post_init() -> None: + """The instance is built by the class's own ``__init__``.""" + + @op.dataclass(name="itl972.Checked") + class Checked: + x: int + + def __post_init__(self) -> None: + """Refuse negative values.""" + if self.x < 0: + raise ValueError("x must not be negative") + + @dataclasses.dataclass + class Raw: + x: int + + assert _convert(Checked, Raw(1)) == Checked(1) + with pytest.raises(ValueError, match="negative"): + _ = _convert(Checked, Raw(-1)) + + +def test_pod_parameter_in_a_container_is_compatible_and_passed_as_is() -> None: + """A list and an optional of another class of the same type are accepted as they are.""" + + @function_pod(output_keys="seen") + def look(ps: list[PairB], maybe: PairB | None) -> str: + """Return the sum of all ``x`` fields.""" + return str(sum(p.x for p in [*ps, maybe] if p is not None)) + + row: dict[str, Any] = { + "k": "a", + "ps": [PairA(1, 2), PairA(3, 4)], + "maybe": PairA(5, 6), + } + source = DictSource( + data=[row], + tag_columns=["k"], + source_id="s", + ) + assert _read(look.pod(source), "seen") == "9" + + +# --------------------------------------------------------------------------- +# Reading without the class +# --------------------------------------------------------------------------- + +WRITER = """ +import os +from pathlib import Path + +import orcapod as op +from child_harness import report +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import DictSource + + +@op.dataclass(name="itl972.child.Point") +class Point: + x: int + y: float + + +@op.dataclass(name="itl972.child.Key", frozen=True) +class Key: + n: int + + +@op.dataclass(name="itl972.child.Sample") +class Sample: + label: str + where: Point + tags: list[str] + note: str | None + file: Path + history: list[Point] + keys: set[Key] + + +@function_pod(output_keys="sample") +def make(x: int) -> Sample: + return Sample(f"s{x}", Point(x, x / 2), ["a", "b"], None, Path("/tmp/f"), [Point(0, 0.0)], {Key(1), Key(2)}) + + +job = PipelineJob(name="w", store=DeltaTableDatabase(Path(os.environ["WS_DB"]))) +with job: + make.pod(DictSource(data=[{"k": "a", "x": 1}], tag_columns=["k"], source_id="s"), label="make") +job.run() +report({"rows": job.nodes["make"].as_table().num_rows}) +""" + +READER = """ +import dataclasses +import os +from pathlib import Path + +import wsjob +from child_harness import report +from orcapod.contexts import get_default_context +from orcapod.databases import DeltaTableDatabase, ExtensionAwareDatabase + +root = Path(os.environ["WS_DB"]) +path = tuple(next(p for p in wsjob.table_paths(root) if "/_result/" in p).split("/")) +converter = get_default_context().type_converter +table = ExtensionAwareDatabase(DeltaTableDatabase(root), converter=converter).get_all_records(path) +out = [] +for row in converter.arrow_table_to_python_dicts(table.select(["sample"])): + value = row["sample"] + out.append({ + "class": type(value).__name__, + "has_from_instance": hasattr(type(value), "from_instance"), + "module": type(value).__module__, + "is_dataclass": dataclasses.is_dataclass(value), + "fields": [f.name for f in dataclasses.fields(value)], + "where_class": type(value.where).__name__, + "where": dataclasses.asdict(value.where), + "tags": value.tags, + "note": value.note, + "file_is_path": isinstance(value.file, Path), + "history": [dataclasses.asdict(p) for p in value.history], + "keys_is_set": isinstance(value.keys, set), + "keys": sorted(key.n for key in value.keys), + }) +report(out) +""" + + +@pytest.mark.integration +def test_stored_values_are_read_without_the_class(tmp_path: Path) -> None: + """A process that does not define the class reads a class built from the metadata.""" + write_files(tmp_path, {"writer.py": WRITER, "reader.py": READER, "wsjob.py": WSJOB}) + (tmp_path / "db").mkdir() + env = {"WS_DB": str(tmp_path / "db")} + assert run_child(["writer.py"], cwd=tmp_path, env=env) == {"rows": 1} + + read = run_child(["reader.py"], cwd=tmp_path, env=env) + assert read == [ + { + "class": "Sample", + "has_from_instance": True, + "module": "orcapod.synthesized", + "is_dataclass": True, + "fields": ["label", "where", "tags", "note", "file", "history", "keys"], + "where_class": "Point", + "where": {"x": 1, "y": 0.5}, + "tags": ["a", "b"], + "note": None, + "file_is_path": True, + "history": [{"x": 0, "y": 0.0}], + "keys_is_set": True, + "keys": [1, 2], + } + ] + + +def test_synthesized_class_equals_and_hashes_like_a_defined_class() -> None: + """A class built from metadata and a defined class of the same type agree. + + Their instances with equal fields are equal in both directions and, both being + hashable, have equal hashes. + """ + from orcapod.contexts import get_default_context + from orcapod.logical_types.structural_class import synthesize_structural_class + + @op.dataclass(name="itl972.Synth", frozen=True) + class Defined: + x: int + label: str + + record = { + "kind": "structural", + "name": "itl972.Synth", + "fields": [["x", "int"], ["label", "str"]], + } + # Non-public entry point: with ``Defined`` in the process a read would use it, so + # the class is built from the record directly, as a read without the class does. + converter: Any = get_default_context().type_converter + synthesized = synthesize_structural_class(record, converter) + assert synthesized is not Defined + assert synthesized.__module__ == "orcapod.synthesized" + + built, defined = synthesized(1, "a"), Defined(1, "a") + assert built == defined + assert defined == built + assert hash(built) == hash(defined) + assert built != Defined(2, "a") + assert len({built, defined}) == 1 diff --git a/tests/test_user_types/test_workshop_scenarios.py b/tests/test_user_types/test_workshop_scenarios.py new file mode 100644 index 000000000..d7226731e --- /dev/null +++ b/tests/test_user_types/test_workshop_scenarios.py @@ -0,0 +1,592 @@ +"""Workshop scenarios W1–W12 for user class identity (ITL-881). + +Each scenario runs notebook cells (IPython ``InteractiveShell``, as a Jupyter kernel runs +them) or scripts in fresh processes, against one ``DeltaTableDatabase``, and drives the +pipeline only through the public API (``DictSource``, ``function_pod``, ``PipelineJob``, +``ExtensionAwareDatabase``). Scenario numbers follow the findings in PR #288. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.test_user_types.child_harness import WSJOB, run_child, write_files + +pytestmark = pytest.mark.integration + +KERNEL = r""" +import warnings +from IPython.core.interactiveshell import InteractiveShell +from child_harness import error_text, report + +shell = InteractiveShell.instance() +CELLS = { + "setup": "import wsjob\nfrom dataclasses import dataclass\nimport orcapod as op\n", + "define": "@dataclass\nclass Point:\n x: int\n y: int\n", + "define_reordered": "@dataclass\nclass Point:\n y: int\n x: int\n", + "define_added": "@dataclass\nclass Point:\n x: int\n y: int\n z: int = 0\n", + "define_model_reordered": "import pydantic\nclass Point(pydantic.BaseModel):\n y: int\n x: int\n", + "define_s": "@op.dataclass\nclass Point:\n x: int\n y: int\n", + "define_s_float": "@op.dataclass\nclass Point:\n x: int\n y: float\n", + "define_float": "@dataclass\nclass Point:\n x: int\n y: float\n", + "define_model": "import pydantic\nclass Point(pydantic.BaseModel):\n x: int\n y: int\n", + "import_mod": "from wsmodels import Point\n", + "reload_mod": "import importlib, wsmodels\nimportlib.reload(wsmodels)\nfrom wsmodels import Point\n", + "pod": "shift = wsjob.make_pod(Point)\n", + "run": "_result = wsjob.run_job(Point)\n", + "run_pod": "_result = wsjob.run_job(Point, shift)\n", +} +log = [] +for step in SCENARIO: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + res = shell.run_cell(CELLS[step], store_history=False, silent=True) + entry = { + "cell": step, + "warnings": [str(w.message) for w in caught if issubclass(w.category, UserWarning)], + } + err = res.error_in_exec or res.error_before_exec + if err is not None: + entry["error"] = error_text(err) + elif step.startswith("run"): + entry.update(shell.user_ns["_result"]) + log.append(entry) +report(log) +""" + +WSMODELS = """ +from dataclasses import dataclass + + +@dataclass +class Point: + '''The scenario's ``Point`` class.''' + x: int + y: int +""" + + +@pytest.fixture +def workdir(tmp_path: Path) -> Path: + """Return a working directory with the job modules and an empty ``db`` directory.""" + write_files(tmp_path, {"wsjob.py": WSJOB, "wsmodels.py": WSMODELS}) + (tmp_path / "db").mkdir() + return tmp_path + + +def _kernel(workdir: Path, cells: list[str]) -> list[dict[str, Any]]: + """Run ``cells`` in a fresh kernel process and return one log entry per cell.""" + code = f"SCENARIO = {cells!r}\n" + KERNEL + return run_child(["-c", code], cwd=workdir, env={"WS_DB": str(workdir / "db")}) + + +def _runs(log: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Assert that no cell failed and return the log entries of the run cells.""" + errors = [e for e in log if "error" in e] + assert not errors, errors + return [e for e in log if e["cell"].startswith("run")] + + +def _script(workdir: Path, code: str, **env: str) -> Any: + """Run ``code`` in a fresh process against the working store; return its report.""" + return run_child( + ["-c", code], cwd=workdir, env={"WS_DB": str(workdir / "db"), **env} + ) + + +# --------------------------------------------------------------------------- +# Subclasses, whatever the registration order (audit ITL-942, finding R) +# --------------------------------------------------------------------------- + +WSFAMILY = """ +from dataclasses import dataclass + +import pydantic + + +@dataclass +class Base: + '''The scenario's ``Base`` class.''' + x: int + + +@dataclass +class Child(Base): + '''The scenario's ``Child`` class.''' + z: int = 0 + + +class BaseModel(pydantic.BaseModel): + '''The scenario's ``BaseModel`` class.''' + x: int + + +class ChildModel(BaseModel): + '''The scenario's ``ChildModel`` class.''' + z: int = 0 +""" + +FAMILY_SCRIPT = """ +import os + +import wsfamily +import wsjob +from child_harness import report +from orcapod.sources import DictSource + +base = getattr(wsfamily, os.environ["WS_BASE"]) +child = getattr(wsfamily, os.environ["WS_CHILD"]) + + +def use_base(): + '''Return the Arrow extension name the base class is stored under.''' + source = DictSource(data=[{"k": "a", "p": base(x=1)}], tag_columns=["k"]) + return source.as_table().schema.field("p").type.extension_name + + +def use_child(): + '''Run the job on the child class and return its result.''' + return wsjob.run_job(child, rows=[{"x": 1, "z": 10}, {"x": 2, "z": 20}]) + + +out = {} +if os.environ["WS_ORDER"] == "base_first": + out["base_name"] = use_base() + out["child"] = use_child() +else: + out["child"] = use_child() + out["base_name"] = use_base() +report(out) +""" + + +@pytest.mark.parametrize( + ("base", "child"), + [("Base", "Child"), ("BaseModel", "ChildModel")], + ids=["dataclass", "pydantic"], +) +def test_subclass_identity_and_fields_do_not_depend_on_registration_order( + workdir: Path, base: str, child: str +) -> None: + """A subclass keeps its own identity and all its fields in both orders. + + Each order runs in a fresh process against a fresh store. On ``main`` the subclass + was stored as its base when the base was registered first: its extra field was + dropped, and the pod's hash and stored path depended on the order. + """ + write_files(workdir, {"wsfamily.py": WSFAMILY}) + out: dict[str, Any] = {} + for order in ("base_first", "sub_first"): + (workdir / order).mkdir() + out[order] = run_child( + ["-c", FAMILY_SCRIPT], + cwd=workdir, + env={ + "WS_DB": str(workdir / order), + "WS_ORDER": order, + "WS_BASE": base, + "WS_CHILD": child, + }, + ) + + for order, result in out.items(): + run = result["child"] + assert (run["calls"], run["rows"]) == (2, 2), order + assert run["q_class_is_point"], order + assert run["extension_name"].split("@")[0] == f"wsfamily.{child}", order + assert run["extension_name"] != result["base_name"], order + assert run["stored_fields"] == [["x", "z"]], order + assert run["values"] == [f"{child}(x=1, z=10)", f"{child}(x=2, z=20)"], order + + first, second = out["base_first"], out["sub_first"] + assert first["base_name"] == second["base_name"] + for key in ( + "extension_name", + "pod_pipeline_hash", + "pod_uri", + "node_identity_path", + "table_paths", + "stored_fields", + "values", + ): + assert first["child"][key] == second["child"][key], key + + +# --------------------------------------------------------------------------- +# Notebook cell re-runs in one kernel +# --------------------------------------------------------------------------- + + +def test_w1_rerun_define_cell_same_fields_rebinds(workdir: Path) -> None: + """Re-running the class cell unchanged reuses the stored results.""" + log = _kernel(workdir, ["setup", "define", "run", "define", "run"]) + first, second = _runs(log) + assert (first["calls"], first["rows"]) == (2, 2) + assert (second["calls"], second["rows"]) == (0, 2) + assert second["q_class_is_point"] + assert first["pod_pipeline_hash"] == second["pod_pipeline_hash"] + assert first["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] + assert second["values"] == first["values"] + + +def test_w1_old_pod_cell_refuses_a_redefined_ordinary_class(workdir: Path) -> None: + """After re-running only the class cell, the earlier pod refuses the new class.""" + log = _kernel(workdir, ["setup", "define", "pod", "run_pod", "define", "run_pod"]) + assert [e["cell"] for e in log if "error" in e] == ["run_pod"] + assert "error" not in log[3] + assert log[-1]["error"].startswith("ValueError") + assert "column 'p'" in log[-1]["error"] + + +def test_w1_old_pod_cell_with_redefined_structural_class(workdir: Path) -> None: + """Re-running only an ``op.dataclass`` cell keeps the earlier pod cell usable.""" + log = _kernel( + workdir, ["setup", "define_s", "pod", "run_pod", "define_s", "run_pod"] + ) + first, second = _runs(log) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + assert first["extension_name"].split("@")[0] == "Point" + + +def test_w2_structural_rerun_with_changed_fields_is_a_new_type(workdir: Path) -> None: + """An ``op.dataclass`` re-run with a changed field type is a new type.""" + log = _kernel(workdir, ["setup", "define_s", "run", "define_s_float", "run"]) + first, second = _runs(log) + assert (second["calls"], second["rows"]) == (2, 2) + assert first["extension_name"] != second["extension_name"] + assert set(first["table_paths"]) < set(second["table_paths"]) + + +def test_w1_run_all_twice(workdir: Path) -> None: + """Running all cells a second time reuses the stored results.""" + log = _kernel(workdir, ["setup", "define", "pod", "run_pod"] * 2) + first, second = _runs(log) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + + +@pytest.mark.parametrize( + ("define", "reordered"), + [("define", "define_reordered"), ("define_model", "define_model_reordered")], + ids=["dataclass", "pydantic"], +) +def test_reordered_fields_are_a_new_type( + workdir: Path, define: str, reordered: str +) -> None: + """Declared field order is identity: the reordered class computes into new tables.""" + log = _kernel(workdir, ["setup", define, "run", reordered, "run", define, "run"]) + first, second, third = _runs(log) + assert (second["calls"], second["rows"]) == (2, 2) + assert second["q_class_is_point"] + assert first["extension_name"] != second["extension_name"] + assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] + assert first["pod_uri"] != second["pod_uri"] + assert set(first["table_paths"]) < set(second["table_paths"]) + assert first["stored_fields"] == [["x", "y"]] + assert second["stored_fields"] == [["x", "y"], ["y", "x"]] + assert first["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] + assert second["values"] == ["Point(y=10, x=11)", "Point(y=20, x=21)"] + assert third["values"] == first["values"] + # The first order's results are untouched and found again. + assert (third["calls"], third["rows"]) == (0, 2) + assert third["pod_pipeline_hash"] == first["pod_pipeline_hash"] + + +def test_class_redefined_with_an_added_field_is_a_new_type(workdir: Path) -> None: + """An added field (even with a default) makes a new type with its own tables.""" + log = _kernel( + workdir, ["setup", "define", "run", "define_added", "run", "define", "run"] + ) + first, second, third = _runs(log) + assert (second["calls"], second["rows"]) == (2, 2) + assert second["q_class_is_point"] + assert first["extension_name"] != second["extension_name"] + assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] + assert set(first["table_paths"]) < set(second["table_paths"]) + assert second["stored_fields"] == [["x", "y"], ["x", "y", "z"]] + assert second["values"] == ["Point(x=10, y=11, z=12)", "Point(x=20, y=21, z=22)"] + # The two-field results are untouched and found again. + assert (third["calls"], third["rows"]) == (0, 2) + assert third["values"] == first["values"] + + +def test_w2_rerun_with_changed_fields_is_a_new_type(workdir: Path) -> None: + """Re-running with a changed field type gives a new type with its own tables.""" + log = _kernel(workdir, ["setup", "define", "run", "define_float", "run"]) + first, second = _runs(log) + assert (second["calls"], second["rows"]) == (2, 2) + assert second["q_class_is_point"] + assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] + assert first["extension_name"].split("@")[0] == "__main__.Point" + assert second["extension_name"].split("@")[0] == "__main__.Point" + assert first["extension_name"] != second["extension_name"] + assert set(first["table_paths"]) < set(second["table_paths"]) + + +def test_w3_pydantic_rerun_rebinds(workdir: Path) -> None: + """Re-running a pydantic class cell unchanged reuses the stored results.""" + log = _kernel(workdir, ["setup", "define_model", "run", "define_model", "run"]) + first, second = _runs(log) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + + +def test_dataclass_and_pydantic_with_same_name_and_fields_are_distinct( + workdir: Path, +) -> None: + """A dataclass and a pydantic model with one name and fields are distinct types.""" + log = _kernel(workdir, ["setup", "define", "run", "define_model", "run"]) + first, second = _runs(log) + assert second["calls"] == 2 + assert first["pod_pipeline_hash"] != second["pod_pipeline_hash"] + assert first["extension_name"] != second["extension_name"] + + +def test_w4_module_reload_rebinds(workdir: Path) -> None: + """Reloading the class's module reuses the stored results.""" + log = _kernel(workdir, ["setup", "import_mod", "run", "reload_mod", "run"]) + first, second = _runs(log) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + assert first["extension_name"].startswith("wsmodels.Point@") + + +# --------------------------------------------------------------------------- +# Across processes +# --------------------------------------------------------------------------- + + +def test_w5_restart_same_class_hits_cache(workdir: Path) -> None: + """The same class in a restarted kernel reuses the stored results.""" + first = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + second = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + + +def test_w6_restart_with_changed_fields_recomputes(workdir: Path) -> None: + """Changed fields after a restart compute into new tables instead of 0 rows.""" + first = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + changed = _runs(_kernel(workdir, ["setup", "define_float", "run"]))[0] + back = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + assert (changed["calls"], changed["rows"]) == (2, 2) + assert first["node_identity_path"] != changed["node_identity_path"] + assert (back["calls"], back["rows"]) == (0, 2) + + +def test_w7_script_as_main_twice(workdir: Path) -> None: + """A script run as a file and then with ``-m`` reuses the stored results.""" + write_files( + workdir, + { + "pkg/__init__.py": "", + "pkg/pipeline.py": """ + from dataclasses import dataclass + import wsjob + from child_harness import report + + @dataclass + class Point: + '''The scenario's ``Point`` class.''' + x: int + y: int + + def main(): + '''Run the job on ``Point`` and report its result.''' + report(wsjob.run_job(Point)) + + if __name__ == "__main__": + main() + """, + }, + ) + env = {"WS_DB": str(workdir / "db")} + first = run_child(["pkg/pipeline.py"], cwd=workdir, env=env) + second = run_child(["-m", "pkg.pipeline"], cwd=workdir, env=env) + assert (first["calls"], second["calls"]) == (2, 0) + + +def test_w8_script_then_import_hits_with_a_structural_class(workdir: Path) -> None: + """An ``op.dataclass`` in a module run as main and then imported shares results.""" + write_files( + workdir, + { + "pkg/__init__.py": "", + "pkg/pipeline.py": """ + import orcapod as op + import wsjob + from child_harness import report + + @op.dataclass + class Point: + '''The scenario's ``Point`` class.''' + x: int + y: int + + def main(): + '''Run the job on ``Point`` and report its result.''' + report(wsjob.run_job(Point)) + + if __name__ == "__main__": + main() + """, + }, + ) + env = {"WS_DB": str(workdir / "db")} + first = run_child(["-m", "pkg.pipeline"], cwd=workdir, env=env) + second = run_child( + ["-c", "import pkg.pipeline; pkg.pipeline.main()"], cwd=workdir, env=env + ) + assert (first["calls"], second["calls"], second["rows"]) == (2, 0, 2) + assert first["node_identity_path"] == second["node_identity_path"] + + +def test_w11_notebook_to_module_with_a_structural_class(workdir: Path) -> None: + """An ``op.dataclass`` moved from a notebook into a module keeps its results.""" + write_files( + workdir, + { + "pkg/__init__.py": "", + "pkg/models.py": """ + import orcapod as op + + @op.dataclass + class Point: + '''The scenario's ``Point`` class.''' + x: int + y: int + """, + }, + ) + notebook = _runs(_kernel(workdir, ["setup", "define_s", "run"]))[0] + module = _script( + workdir, + "import wsjob\nfrom child_harness import report\nfrom pkg.models import Point\n" + "report(wsjob.run_job(Point))\n", + ) + assert (module["calls"], module["rows"]) == (0, 2) + assert module["pod_pipeline_hash"] == notebook["pod_pipeline_hash"] + assert module["node_identity_path"] == notebook["node_identity_path"] + assert module["table_paths"] == notebook["table_paths"] + assert module["extension_name"] == notebook["extension_name"] + assert module["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] + assert module["values"] == notebook["values"] + + +def test_w11_ordinary_module_class_is_a_different_type(workdir: Path) -> None: + """An ordinary class in a module is another type than the notebook's class.""" + notebook = _runs(_kernel(workdir, ["setup", "define", "run"]))[0] + module = _runs(_kernel(workdir, ["setup", "import_mod", "run"]))[0] + assert module["calls"] == 2 + assert module["pod_pipeline_hash"] != notebook["pod_pipeline_hash"] + + +# --------------------------------------------------------------------------- +# Reading stored values +# --------------------------------------------------------------------------- + +READER = """ +import os +from pathlib import Path +from child_harness import error_text, report +from orcapod.contexts import get_default_context +from orcapod.databases import DeltaTableDatabase, ExtensionAwareDatabase +import wsjob + +{define} + +root = Path(os.environ["WS_DB"]) +path = tuple(next(p for p in wsjob.table_paths(root) if "/_result/" in p).split("/")) +converter = get_default_context().type_converter +try: + table = ExtensionAwareDatabase(DeltaTableDatabase(root), converter=converter).get_all_records(path) + rows = converter.arrow_table_to_python_dicts(table.select(["q"])) + report({{"values": sorted(repr(r["q"]) for r in rows), "classes": sorted({{type(r["q"]).__module__ + "." + type(r["q"]).__qualname__ for r in rows}})}}) +except Exception as exc: + report({{"error": error_text(exc), "message": str(exc)}}) +""" + + +def _store_from_notebook(workdir: Path, define: str = "define") -> None: + """Store results for the notebook ``Point`` class in the working store.""" + _runs(_kernel(workdir, ["setup", define, "run"])) + + +def test_w12_read_notebook_results_from_script_with_same_class(workdir: Path) -> None: + """A script defining the same class reads the notebook's stored values.""" + _store_from_notebook(workdir) + define = "from dataclasses import dataclass\n@dataclass\nclass Point:\n x: int\n y: int\n" + out = _script(workdir, READER.format(define=define)) + assert out.get("error") is None, out + assert out["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] + assert out["classes"] == ["__main__.Point"] + + +def test_read_with_a_structural_class_from_a_module(workdir: Path) -> None: + """A structural class of the stored name and fields in a module reads the values.""" + _store_from_notebook(workdir, "define_s") + write_files( + workdir, + { + "geo.py": """ + import orcapod as op + + @op.dataclass + class Point: + '''The scenario's ``Point`` class.''' + x: int + y: int + """ + }, + ) + out = _script(workdir, READER.format(define="import geo")) + assert out.get("error") is None, out + assert out["classes"] == ["geo.Point"] + + +def test_read_without_an_ordinary_class_names_it(workdir: Path) -> None: + """A read without the ordinary class raises, naming the column, class and fields.""" + _store_from_notebook(workdir) + out = _script(workdir, READER.format(define="")) + assert out["error"].startswith("ImportError: Column 'q'") + assert "'__main__.Point'" in out["message"] + assert "{x: int, y: int}" in out["message"] + + +def test_read_without_a_structural_class_builds_it(workdir: Path) -> None: + """A read without the ``op.dataclass`` gives values of a class built from metadata.""" + _store_from_notebook(workdir, "define_s") + out = _script(workdir, READER.format(define="")) + assert out.get("error") is None, out + assert out["values"] == ["Point(x=10, y=11)", "Point(x=20, y=21)"] + assert out["classes"] == ["orcapod.synthesized.Point"] + + +def test_ordinary_module_class_does_not_read_notebook_results(workdir: Path) -> None: + """An ordinary class with the same fields at another module path is not the type.""" + _store_from_notebook(workdir) + write_files( + workdir, + { + "geo.py": """ + from dataclasses import dataclass + + @dataclass + class Point: + '''The scenario's ``Point`` class.''' + x: int + y: int + """ + }, + ) + out = _script(workdir, READER.format(define="import geo")) + assert out["error"].startswith("ImportError: Column 'q'"), out + + +def test_read_with_different_fields_raises_naming_both_shapes(workdir: Path) -> None: + """A read with a class of other fields raises, naming both shapes.""" + _store_from_notebook(workdir) + define = "from dataclasses import dataclass\n@dataclass\nclass Point:\n x: int\n y: float\n" + out = _script(workdir, READER.format(define=define)) + assert out["error"].startswith("TypeIdentityError: Column 'q'"), out + assert "{x: int, y: int}" in out["message"] + assert "{x: int, y: float}" in out["message"]