diff --git a/.zed/rules b/.zed/rules index d53a114e0..9feb3a851 100644 --- a/.zed/rules +++ b/.zed/rules @@ -496,6 +496,20 @@ Orcapod stores or embeds in a name is pinned in the golden folder of the current generator writes the new value. `--release 0.4.0` freezes v1. Any version published before it, a pre-release included, would freeze v1 there: the Release workflow requires the release step for every version. A withdrawn version is never released again. +- Provisional operators are outside the freeze (developer, November 18, 2026, ITL-1266): + `orcapod.operators.PROVISIONAL_OPERATORS` (`core/operators/provisional.py`) is the one + list (name to the issue deciding its identity; today `PolarsFilter`), and each listed class + sets `PROVISIONAL = True` (a test checks they agree). Building one issues + `ProvisionalHashWarning` once per operator per process. A golden case that depends on one + declares it (`@case(..., provisional=("PolarsFilter",))`; `compute_group_kinds` refuses a + case that builds an undeclared one); its entries live in + `data//provisional/.json`, listed in `golden.provisional` of the manifest, + outside the history, so no release freezes them. They are checked like every other value + and change only with the developer's approval of the exact list, without a new data + context. `generate_golden_hashes.py --relocate` moves entries, values unchanged, between + frozen and provisional files; an operator leaving the group has its entries relocated into + a frozen file before the release step. Never put a value that depends on a provisional + operator in a frozen file. - The unit of immutability is the golden file. The manifest's `golden.history` (`src/orcapod/contexts/data/.json`) lists every file with the sha256 of its bytes; a release appends a revision with `added: {}`, pinned in diff --git a/CLAUDE.md b/CLAUDE.md index 77cdf90cb..8d58e8862 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -507,6 +507,20 @@ Orcapod stores or embeds in a name is pinned in the golden folder of the current generator writes the new value. `--release 0.4.0` freezes v1. Any version published before it, a pre-release included, would freeze v1 there: the Release workflow requires the release step for every version. A withdrawn version is never released again. +- Provisional operators are outside the freeze (developer, November 18, 2026, ITL-1266): + `orcapod.operators.PROVISIONAL_OPERATORS` (`core/operators/provisional.py`) is the one + list (name to the issue deciding its identity; today `PolarsFilter`), and each listed class + sets `PROVISIONAL = True` (a test checks they agree). Building one issues + `ProvisionalHashWarning` once per operator per process. A golden case that depends on one + declares it (`@case(..., provisional=("PolarsFilter",))`; `compute_group_kinds` refuses a + case that builds an undeclared one); its entries live in + `data//provisional/.json`, listed in `golden.provisional` of the manifest, + outside the history, so no release freezes them. They are checked like every other value + and change only with the developer's approval of the exact list, without a new data + context. `generate_golden_hashes.py --relocate` moves entries, values unchanged, between + frozen and provisional files; an operator leaving the group has its entries relocated into + a frozen file before the release step. Never put a value that depends on a provisional + operator in a frozen file. - The unit of immutability is the golden file. The manifest's `golden.history` (`src/orcapod/contexts/data/.json`) lists every file with the sha256 of its bytes; a release appends a revision with `added: {}`, pinned in diff --git a/RELEASING.md b/RELEASING.md index 1ab072e86..50189037a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -127,6 +127,13 @@ against are in `docs/reference/golden-hashes.md`. step of `0.4.0` freezes v1. Every published version needs the release step, a pre-release included (the gate below refuses to tag one without it), so **publishing `0.4.0rc1` would freeze v1 at the release candidate**: plan the release around it. +- Provisional operators (`orcapod.operators.PROVISIONAL_OPERATORS`, ITL-1266) are outside + the freeze: the golden values of the cases that depend on one are in + `tests/test_hashing/golden/data/v1/provisional/`, which no release step lists or pins. An + operator that stops being provisional in a release leaves the list (with its `PROVISIONAL` + marker and its golden cases' declarations) before the release step, and + `generate_golden_hashes.py --relocate` moves its golden values, unchanged, into a frozen + file, so that the release step freezes them. - Release tags are protected on `metamorphic-labs/orcapod-python` (and on `nauticalab/orcapod-python`, the release repository up to `0.3.0`): the repository ruleset `protect-release-tags` (target `refs/tags/v*`; rules deletion, update and non-fast-forward; diff --git a/changelog.d/ITL-1266.md b/changelog.d/ITL-1266.md new file mode 100644 index 000000000..5dc5532ef --- /dev/null +++ b/changelog.d/ITL-1266.md @@ -0,0 +1,11 @@ +### Added + +- Provisional operators: `orcapod.operators.PROVISIONAL_OPERATORS` lists the operators whose + identity is not frozen with data context v1 (ITL-1266). Today it holds only `PolarsFilter` + (its identity is decided under ITL-1247). Building a provisional operator, directly or + through a stream method such as `polars_filter`, issues `ProvisionalHashWarning` once per + operator per process. A later release may change its identity, and with it the stored + paths and record ids of its results and of every step downstream of it; those results are + then computed again (nothing stored is lost or wrong). The hash stability that data context + v1 promises from 0.4.0 holds for a pipeline only when it contains no provisional operator. + See "Provisional operators" in `docs/concepts/operators.md`. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 4b73ea29c..cfebf1846 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -485,6 +485,11 @@ A new name already taken by a column that is not renamed is refused too. ### PolarsFilter +!!! warning "Provisional" + `PolarsFilter` is a [provisional operator](#provisional-operators): its identity is not + frozen with data context v1 and may change in a later release (ITL-1247). Building one + issues `ProvisionalHashWarning` once per process. + Filter rows using Polars expressions: ```python @@ -749,6 +754,44 @@ large variants as described in [Arrow types of operator outputs](#arrow-types-of that is `None` raises `ValueError` naming the operator, the column and the row's tag, whatever `fail_on_miss` is. +## Provisional operators + +Data context v1 freezes the hashes of Orcapod at the 0.4.0 release: from then on a pipeline +keeps its stored paths, record ids and default source ids from one release to the next. A +provisional operator is outside that freeze. Its identity is still being decided, so a later +release may change it, and with it the content and pipeline hashes, stored paths and record +ids of its results and of every step downstream of it. A pipeline that contains one keeps its +hashes across releases only until such a change: after it, those steps compute their results +again instead of reusing them. Nothing stored is lost or wrong; the earlier rows stay where +they are. + +The provisional operators are listed in one place: + +```python +from orcapod.operators import PROVISIONAL_OPERATORS + +print(sorted(PROVISIONAL_OPERATORS)) +# ['PolarsFilter'] +``` + +Each name maps to the issue that decides the operator's identity (`PolarsFilter`: whether +the order of its predicates belongs to it). Building a provisional operator, directly or +through a stream method such as `polars_filter`, issues `orcapod.errors.ProvisionalHashWarning` +once per operator per process. To silence it once you have taken note, filter its class after +importing Orcapod: + +```python +import warnings + +from orcapod.errors import ProvisionalHashWarning + +warnings.filterwarnings("ignore", category=ProvisionalHashWarning) +``` + +An operator leaves the group at a release: from that release on its identity is frozen like +every other operator's. Its golden values then move into the frozen golden files (see +[Golden hashes](../reference/golden-hashes.md#provisional-files)). + ## Arrow types of operator outputs `output_schema()` reports Python types, but every column also has an Arrow type, and diff --git a/docs/dev/scripts/itl_1266_moved_entries.py b/docs/dev/scripts/itl_1266_moved_entries.py new file mode 100644 index 000000000..7fa58d479 --- /dev/null +++ b/docs/dev/scripts/itl_1266_moved_entries.py @@ -0,0 +1,78 @@ +"""Prove that ITL-1266 only moved golden entries between files, values unchanged. + +Usage:: + + uv run python docs/dev/scripts/itl_1266_moved_entries.py + +Reads every golden file of data context v1 (frozen and provisional) at the base revision +(git) and in the working tree, and compares: the same entry names, the same values, and +the list of entries whose file changed. Exits 1 on any other difference. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +FOLDER = "tests/test_hashing/golden/data/v1" + + +def _at(revision: str) -> dict[str, tuple[str, object]]: + names = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", revision, FOLDER], + check=True, + capture_output=True, + text=True, + ).stdout.split() + out: dict[str, tuple[str, object]] = {} + for name in names: + text = subprocess.run( + ["git", "show", f"{revision}:{name}"], + check=True, + capture_output=True, + text=True, + ).stdout + for entry, value in json.loads(text).items(): + assert entry not in out, entry + out[entry] = (name.removeprefix(FOLDER + "/"), value) + return out + + +def _tree() -> dict[str, tuple[str, object]]: + out: dict[str, tuple[str, object]] = {} + for path in sorted(Path(FOLDER).rglob("*.json")): + for entry, value in json.loads(path.read_text()).items(): + assert entry not in out, entry + out[entry] = (path.relative_to(FOLDER).as_posix(), value) + return out + + +def main() -> int: + before, after = _at(sys.argv[1]), _tree() + failures = 0 + if set(before) != set(after): + print("entry names differ:", sorted(set(before) ^ set(after))) + failures += 1 + changed_value = sorted( + e for e in set(before) & set(after) if before[e][1] != after[e][1] + ) + if changed_value: + print("values differ:", changed_value) + failures += 1 + moved: dict[tuple[str, str], list[str]] = {} + for entry in sorted(set(before) & set(after)): + if before[entry][0] != after[entry][0]: + moved.setdefault((before[entry][0], after[entry][0]), []).append(entry) + print(f"entries before: {len(before)}, after: {len(after)}") + print(f"values changed: {len(changed_value)}") + for (source, target), entries in sorted(moved.items()): + cases = sorted({e.split(".", 1)[0] for e in entries}) + print(f"moved {len(entries)}: {source} -> {target}; cases {cases}") + print(f"moved in all: {sum(len(e) for e in moved.values())}") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/reference/golden-hashes.md b/docs/reference/golden-hashes.md index ec5994b69..f158e5fcb 100644 --- a/docs/reference/golden-hashes.md +++ b/docs/reference/golden-hashes.md @@ -24,6 +24,16 @@ wanted on purpose starts a new data context version. It never edits a released v entries that move, each explained. 0.4.0 freezes `v1`. See [Withdrawing a release](#withdrawing-a-release). +!!! note "Provisional operators are outside the freeze" + A provisional operator (`orcapod.operators.PROVISIONAL_OPERATORS`, today only + `PolarsFilter`) has an identity that no release freezes yet (ITL-1266). The golden values + of every case that depends on one, its own hashes and those of every step downstream of + it, are in the provisional files, `data/v1/provisional/.json`. They are computed + and checked on every run like every other value, but no release lists them: they can + change in a later release without a new data context, with the developer's approval of + the exact list. So the promise on this page holds for a pipeline only when it contains + no provisional operator. See [Provisional files](#provisional-files). + ## The pieces at a glance The mechanism has five layers. Each one is checked against the layer below it, and the last @@ -68,12 +78,16 @@ tests/test_hashing/golden/data/ semantic-01.json torch-01.json semantic-02.json # added after 0.4.0: new semantic entries only + provisional/ # never frozen: see "Provisional files" + pipeline.json + semantic.json ``` - A file is named after the group of cases it pins plus a two-digit counter: `-NN.json`, where the group is one of `cases.GROUPS` (`semantic`, `schema`, `arrow`, `datagram`, `pipeline`, `torch`). Names carry no date, issue or release; that - information lives in the manifest. The folder holds nothing else. + information lives in the manifest. The folder holds nothing else but the `provisional` + subfolder ([Provisional files](#provisional-files)). - A file maps entry names to values. An entry name appears in exactly one file of the folder, whatever its group. - A file listed by a released revision is never edited. New values go into the group's next @@ -99,7 +113,8 @@ tests/test_hashing/golden/data/ "previous": "d002...", "digest": "d003..."}, {"date": "2026-12-09", "release": "0.4.1", "added": {}, "previous": "d003...", "digest": "d004..."} - ] + ], + "provisional": {"pipeline.json": "c86b...", "semantic.json": "2f1d..."} } ``` @@ -112,6 +127,8 @@ tests/test_hashing/golden/data/ file they list, are fixed. - `date` is the date the revision was last changed. It is informational: nothing can verify it. +- `provisional` lists the provisional files, each with the sha256 of its bytes, outside the + history: no revision lists them, so no release freezes them. ### The pins: `src/orcapod/contexts/released_manifests.json` @@ -130,6 +147,37 @@ tests/test_hashing/golden/data/ the `manifest` pin and release revision digest (`revision`) that release shipped. A withdrawn release has no `releases` line and no release revision, so it freezes nothing. +### Provisional files + +`tests/test_hashing/golden/data/v1/provisional/.json` holds the values of the cases +that depend on a provisional operator (ITL-1266), one file per group and no counter: such a +file is never frozen, and the generator rewrites it in place. + +- **Which entries.** A case declares the provisional operators it depends on: + `@case("pipeline", "polars_filter", provisional=("PolarsFilter",))`. The generator runs + every case while Orcapod records which provisional operators it builds + (`orcapod.utils.provisional.recording_provisional_uses`), and refuses a case that builds + one without declaring it, or that declares a name `PROVISIONAL_OPERATORS` does not hold. + The golden test checks that every entry is in the kind of file its case declares. So a + value that depends on a provisional operator cannot be pinned in a frozen file: neither + through a stream method that builds it (`stream.polars_filter`) nor in a step downstream + of it. +- **Today:** the `pipeline` cases `polars_filter`, `polars_filter_expr` and + `polars_filter_expr_job`, and the `semantic` case `polars_expr`: the `polars_expr_v1` + hashes, which only `PolarsFilter`'s identity feeds in the library. The 86 entries moved + out of `pipeline-01.json` and `semantic-01.json`, values unchanged, before `v1` was frozen. +- **What holds.** Every provisional value is computed and compared on every CI run, so a + change nobody intended fails. An intended change needs the developer's approval of the + exact list of entries that move, each explained (`--remove`, then `--add`), but no new + data context, before or after a release. `--release` neither lists nor pins a provisional + file, the manifest pin leaves out `golden`, and the tag check does not compare them. +- **Leaving the group.** When an operator stops being provisional, at a release, it leaves + `PROVISIONAL_OPERATORS`, its `PROVISIONAL` marker and its cases' declarations go, and + `generate_golden_hashes.py --relocate` moves its entries, values unchanged, into the open + frozen file of their group. The `--release` step then freezes them. +- **Joining the group.** Only an operator whose values no release lists can become + provisional: `--relocate` refuses to move an entry out of a file a release lists. + ### Digest definitions | Value | Computed as | @@ -161,7 +209,12 @@ writes go through `tests/test_hashing/golden/generate_golden_hashes.py`. existing value, and it writes nothing when the tree already breaks a rule. 3. The diff shows only added entries or a new file, and the edit to the open revision. 4. `--check` computes every case and reports moved, unpinned and no longer produced entries - without writing anything. + without writing anything, and entries pinned in the wrong kind of file (a provisional + case in a frozen file, or the other way round). + +An entry of a case that declares a provisional operator goes to the provisional file of its +group instead ([Provisional files](#provisional-files)). `--relocate` moves pinned entries +between the frozen and the provisional files, values unchanged, as their cases declare. Before a release, a value that no release lists can change only with the developer's approval and a written migration assessment. `--remove ` (repeatable) takes the entry @@ -188,6 +241,9 @@ edit and the generator refuses to write over it. 4. The tag and the PyPI wheel and sdist, which carry `v1.json` and `released_manifests.json`, are records of what shipped, outside the repository. +An operator that stops being provisional in this release has its entries moved into a frozen +file (`--relocate`) before the release step, so that the release freezes them. + Every published version needs the release step, a pre-release included: the release gate refuses to tag a version without it. So the first version of a data context published on PyPI freezes it; for `v1`, publishing `0.4.0rc1` would freeze `v1` at the release candidate. @@ -384,8 +440,19 @@ so the bytes that are hashed are the bytes that were committed (4). `tests/test_hashing/golden/test_golden_hashes.py` and the generator -- **Checks:** the folder holds only `-NN.json` files of known groups, and an entry name - appears in only one of them (8). +- **Checks:** the folder holds only `-NN.json` files of known groups and the + `provisional` subfolder of `.json` files, and an entry name appears in only one of + all of them (8). + +### Placement check + +`tests/test_hashing/golden/cases.py` (`compute_group_kinds`) and +`tests/test_hashing/golden/test_golden_hashes.py` + +- **Checks:** a case that builds a provisional operator declares it; every entry of a case + that declares one is in a provisional file, and every other entry in a frozen file. +- **Protects against:** a value that depends on a provisional operator frozen by a release, + and a frozen value moved out of the freeze without a declaration. ## Residual risks and assumptions @@ -398,6 +465,7 @@ so the bytes that are hashed are the bytes that were committed (4). | Values on `main` after the last release are open | An unreleased commit has no guarantee | Use a released version for anything that must stay stable | | Optional groups skip without their package | `torch` values are unchecked where torch is missing | CI installs every extra | | The `date` field cannot be verified | Informational | Order and integrity come from the chain and the tags | +| Provisional values are outside the freeze | A pipeline with a provisional operator may compute its results again after an upgrade | `ProvisionalHashWarning` when the operator is built; the list in `PROVISIONAL_OPERATORS` and on the operators page | A deliberate attacker with merge rights who also rewrites history and tags is beyond what any check inside the repository can stop; the PyPI artifacts remain the record of what shipped. @@ -405,6 +473,8 @@ check inside the repository can stop; the PyPI artifacts remain the record of wh ## Rules - A released golden file never changes, and is never removed or renamed. +- A value of a case that depends on a provisional operator is pinned in a provisional file, + never in a frozen one; no release freezes a provisional file. - `v1` is open until 0.4.0 (0.3.0 was withdrawn on 2026-11-17): until then, a change to a `v1` value needs the developer's approval of the exact list of entries that move, each explained. Never edit or regenerate a golden file to make a test pass. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 02a5059a0..e47d54b1f 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -4,7 +4,11 @@ This document is a developer-facing reference for every hash computation site in orcapod-python framework. For conceptual background on the two identity chains (`content_hash` and `pipeline_hash`), see [Identity & Hashing](../concepts/identity.md). Every hash listed here that Orcapod stores or embeds in a name is pinned as a golden value; -how released values are protected is described in [Golden Hashes](golden-hashes.md). +how released values are protected is described in [Golden Hashes](golden-hashes.md). A +release freezes every golden value except those of the provisional operators +(`orcapod.operators.PROVISIONAL_OPERATORS`, today `PolarsFilter`) and of every step downstream +of one: their identity may still change in a later release (see +[Provisional operators](../concepts/operators.md#provisional-operators)). --- @@ -575,6 +579,10 @@ The pipeline hash uses the same `SemanticAwarePythonHasher` with the **`pipeline ### 1c. Polars expression predicates +**Provisional.** `PolarsFilter` is a provisional operator (ITL-1266): its identity, and the +`polars_expr_v1` hashes that only it feeds, are not frozen with data context v1. Their golden +values are in the provisional golden files ([Golden Hashes](golden-hashes.md#provisional-files)). + `PolarsFilter.identity_structure()` is `("PolarsFilter", predicates, constraints)`, with `predicates` always a tuple. Keyword constraints hash as plain values. A `pl.Expr` predicate hashes through `PolarsExprHandler` to the leaf `polars_expr_v1:`. @@ -629,8 +637,8 @@ definition. Consequences: change to the encoding of a supported construct needs a new label (`polars_expr_v2`). The canonical trees of a corpus of expressions are pinned in -`tests/test_hashing/test_polars_expr_hashing.py`, and their hashes in the golden suite -([Golden Hashes](golden-hashes.md)). +`tests/test_hashing/test_polars_expr_hashing.py`, and their hashes in the provisional golden +files ([Golden Hashes](golden-hashes.md#provisional-files)). ### 1d. Function values and parameter defaults diff --git a/scripts/check_golden_tags.py b/scripts/check_golden_tags.py index 618c74f3c..2f5454c5c 100644 --- a/scripts/check_golden_tags.py +++ b/scripts/check_golden_tags.py @@ -20,6 +20,9 @@ shipped. So the check passes the same way whether the tag still exists or was deleted, and a record never excuses a release it does not describe. +The provisional golden files (``golden.provisional``, ITL-1266) are not compared: no +revision lists them, so no release freezes them. + The current data context is the one manifest in ``src/orcapod/contexts/data/`` at HEAD. Usage: diff --git a/src/orcapod/contexts/data/schemas/context_schema.json b/src/orcapod/contexts/data/schemas/context_schema.json index dded61b31..bf4cbd5b0 100644 --- a/src/orcapod/contexts/data/schemas/context_schema.json +++ b/src/orcapod/contexts/data/schemas/context_schema.json @@ -147,11 +147,19 @@ "additionalProperties": false }, "description": "Append-only chain of revisions. A revision lists the golden files it added, each with the sha256 of its bytes, or marks a release (added empty, release set); digest is the sha256 of the canonical JSON (sorted keys, no spaces) of date, release, added and previous" + }, + "provisional": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "The provisional golden files (ITL-1266), /provisional/.json, each with the sha256 of its bytes: the values of the cases that depend on a provisional operator. Outside the history, so no release freezes them" } }, "required": [ "folder", - "history" + "history", + "provisional" ], "additionalProperties": false, "description": "The golden hash files of this data context (ITL-946). The revisions up to the last release revision, and the files they list, never change" diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index b6053916a..e3e9f8efe 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -230,18 +230,18 @@ "folder": "v1", "history": [ { - "date": "2026-11-16", + "date": "2026-11-18", "release": null, "added": { "arrow-01.json": "c66fa55619178f2631d04232555cd58059813e97e9ecd2b9de9e63699e60c964", "datagram-01.json": "598d4a2851819d76ba674e189222e61733ca1861ca82a59e990a8ee5e1b0a7b3", - "pipeline-01.json": "ec0049a39e85645d59d9a276a17528bfa64a91f349aa0094052a6d55bc930be5", + "pipeline-01.json": "a17a0c3690cb28c7a1d3f1509e90870b92ca5e9ba3eb9c2fdc77353b1dc9f9c8", "schema-01.json": "79aa2a068e0724891cf43ee528ce8f569e65f08864faa5e34773bfc69ac9576e", - "semantic-01.json": "8e5d9dfff461f9f1d6bdd49a3019e7bf6122e75d799d725b17531d31b080003c", + "semantic-01.json": "45840a7406d232a7686e55805aabe5953594409b447ed175fdff791c49a3f947", "torch-01.json": "af2e8b68d66dffaa37e1d0ad75acd42f595d65b9fae00ddcde81bf9e13a265c8" }, "previous": null, - "digest": "0a789cf9e8d1df9fb762d6dc2fabffcd3383cd6a21ef147e8c6583f963280c2e" + "digest": "ba5b3f91710cfa607d16df622635b944b47ecd4b027573ab42a0b2b9a97d9357" }, { "date": "2026-11-18", @@ -249,10 +249,14 @@ "added": { "pipeline-02.json": "1ec3cbbe09bbb353366d1c8f2573881467d1c8e1ae53a20f8df4e7f43bbfa74e" }, - "previous": "0a789cf9e8d1df9fb762d6dc2fabffcd3383cd6a21ef147e8c6583f963280c2e", - "digest": "71b1edf7a9091477d5a39f5882f2da72e1324b3c353bbfaf755d444469c84b28" + "previous": "ba5b3f91710cfa607d16df622635b944b47ecd4b027573ab42a0b2b9a97d9357", + "digest": "b125d745d8a800c0a6c08e6e4626fe4b6b73984c82e0dc877659d1e635c4ecbf" } - ] + ], + "provisional": { + "pipeline.json": "e3825b41865f33b8ae1e2166a191be7f453b997c974e3a83eebcd389bf6349e7", + "semantic.json": "0adfab60f8ced647d841d0a74a0603096b00fab310a8f4686246309d5fb39d3d" + } }, "changes": [ { diff --git a/src/orcapod/core/operators/__init__.py b/src/orcapod/core/operators/__init__.py index acd3d5857..f0e75008e 100644 --- a/src/orcapod/core/operators/__init__.py +++ b/src/orcapod/core/operators/__init__.py @@ -16,6 +16,7 @@ from .mappers import MapColumns, MapData, MapTags from .merge_join import MergeJoin from .pick import Pick +from .provisional import PROVISIONAL_OPERATORS from .semijoin import SemiJoin from .tag_by import TagBy @@ -40,4 +41,5 @@ "Pick", "Index", "TagBy", + "PROVISIONAL_OPERATORS", ] diff --git a/src/orcapod/core/operators/filters.py b/src/orcapod/core/operators/filters.py index 16995a7fa..45d553f0a 100644 --- a/src/orcapod/core/operators/filters.py +++ b/src/orcapod/core/operators/filters.py @@ -36,6 +36,9 @@ class PolarsFilter(UnaryOperator): # A filter chooses rows and never changes a column (ITL-912). OUTPUT_SCHEMA_IS_INPUT_SCHEMA = True + # Its identity is not frozen with data context v1 (ITL-1266, ITL-1247). + PROVISIONAL = True + def __init__( self, predicates: Collection[ diff --git a/src/orcapod/core/operators/provisional.py b/src/orcapod/core/operators/provisional.py new file mode 100644 index 000000000..67cf136b3 --- /dev/null +++ b/src/orcapod/core/operators/provisional.py @@ -0,0 +1,96 @@ +"""The provisional operators: identities not frozen with data context v1 (ITL-1266). + +Data context v1 freezes every hash at the 0.4.0 release. A provisional operator is outside +that freeze: its identity (and so the content and pipeline hashes, stored paths and record +ids of its results and of every step downstream of it) may still change in a later release. +Its golden values live in the golden store's provisional files, which are computed and +checked on every run but never frozen by a release. + +``PROVISIONAL_OPERATORS`` is the one list. Each listed operator also sets the class +attribute ``PROVISIONAL = True`` (declared on ``StaticOutputOperatorPod``); a test checks +that the list and the markers agree. Constructing a provisional operator issues +``ProvisionalHashWarning`` once per operator per process and notes the use +(``utils/provisional.note_provisional_use``). An operator leaves the list at a release: +its golden values then move into a frozen golden file, and its identity is frozen from that +release on. +""" + +from __future__ import annotations + +import threading +import warnings +from collections.abc import Mapping +from types import MappingProxyType + +from orcapod.errors import ProvisionalHashWarning +from orcapod.utils.provisional import note_provisional_use +from orcapod.utils.warning_utils import outside_orcapod_stacklevel + +PROVISIONAL_OPERATORS: Mapping[str, str] = MappingProxyType( + { + # Whether the order of its predicates is part of its identity (ITL-1247), and + # whether a general Filter over a Condition supersedes it (ITL-1265). + "PolarsFilter": "ITL-1247", + } +) +"""The provisional operators: class name to the issue that decides its identity.""" + +_lock = threading.RLock() +_issued: set[str] = set() + + +def provisional_operator_name(cls: type) -> str | None: + """The provisional operator a class is, or derives from. + + Args: + cls: An operator class. + + Returns: + The name of the nearest class in ``cls``'s method resolution order that sets + ``PROVISIONAL = True`` itself, or ``None`` when ``cls`` is not provisional. + """ + if not getattr(cls, "PROVISIONAL", False): + return None + for klass in cls.__mro__: + if klass.__dict__.get("PROVISIONAL") is True: + return klass.__name__ + return None + + +def warn_provisional_operator(name: str) -> None: + """Note the use of a provisional operator and warn, once per operator per process. + + The use is noted every time. The warning is recorded as issued only after + ``warnings.warn`` returns, so under an error filter every construction raises. It is + attributed to the first frame outside Orcapod, the code that built the operator. + + Args: + name: The provisional operator's class name. + """ + note_provisional_use(name) + if name in _issued: + return + with _lock: + if name in _issued: + return + warnings.warn( + f"{name} is provisional: its identity is not frozen with data context v1 " + "and may change in a later release. Then the stored paths and record ids " + f"of its results, and of every step downstream of {name}, change, and " + "those results are computed again instead of reused (nothing stored is " + "lost or wrong). See 'Provisional operators' in docs/concepts/operators.md. " + "To silence: warnings.filterwarnings('ignore', " + "category=orcapod.errors.ProvisionalHashWarning).", + ProvisionalHashWarning, + stacklevel=outside_orcapod_stacklevel(), + ) + _issued.add(name) + + +def reset_provisional_operator_warnings() -> None: + """Forget which provisional operators have warned, so each warns again. + + A test hook: a test that checks the warning starts as a new process would. + """ + with _lock: + _issued.clear() diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 9d3122fe1..f7db5505d 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -18,6 +18,10 @@ stream_data_tags, ) from orcapod.core.function_pod import FunctionPodStream +from orcapod.core.operators.provisional import ( + provisional_operator_name, + warn_provisional_operator, +) from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import ( ZERO_ROW_INPUT_COLUMNS, @@ -76,9 +80,20 @@ class StaticOutputOperatorPod(TraceableBase): predicate, so no user code runs to learn the column types (ITL-912). """ + PROVISIONAL: ClassVar[bool] = False + """Whether the operator is provisional: its identity is not frozen (ITL-1266). + + Set to ``True`` exactly by the operators listed in ``PROVISIONAL_OPERATORS`` + (``core/operators/provisional.py``); constructing one issues + ``ProvisionalHashWarning`` once per operator per process. + """ + def __init__( self, tracker_manager: TrackerManagerProtocol | None = None, **kwargs ) -> None: + provisional = provisional_operator_name(type(self)) + if provisional is not None: + warn_provisional_operator(provisional) self.tracker_manager = tracker_manager or DEFAULT_TRACKER_MANAGER super().__init__(**kwargs) diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index f95ec06b7..61593b3d8 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -584,7 +584,7 @@ class OrcapodDatabaseContentWarning(UserWarning): class ProvisionalHashWarning(UserWarning): - """A value type was used whose value hash and stored identity are provisional. + """A value type or an operator was used whose identity is provisional. The library value types (``numpy.ndarray``, ``torch.Tensor``, pandas and polars ``DataFrame`` and ``Series``) are hashed and stored with formats that may still @@ -595,6 +595,12 @@ class ProvisionalHashWarning(UserWarning): a stored column read. Which types are provisional is declared in the ``provisional`` list of ``orcapod/contexts/logical_types.json``. + A provisional operator (``orcapod.operators.PROVISIONAL_OPERATORS``, ITL-1266) has an + identity that is not frozen with data context v1: it may change in a later release, + and with it the stored paths and record ids of its results and of every step + downstream of it, which are then computed again. The warning is issued once per + operator per process, when the operator is constructed. + To silence it, call ``warnings.filterwarnings("ignore", category=ProvisionalHashWarning)`` after importing Orcapod. Python's ``-W`` option and ``PYTHONWARNINGS`` cannot name this class: they are read before installed diff --git a/src/orcapod/utils/provisional.py b/src/orcapod/utils/provisional.py new file mode 100644 index 000000000..cf61e832d --- /dev/null +++ b/src/orcapod/utils/provisional.py @@ -0,0 +1,55 @@ +"""Uses of provisional features: the record of which ones a block of code used (ITL-1266). + +A provisional feature is one whose identity is not frozen with the current data context: a +provisional operator (``core/operators/provisional.py``). Its hashes may change in a later +release, so the golden values that depend on it are kept outside the frozen golden files. +Each provisional feature calls ``note_provisional_use`` with its name whenever it is used, +before its once-per-process warning; ``recording_provisional_uses`` collects those names +for a block of code. The golden hash tool runs each golden case inside one, so a case that +uses a provisional feature it does not declare is refused. Recording changes no warning: +the warnings keep their own once-per-process bookkeeping. +""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager + +_lock = threading.RLock() +_recorders: list[set[str]] = [] + + +def note_provisional_use(name: str) -> None: + """Record that the provisional feature ``name`` was used. + + Every block recording at the time (``recording_provisional_uses``) receives the name, + whichever thread used the feature. + + Args: + name: The feature's name, e.g. ``PolarsFilter``. + """ + with _lock: + for recorder in _recorders: + recorder.add(name) + + +@contextmanager +def recording_provisional_uses() -> Iterator[set[str]]: + """Collect the names of the provisional features used inside the block. + + Blocks may nest; each receives every name noted while it is open. + + Yields: + The set of names noted so far; it keeps growing until the block exits. + """ + recorder: set[str] = set() + with _lock: + _recorders.append(recorder) + try: + yield recorder + finally: + with _lock: + # By identity: two recorders holding the same names are equal sets. + index = next(i for i, r in enumerate(_recorders) if r is recorder) + del _recorders[index] diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py index 3b223553a..ae462bd06 100644 --- a/tests/test_check_golden_tags.py +++ b/tests/test_check_golden_tags.py @@ -122,6 +122,32 @@ def test_a_golden_file_added_after_the_release_passes(repo: Path) -> None: assert result.returncode == 0, result.stderr +def _change_a_provisional_file(repo: Path) -> None: + """Remove one entry of a provisional file with the tool, as an approved change would.""" + path = store.provisional_folder(_folder(repo)) / "semantic.json" + entry = sorted(store.read_json(path))[0] + main(["--root", str(repo), "--remove", entry]) + assert entry not in store.read_json(path) + _commit(repo, f"remove {entry} from a provisional file") + assert store.tree_problems(repo, VERSION) == [] + + +def test_a_provisional_file_changed_after_the_release_passes(repo: Path) -> None: + """A release never freezes a provisional file (ITL-1266): the tag check ignores it.""" + _change_a_provisional_file(repo) + result = _check(repo, ci=True) + assert result.returncode == 0, result.stderr + + +def test_the_gate_passes_a_provisional_change_after_the_release_step( + repo: Path, +) -> None: + """A provisional file changed after the release step adds no revision.""" + _change_a_provisional_file(repo) + result = _gate(repo, "0.3.0") + assert result.returncode == 0, result.stderr + + def test_an_edited_released_file_fails(repo: Path) -> None: path = _folder(repo) / "semantic-01.json" _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) diff --git a/tests/test_core/operators/test_provisional_operators.py b/tests/test_core/operators/test_provisional_operators.py new file mode 100644 index 000000000..4d3abba9a --- /dev/null +++ b/tests/test_core/operators/test_provisional_operators.py @@ -0,0 +1,276 @@ +"""Provisional operators: the list, the markers, the warning and the record of uses (ITL-1266).""" + +from __future__ import annotations + +import inspect +import warnings +from collections.abc import Iterator +from typing import Any, ClassVar + +import pyarrow as pa +import pytest + +import orcapod.operators as operators +from orcapod.core.operators.base import UnaryOperator +from orcapod.core.operators.provisional import ( + PROVISIONAL_OPERATORS, + provisional_operator_name, + reset_provisional_operator_warnings, + warn_provisional_operator, +) +from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod +from orcapod.errors import ProvisionalHashWarning +from orcapod.operators import PolarsFilter +from orcapod.sources import ArrowTableSource +from orcapod.utils.provisional import recording_provisional_uses + + +@pytest.fixture(autouse=True) +def fresh_process() -> Iterator[None]: + """Each test starts as a new process would: no provisional operator has warned.""" + reset_provisional_operator_warnings() + yield + reset_provisional_operator_warnings() + + +def _provisional_warnings(caught: list[warnings.WarningMessage]) -> list[str]: + return [ + str(w.message) for w in caught if issubclass(w.category, ProvisionalHashWarning) + ] + + +def _exported_operators() -> dict[str, type]: + return { + name: obj + for name, obj in vars(operators).items() + if name in operators.__all__ + and inspect.isclass(obj) + and issubclass(obj, StaticOutputOperatorPod) + } + + +class _ProvisionalA(UnaryOperator): + """A provisional test operator.""" + + PROVISIONAL: ClassVar[bool] = True + + def unary_static_process(self, stream: Any) -> Any: + return stream + + def validate_unary_input(self, stream: Any) -> None: + return None + + def unary_output_schema(self, stream: Any, **kwargs: Any) -> Any: + return stream.output_schema(**kwargs) + + def identity_structure(self) -> Any: + return (type(self).__name__,) + + +class _ProvisionalB(_ProvisionalA): + """A second provisional test operator, marked on its own class.""" + + PROVISIONAL: ClassVar[bool] = True + + +class _SubclassOfA(_ProvisionalA): + """Provisional through its base only.""" + + +def _stream() -> ArrowTableSource: + table = pa.table( + { + "subject": ["s1", "s2", "s3"], + "session": [1, 2, 1], + "age": [2, 4, 6], + "weight": [20.5, 22.25, 19.0], + } + ) + return ArrowTableSource( + table, tag_columns=["subject", "session"], tag_only_columns=True + ) + + +# --------------------------------------------------------------------------- +# The list and the markers agree +# --------------------------------------------------------------------------- + + +def test_the_list_and_the_markers_agree() -> None: + exported = _exported_operators() + assert set(PROVISIONAL_OPERATORS) <= set(exported) + marked = { + name + for name, cls in exported.items() + if cls.__dict__.get("PROVISIONAL") is True + } + assert marked == set(PROVISIONAL_OPERATORS) + for name, cls in exported.items(): + expected = name if name in PROVISIONAL_OPERATORS else None + assert provisional_operator_name(cls) == expected, name + + +def test_polars_filter_is_provisional_and_names_its_issue() -> None: + assert dict(PROVISIONAL_OPERATORS) == {"PolarsFilter": "ITL-1247"} + assert "PROVISIONAL_OPERATORS" in operators.__all__ + + +def test_a_subclass_is_provisional_as_its_nearest_marked_base() -> None: + assert provisional_operator_name(_ProvisionalA) == "_ProvisionalA" + assert provisional_operator_name(_ProvisionalB) == "_ProvisionalB" + assert provisional_operator_name(_SubclassOfA) == "_ProvisionalA" + + class _UserFilter(PolarsFilter): + pass + + assert provisional_operator_name(_UserFilter) == "PolarsFilter" + + +# --------------------------------------------------------------------------- +# The warning: at construction, once per operator per process +# --------------------------------------------------------------------------- + + +def test_constructing_polars_filter_warns_once_per_process() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _ = PolarsFilter(constraints={"subject": "s1"}) + _ = PolarsFilter(constraints={"subject": "s2", "session": 1}) + messages = _provisional_warnings(caught) + assert len(messages) == 1 + message = messages[0] + assert message.startswith("PolarsFilter is provisional:") + for part in ( + "not frozen with data context v1", + "stored paths and record ids", + "every step downstream of PolarsFilter", + "'Provisional operators' in docs/concepts/operators.md", + "category=orcapod.errors.ProvisionalHashWarning", + ): + assert part in message + + +def test_the_warning_points_at_the_code_that_built_the_operator() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _ = PolarsFilter() + (found,) = [w for w in caught if issubclass(w.category, ProvisionalHashWarning)] + assert found.filename == __file__ + + +def test_the_stream_method_warns_too() -> None: + import polars as pl + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + filtered = _stream().polars_filter(pl.col("age") > 3, session=1) + assert len(_provisional_warnings(caught)) == 1 + assert filtered.as_table()["subject"].to_pylist() == ["s3"] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + filtered = _stream().polars_filter(pl.col("age") > 3, pl.col("weight") > 20) + rows = filtered.as_table() + assert _provisional_warnings(caught) == [] + assert rows["subject"].to_pylist() == ["s2"] + + +@pytest.mark.parametrize( + "order", [(_ProvisionalA, _ProvisionalB), (_ProvisionalB, _ProvisionalA)] +) +def test_two_provisional_operators_warn_once_each_in_any_order( + order: tuple[type, type], +) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + for cls in (*order, *order): + _ = cls() + messages = _provisional_warnings(caught) + assert [m.split(" ", 1)[0] for m in messages] == [cls.__name__ for cls in order] + + +def test_a_subclass_warns_under_the_name_of_its_provisional_base() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _ = _SubclassOfA() + _ = _ProvisionalA() + messages = _provisional_warnings(caught) + assert len(messages) == 1 + assert messages[0].startswith("_ProvisionalA is provisional:") + + +def test_an_operator_that_is_not_provisional_does_not_warn() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _ = operators.SelectDataColumns(columns=["age", "weight"]) + _ = operators.Join() + assert _provisional_warnings(caught) == [] + + +def test_under_an_error_filter_every_construction_raises() -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error", ProvisionalHashWarning) + for _ in range(2): + with pytest.raises(ProvisionalHashWarning): + _ = PolarsFilter() + + +# --------------------------------------------------------------------------- +# The record of uses +# --------------------------------------------------------------------------- + + +def test_construction_is_recorded_even_after_the_warning() -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + _ = PolarsFilter() + with recording_provisional_uses() as used: + _ = PolarsFilter(constraints={"subject": "s1"}) + assert used == {"PolarsFilter"} + + +@pytest.mark.parametrize("names", [("A", "B"), ("B", "A")]) +def test_a_record_holds_every_name_used_in_any_order(names: tuple[str, str]) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with recording_provisional_uses() as used: + for name in (*names, *names): + warn_provisional_operator(name) + assert used == {"A", "B"} + + +def test_nested_records_each_see_what_was_used_while_they_were_open() -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with recording_provisional_uses() as outer: + warn_provisional_operator("A") + with recording_provisional_uses() as inner: + warn_provisional_operator("B") + # An equal set, open beside it, is a different record. + with ( + recording_provisional_uses() as first, + recording_provisional_uses() as second, + ): + pass + warn_provisional_operator("C") + assert outer == {"A", "B", "C"} + assert inner == {"B"} + assert first == second == set() + + +def test_a_record_closed_by_an_error_stops_recording() -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with pytest.raises(RuntimeError), recording_provisional_uses() as used: + raise RuntimeError("case failed") + warn_provisional_operator("A") + assert used == set() + + +def test_recording_changes_no_warning() -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with recording_provisional_uses(): + _ = PolarsFilter() + with recording_provisional_uses(): + _ = PolarsFilter() + assert len(_provisional_warnings(caught)) == 1 diff --git a/tests/test_hashing/golden/README.md b/tests/test_hashing/golden/README.md index 9c8431920..ff2876454 100644 --- a/tests/test_hashing/golden/README.md +++ b/tests/test_hashing/golden/README.md @@ -13,6 +13,12 @@ In short: `src/orcapod/contexts/released_manifests.json`. - A file listed by a released revision never changes. New values go into new files, in any release. A change to a released value needs a new data context version. +- `data//provisional/.json` holds the values of the cases that depend on a + provisional operator (`orcapod.operators.PROVISIONAL_OPERATORS`; ITL-1266). A case + declares them (`@case(..., provisional=("PolarsFilter",))`); a case that builds one without + declaring it is refused. The manifest lists these files in `golden.provisional`, outside + the history: no release freezes them. `--relocate` moves entries, values unchanged, + between the frozen and the provisional files as their cases declare. - Data context v1 is open until 0.4.0: 0.3.0 was withdrawn on 2026-11-17 (ITL-1255). Its release revision is out of the history, and the pins it shipped are kept under `withdrawn` in `released_manifests.json`, with the date and the reason. Until 0.4.0 a change to a v1 @@ -22,7 +28,7 @@ In short: | File | Role | | --- | --- | | `cases.py` | The cases. Never move or rename a fixture: they hash with their module path. | -| `generate_golden_hashes.py` | Adds new entries (default), reports drift (`--check`), removes an unreleased entry (`--remove`), records a release (`--release X.Y.Z`). Read its header first. | +| `generate_golden_hashes.py` | Adds new entries (default), reports drift (`--check`), removes an unreleased entry (`--remove`), moves entries between frozen and provisional files (`--relocate`), records a release (`--release X.Y.Z`). Read its header first. | | `store.py` | The rules: file names, digests, history, pins. | | `test_golden_hashes.py` | Computes every entry again; refuses skips but the optional-package rule. | | `test_golden_store.py` | The rules and the generator, on copies of the tree. | diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index add9bcb9e..c35bb6fcc 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -75,55 +75,130 @@ def available_groups() -> tuple[str, ...]: return tuple(g for g in GROUPS if group_available(g)) -_CASES: dict[str, list[tuple[str, CaseFn]]] = {group: [] for group in GROUPS} +_CASES: dict[str, list[tuple[str, CaseFn, tuple[str, ...]]]] = { + group: [] for group in GROUPS +} -def case(group: str, name: str) -> Callable[[CaseFn], CaseFn]: +def case( + group: str, name: str, *, provisional: tuple[str, ...] = () +) -> Callable[[CaseFn], CaseFn]: """Register a golden case under ``group``. Args: group: One of ``GROUPS``; selects the golden files (``-NN.json``) that pin the entries. name: Case name, used as the prefix of every entry the case returns. + provisional: The provisional features the case depends on + (``declarable_provisional_names``; ITL-1266). A case that declares one is + pinned in the provisional file of its group, which no release freezes. A case + that uses a provisional feature must declare it (``compute_group_kinds``). Returns: A decorator that registers the case function and returns it unchanged. """ def register(fn: CaseFn) -> CaseFn: - _CASES[group].append((name, fn)) + _CASES[group].append((name, fn, tuple(provisional))) return fn return register -def compute_group(group: str, *, allow_errors: bool = False) -> dict[str, Any]: - """Compute every entry of one golden group. +def declarable_provisional_names() -> frozenset[str]: + """The names a case may declare provisional: the provisional operators (ITL-1266).""" + from orcapod.core.operators.provisional import PROVISIONAL_OPERATORS + + return frozenset(PROVISIONAL_OPERATORS) + + +def _recording() -> Any: + """``recording_provisional_uses`` of the library, or an empty record for an older one. + + The generator is also run against an older checkout's library (``--stdout``), which + has no record of provisional uses. + """ + try: + from orcapod.utils.provisional import recording_provisional_uses + except ImportError: + return contextlib.nullcontext(set()) + return recording_provisional_uses() + + +def compute_group_kinds( + group: str, *, allow_errors: bool = False +) -> tuple[dict[str, Any], frozenset[str]]: + """Compute every entry of one golden group, and say which entries are provisional. + + Each case runs while the library records the provisional features it uses + (``orcapod.utils.provisional.recording_provisional_uses``). A case that uses one it + does not declare, or declares a name ``declarable_provisional_names`` does not hold, + is refused, so a value that depends on a provisional operator can never be pinned in + a frozen file. Args: group: One of ``GROUPS``. allow_errors: When true, a case that raises records a single - ``"ERROR: : "`` entry instead of propagating. Used only when - running the generator against an older release. + ``"ERROR: : "`` entry instead of propagating, and the + declarations are not checked. Used only when running the generator against an + older release. Returns: - A dict of ``.`` names to JSON-compatible values, sorted by name. + A dict of ``.`` names to JSON-compatible values, sorted by name, and + the names of the entries of cases that declare a provisional feature. + + Raises: + ValueError: A case declares an unknown provisional name, uses a provisional + feature it does not declare, or two cases produce one entry name. """ out: dict[str, Any] = {} - for name, fn in _CASES[group]: + provisional_entries: set[str] = set() + for name, fn, declared in _CASES[group]: + if not allow_errors: + unknown = sorted(set(declared) - declarable_provisional_names()) + if unknown: + raise ValueError( + f"golden case {group}:{name} declares {unknown} provisional, but " + "they are not provisional (orcapod.operators.PROVISIONAL_OPERATORS)" + ) try: - values = fn() + with _recording() as used: + values = fn() except Exception as exc: if not allow_errors: raise out[name] = f"ERROR: {type(exc).__name__}: {str(exc).splitlines()[0][:200]}" continue + undeclared = sorted(set(used) - set(declared)) + if undeclared and not allow_errors: + raise ValueError( + f"golden case {group}:{name} uses the provisional {undeclared} without " + f"declaring it: register it with @case({group!r}, {name!r}, " + f"provisional={tuple(sorted(used))!r}), so that its values go to the " + "provisional file, never to a frozen one" + ) for key, value in values.items(): full = f"{name}.{key}" if key else name if full in out: raise ValueError(f"duplicate golden entry {full!r}") out[full] = value - return dict(sorted(out.items())) + if declared: + provisional_entries.add(full) + return dict(sorted(out.items())), frozenset(provisional_entries) + + +def compute_group(group: str, *, allow_errors: bool = False) -> dict[str, Any]: + """Compute every entry of one golden group (``compute_group_kinds``, values only). + + Args: + group: One of ``GROUPS``. + allow_errors: Passed to ``compute_group_kinds``. + + Returns: + A dict of ``.`` names to JSON-compatible values, sorted by name. + """ + values, _ = compute_group_kinds(group, allow_errors=allow_errors) + return values def compute_all(*, allow_errors: bool = False) -> dict[str, dict[str, Any]]: @@ -1673,7 +1748,7 @@ def pipeline_map_columns() -> dict[str, Any]: return out -@case("pipeline", "polars_filter") +@case("pipeline", "polars_filter", provisional=("PolarsFilter",)) def pipeline_polars_filter() -> dict[str, Any]: from orcapod.operators import PolarsFilter @@ -1918,14 +1993,16 @@ def _polars_exprs() -> dict[str, Any]: } -@case("semantic", "polars_expr") +# In the library only PolarsFilter feeds polars_expr_v1 (its predicates), so these hashes are +# as provisional as PolarsFilter is (ITL-1266), although the case builds no operator. +@case("semantic", "polars_expr", provisional=("PolarsFilter",)) def semantic_polars_exprs() -> dict[str, Any]: """Record the ``polars_expr_v1`` semantic hash of each representative expression.""" hasher = _ctx().semantic_hasher return {k: hasher.hash_object(e).to_string() for k, e in _polars_exprs().items()} -@case("pipeline", "polars_filter_expr") +@case("pipeline", "polars_filter_expr", provisional=("PolarsFilter",)) def pipeline_polars_filter_expr() -> dict[str, Any]: """Record expression-only and expression-plus-keyword filter streams.""" import polars as pl @@ -1949,7 +2026,7 @@ def pipeline_polars_filter_expr() -> dict[str, Any]: return out -@case("pipeline", "polars_filter_expr_job") +@case("pipeline", "polars_filter_expr_job", provisional=("PolarsFilter",)) def pipeline_polars_filter_expr_job() -> dict[str, Any]: """Record a job with an expression filter upstream of a join and a function pod.""" import polars as pl diff --git a/tests/test_hashing/golden/data/v1/pipeline-01.json b/tests/test_hashing/golden/data/v1/pipeline-01.json index 4bf77eab0..0e935f79f 100644 --- a/tests/test_hashing/golden/data/v1/pipeline-01.json +++ b/tests/test_hashing/golden/data/v1/pipeline-01.json @@ -526,117 +526,6 @@ "v0", "python.function.v0" ], - "polars_filter.content_hash": "semantic_v1:54997acd8e669fd9a0cbb1010e0e7e4551bb1a7a830432d6d12a281605955545", - "polars_filter.pipeline_hash": "semantic_v1:5df6831d8a21a134aef33ea7b765c7ee081e0742852c5e4c876ffd7b9c8b7b3e", - "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_0", - "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", - "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s1\"}].data_hash": "arrow_v1:000002c734791b7452bb2affc60031491e7541bb53fb971cf3ef8b9af919eb0bf1f56e", - "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_1", - "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", - "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s2\"}].data_hash": "arrow_v1:00000252e4f32a133cd6a8813f816e08b6bde1cdb467baa95d026fc41c5672b6b98110", - "polars_filter.system_tag_columns": [ - "_tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b", - "_tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b" - ], - "polars_filter_expr.expr.content_hash": "semantic_v1:2d0ffcd23b7b71647572734ba79b66d404563017dd9ce698cda0429d1b24e358", - "polars_filter_expr.expr.pipeline_hash": "semantic_v1:534dc49294c37c9c5a0b7af8b3b9289d43a68d693bfd7e3e35944b9a2073dc11", - "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_0", - "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", - "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s1\"}].data_hash": "arrow_v1:000002c734791b7452bb2affc60031491e7541bb53fb971cf3ef8b9af919eb0bf1f56e", - "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_1", - "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", - "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s2\"}].data_hash": "arrow_v1:00000252e4f32a133cd6a8813f816e08b6bde1cdb467baa95d026fc41c5672b6b98110", - "polars_filter_expr.expr.system_tag_columns": [ - "_tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b", - "_tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b" - ], - "polars_filter_expr.mixed.content_hash": "semantic_v1:58096b69c61a4fb9e3b44a5b02575a073791fa26ffd566fbc75d55d4cac26107", - "polars_filter_expr.mixed.pipeline_hash": "semantic_v1:64e19a9f7956c079cf91384dab5eaf4d8f153555371f2854948338736495bc23", - "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_0", - "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", - "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s1\"}].data_hash": "arrow_v1:000002c734791b7452bb2affc60031491e7541bb53fb971cf3ef8b9af919eb0bf1f56e", - "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_1", - "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", - "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s2\"}].data_hash": "arrow_v1:00000252e4f32a133cd6a8813f816e08b6bde1cdb467baa95d026fc41c5672b6b98110", - "polars_filter_expr.mixed.system_tag_columns": [ - "_tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b", - "_tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b" - ], - "polars_filter_expr_job.node[adjust].content_hash": "semantic_v1:0bba107cb608528b0c7a7faf80ed7a3ae2eb9c54ba5b213af364f05def99d9be", - "polars_filter_expr_job.node[adjust].pipeline_hash": "semantic_v1:842a8c9fbaf166db981e440406c05a3735bda77517ad4b6c5f9bcbd41409931e", - "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}]._tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0": "row_1", - "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1": "row_1", - "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}]._tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0": "scans", - "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1": "subjects", - "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:0000023565098d1a88ecd75dbc8ddc4900dc8304e066dc0c889114a2a41e2fe495fc30", - "polars_filter_expr_job.node[adjust].system_tag_columns": [ - "_tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0", - "_tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0", - "_tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1", - "_tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1" - ], - "polars_filter_expr_job.node[joined].content_hash": "semantic_v1:5ff4fb38970380dfd2759f649ded300073d54ac12357f2bc767f1f0b4c876b0f", - "polars_filter_expr_job.node[joined].pipeline_hash": "semantic_v1:e9b5cec0f507f21a90e40dceed0bcfa8e91f12a33f1d697cf871e48b73b1efd1", - "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}]._tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0": "row_1", - "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1": "row_1", - "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}]._tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0": "scans", - "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1": "subjects", - "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:0000026b8369e60be10e43dd72a0be4e8d93a992f827cd6baab33909331dcdaeea8b64", - "polars_filter_expr_job.node[joined].system_tag_columns": [ - "_tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0", - "_tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0", - "_tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1", - "_tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1" - ], - "polars_filter_expr_job.node[older].content_hash": "semantic_v1:1908d5e1122ed5c834dbb1403fa1c5ec35deab663f8b5d3e667f1ce084bfb3a7", - "polars_filter_expr_job.node[older].pipeline_hash": "semantic_v1:701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369", - "polars_filter_expr_job.node[older].row[{\"subject\":\"s2\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_1", - "polars_filter_expr_job.node[older].row[{\"subject\":\"s2\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", - "polars_filter_expr_job.node[older].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:000002338ce8710f5ecb156a72c8446651adb7b0c21635d0aa92d298f09da56faed0f8", - "polars_filter_expr_job.node[older].row[{\"subject\":\"s3\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_2", - "polars_filter_expr_job.node[older].row[{\"subject\":\"s3\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", - "polars_filter_expr_job.node[older].row[{\"subject\":\"s3\"}].data_hash": "arrow_v1:0000023401f1ce40c8cfdd6e4f343d7ffdcd51c1f0521f44cf6030e27e1ca0da154616", - "polars_filter_expr_job.node[older].system_tag_columns": [ - "_tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373", - "_tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373" - ], - "polars_filter_expr_job.node[scans].content_hash": "semantic_v1:63daeb3e3c8a817ebd5c6dd711409161ef7c97868666af235a37cdb871975465", - "polars_filter_expr_job.node[scans].pipeline_hash": "semantic_v1:24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3", - "polars_filter_expr_job.node[scans].row[{\"subject\":\"s1\"}]._tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3": "row_0", - "polars_filter_expr_job.node[scans].row[{\"subject\":\"s1\"}]._tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3": "scans", - "polars_filter_expr_job.node[scans].row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:000002ef2b37a1979bd734849c241456c50e96644db0ba5a7b25ecb84cee278e6686f2", - "polars_filter_expr_job.node[scans].row[{\"subject\":\"s2\"}]._tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3": "row_1", - "polars_filter_expr_job.node[scans].row[{\"subject\":\"s2\"}]._tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3": "scans", - "polars_filter_expr_job.node[scans].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:00000210b14e22145ce1d5f71e190ade4ca03ba523ad98d8749ea7beef3c15dd749c20", - "polars_filter_expr_job.node[scans].system_tag_columns": [ - "_tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3", - "_tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3" - ], - "polars_filter_expr_job.node[subjects].content_hash": "semantic_v1:150c29f4961fce087d8c8c9706a7347e4a171aacc3e64ac8c2cc22d3ab9d0bfd", - "polars_filter_expr_job.node[subjects].pipeline_hash": "semantic_v1:ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s1\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_0", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s1\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:000002d1c5b49de38ef6efc552c2a47b8ab59dd267f8eb55d91814edee99b1b52acc65", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s2\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_1", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s2\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:000002338ce8710f5ecb156a72c8446651adb7b0c21635d0aa92d298f09da56faed0f8", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s3\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_2", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s3\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", - "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s3\"}].data_hash": "arrow_v1:0000023401f1ce40c8cfdd6e4f343d7ffdcd51c1f0521f44cf6030e27e1ca0da154616", - "polars_filter_expr_job.node[subjects].system_tag_columns": [ - "_tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373", - "_tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373" - ], - "polars_filter_expr_job.table[golden_filter_job/_result/golden_adjust/a2fa1ad227c2c0b24315bf0bfca5b3f5473e1f044a6d48cfcf3b78977510a560/v0/python.function.v0/rdb_v2].input_data_hashes": [ - "arrow_v1:0000026b8369e60be10e43dd72a0be4e8d93a992f827cd6baab33909331dcdaeea8b64" - ], - "polars_filter_expr_job.table[golden_filter_job/golden_adjust/a2fa1ad227c2c0b24315bf0bfca5b3f5473e1f044a6d48cfcf3b78977510a560/v0/python.function.v0/schema:842a8c9fbaf166db981e440406c05a3735bda77517ad4b6c5f9bcbd41409931e/pdb_v2].record_ids": [ - "arrow_v1:000002805f559cb2ee882683918a4bb57099b86c725fb9c48a42490461633f8833bf5f" - ], - "polars_filter_expr_job.table_paths": [ - "golden_filter_job/_result/golden_adjust/a2fa1ad227c2c0b24315bf0bfca5b3f5473e1f044a6d48cfcf3b78977510a560/v0/python.function.v0/rdb_v2", - "golden_filter_job/golden_adjust/a2fa1ad227c2c0b24315bf0bfca5b3f5473e1f044a6d48cfcf3b78977510a560/v0/python.function.v0/schema:842a8c9fbaf166db981e440406c05a3735bda77517ad4b6c5f9bcbd41409931e/pdb_v2" - ], "semi_join.content_hash": "semantic_v1:0323ff1b5b8bb4ccc3be3b5dabc54907706462d16a50154a94888acbacab718d", "semi_join.pipeline_hash": "semantic_v1:3d0cfb82fa371449d121f28cf63dc0982ba5cd35df2eae729f0b22f0f05c3a47", "semi_join.row[{\"subject\":\"s1\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_0", diff --git a/tests/test_hashing/golden/data/v1/provisional/pipeline.json b/tests/test_hashing/golden/data/v1/provisional/pipeline.json new file mode 100644 index 000000000..535e60b2f --- /dev/null +++ b/tests/test_hashing/golden/data/v1/provisional/pipeline.json @@ -0,0 +1,113 @@ +{ + "polars_filter.content_hash": "semantic_v1:54997acd8e669fd9a0cbb1010e0e7e4551bb1a7a830432d6d12a281605955545", + "polars_filter.pipeline_hash": "semantic_v1:5df6831d8a21a134aef33ea7b765c7ee081e0742852c5e4c876ffd7b9c8b7b3e", + "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_0", + "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", + "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s1\"}].data_hash": "arrow_v1:000002c734791b7452bb2affc60031491e7541bb53fb971cf3ef8b9af919eb0bf1f56e", + "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_1", + "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", + "polars_filter.row[{\"cohort\":\"a\",\"subject\":\"s2\"}].data_hash": "arrow_v1:00000252e4f32a133cd6a8813f816e08b6bde1cdb467baa95d026fc41c5672b6b98110", + "polars_filter.system_tag_columns": [ + "_tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b", + "_tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b" + ], + "polars_filter_expr.expr.content_hash": "semantic_v1:2d0ffcd23b7b71647572734ba79b66d404563017dd9ce698cda0429d1b24e358", + "polars_filter_expr.expr.pipeline_hash": "semantic_v1:534dc49294c37c9c5a0b7af8b3b9289d43a68d693bfd7e3e35944b9a2073dc11", + "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_0", + "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", + "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s1\"}].data_hash": "arrow_v1:000002c734791b7452bb2affc60031491e7541bb53fb971cf3ef8b9af919eb0bf1f56e", + "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_1", + "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", + "polars_filter_expr.expr.row[{\"cohort\":\"a\",\"subject\":\"s2\"}].data_hash": "arrow_v1:00000252e4f32a133cd6a8813f816e08b6bde1cdb467baa95d026fc41c5672b6b98110", + "polars_filter_expr.expr.system_tag_columns": [ + "_tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b", + "_tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b" + ], + "polars_filter_expr.mixed.content_hash": "semantic_v1:58096b69c61a4fb9e3b44a5b02575a073791fa26ffd566fbc75d55d4cac26107", + "polars_filter_expr.mixed.pipeline_hash": "semantic_v1:64e19a9f7956c079cf91384dab5eaf4d8f153555371f2854948338736495bc23", + "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_0", + "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s1\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", + "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s1\"}].data_hash": "arrow_v1:000002c734791b7452bb2affc60031491e7541bb53fb971cf3ef8b9af919eb0bf1f56e", + "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "row_1", + "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s2\"}]._tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b": "cohorts", + "polars_filter_expr.mixed.row[{\"cohort\":\"a\",\"subject\":\"s2\"}].data_hash": "arrow_v1:00000252e4f32a133cd6a8813f816e08b6bde1cdb467baa95d026fc41c5672b6b98110", + "polars_filter_expr.mixed.system_tag_columns": [ + "_tag_source_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b", + "_tag_record_id::10962e3780f562f563f6567468578100550aaeed130ff5e6bf2da41bd2239b5b" + ], + "polars_filter_expr_job.node[adjust].content_hash": "semantic_v1:0bba107cb608528b0c7a7faf80ed7a3ae2eb9c54ba5b213af364f05def99d9be", + "polars_filter_expr_job.node[adjust].pipeline_hash": "semantic_v1:842a8c9fbaf166db981e440406c05a3735bda77517ad4b6c5f9bcbd41409931e", + "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}]._tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0": "row_1", + "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1": "row_1", + "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}]._tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0": "scans", + "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1": "subjects", + "polars_filter_expr_job.node[adjust].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:0000023565098d1a88ecd75dbc8ddc4900dc8304e066dc0c889114a2a41e2fe495fc30", + "polars_filter_expr_job.node[adjust].system_tag_columns": [ + "_tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0", + "_tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0", + "_tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1", + "_tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1" + ], + "polars_filter_expr_job.node[joined].content_hash": "semantic_v1:5ff4fb38970380dfd2759f649ded300073d54ac12357f2bc767f1f0b4c876b0f", + "polars_filter_expr_job.node[joined].pipeline_hash": "semantic_v1:e9b5cec0f507f21a90e40dceed0bcfa8e91f12a33f1d697cf871e48b73b1efd1", + "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}]._tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0": "row_1", + "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1": "row_1", + "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}]._tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0": "scans", + "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1": "subjects", + "polars_filter_expr_job.node[joined].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:0000026b8369e60be10e43dd72a0be4e8d93a992f827cd6baab33909331dcdaeea8b64", + "polars_filter_expr_job.node[joined].system_tag_columns": [ + "_tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0", + "_tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3:0", + "_tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1", + "_tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373::701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369:1" + ], + "polars_filter_expr_job.node[older].content_hash": "semantic_v1:1908d5e1122ed5c834dbb1403fa1c5ec35deab663f8b5d3e667f1ce084bfb3a7", + "polars_filter_expr_job.node[older].pipeline_hash": "semantic_v1:701a92df2c5cdea97f8239435c1de09e875eb22a843c5e279c77b21be256f369", + "polars_filter_expr_job.node[older].row[{\"subject\":\"s2\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_1", + "polars_filter_expr_job.node[older].row[{\"subject\":\"s2\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", + "polars_filter_expr_job.node[older].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:000002338ce8710f5ecb156a72c8446651adb7b0c21635d0aa92d298f09da56faed0f8", + "polars_filter_expr_job.node[older].row[{\"subject\":\"s3\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_2", + "polars_filter_expr_job.node[older].row[{\"subject\":\"s3\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", + "polars_filter_expr_job.node[older].row[{\"subject\":\"s3\"}].data_hash": "arrow_v1:0000023401f1ce40c8cfdd6e4f343d7ffdcd51c1f0521f44cf6030e27e1ca0da154616", + "polars_filter_expr_job.node[older].system_tag_columns": [ + "_tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373", + "_tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373" + ], + "polars_filter_expr_job.node[scans].content_hash": "semantic_v1:63daeb3e3c8a817ebd5c6dd711409161ef7c97868666af235a37cdb871975465", + "polars_filter_expr_job.node[scans].pipeline_hash": "semantic_v1:24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3", + "polars_filter_expr_job.node[scans].row[{\"subject\":\"s1\"}]._tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3": "row_0", + "polars_filter_expr_job.node[scans].row[{\"subject\":\"s1\"}]._tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3": "scans", + "polars_filter_expr_job.node[scans].row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:000002ef2b37a1979bd734849c241456c50e96644db0ba5a7b25ecb84cee278e6686f2", + "polars_filter_expr_job.node[scans].row[{\"subject\":\"s2\"}]._tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3": "row_1", + "polars_filter_expr_job.node[scans].row[{\"subject\":\"s2\"}]._tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3": "scans", + "polars_filter_expr_job.node[scans].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:00000210b14e22145ce1d5f71e190ade4ca03ba523ad98d8749ea7beef3c15dd749c20", + "polars_filter_expr_job.node[scans].system_tag_columns": [ + "_tag_source_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3", + "_tag_record_id::24bf4e4f1b5fbfa7ff9ee7280a5f680888f965fa0a855159ea5779047a4b1aa3" + ], + "polars_filter_expr_job.node[subjects].content_hash": "semantic_v1:150c29f4961fce087d8c8c9706a7347e4a171aacc3e64ac8c2cc22d3ab9d0bfd", + "polars_filter_expr_job.node[subjects].pipeline_hash": "semantic_v1:ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s1\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_0", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s1\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s1\"}].data_hash": "arrow_v1:000002d1c5b49de38ef6efc552c2a47b8ab59dd267f8eb55d91814edee99b1b52acc65", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s2\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_1", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s2\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s2\"}].data_hash": "arrow_v1:000002338ce8710f5ecb156a72c8446651adb7b0c21635d0aa92d298f09da56faed0f8", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s3\"}]._tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "row_2", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s3\"}]._tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373": "subjects", + "polars_filter_expr_job.node[subjects].row[{\"subject\":\"s3\"}].data_hash": "arrow_v1:0000023401f1ce40c8cfdd6e4f343d7ffdcd51c1f0521f44cf6030e27e1ca0da154616", + "polars_filter_expr_job.node[subjects].system_tag_columns": [ + "_tag_source_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373", + "_tag_record_id::ac699f6aae66ff1745c42c76a3cb6cf59ba8588efc02d66e4f23de4be09ac373" + ], + "polars_filter_expr_job.table[golden_filter_job/_result/golden_adjust/a2fa1ad227c2c0b24315bf0bfca5b3f5473e1f044a6d48cfcf3b78977510a560/v0/python.function.v0/rdb_v2].input_data_hashes": [ + "arrow_v1:0000026b8369e60be10e43dd72a0be4e8d93a992f827cd6baab33909331dcdaeea8b64" + ], + "polars_filter_expr_job.table[golden_filter_job/golden_adjust/a2fa1ad227c2c0b24315bf0bfca5b3f5473e1f044a6d48cfcf3b78977510a560/v0/python.function.v0/schema:842a8c9fbaf166db981e440406c05a3735bda77517ad4b6c5f9bcbd41409931e/pdb_v2].record_ids": [ + "arrow_v1:000002805f559cb2ee882683918a4bb57099b86c725fb9c48a42490461633f8833bf5f" + ], + "polars_filter_expr_job.table_paths": [ + "golden_filter_job/_result/golden_adjust/a2fa1ad227c2c0b24315bf0bfca5b3f5473e1f044a6d48cfcf3b78977510a560/v0/python.function.v0/rdb_v2", + "golden_filter_job/golden_adjust/a2fa1ad227c2c0b24315bf0bfca5b3f5473e1f044a6d48cfcf3b78977510a560/v0/python.function.v0/schema:842a8c9fbaf166db981e440406c05a3735bda77517ad4b6c5f9bcbd41409931e/pdb_v2" + ] +} diff --git a/tests/test_hashing/golden/data/v1/provisional/semantic.json b/tests/test_hashing/golden/data/v1/provisional/semantic.json new file mode 100644 index 000000000..c2484b5e5 --- /dev/null +++ b/tests/test_hashing/golden/data/v1/provisional/semantic.json @@ -0,0 +1,12 @@ +{ + "polars_expr.and_two_columns": "polars_expr_v1:f8d1228cb5a4f9ca281f2918acc70ac9067052301c4784ec205ed71d6648afd7", + "polars_expr.eq_string": "polars_expr_v1:64fbdf549ede833eed74ee19dd1884292576534c215bfdb2521ce16df514132a", + "polars_expr.gt_datetime_utc": "polars_expr_v1:b791db9b35617d6296304cc8eda154da2fe7049565b9d0e209c9a39ac731b603", + "polars_expr.gt_dyn_float": "polars_expr_v1:4de5bd98bb5ddcf20b0bc3fef204586e995ac3a33b4a1617896897b26c2ebe12", + "polars_expr.gt_dyn_int": "polars_expr_v1:347941c97f28667e53bdfe3a2c02599987ec948b6c5e7a5f7339614cdf800e4e", + "polars_expr.gt_int8": "polars_expr_v1:20e3bc7bc11db8434b9f3aeb3cd27ae7450108ae8d308c394a8497727ec686ee", + "polars_expr.is_between_dates": "polars_expr_v1:1a2d2d7c79256108f22085b36703248fc21f9460a87027ba9285bb09b7cceb3c", + "polars_expr.is_in_strings": "polars_expr_v1:cd510b62f38f9e3e3f1b1aba775926ccc2b9b4345867936af59b1779d9c2fc72", + "polars_expr.is_not_null_cast": "polars_expr_v1:7a45a5e5c723df3d392600b79eb4dcfc3eb454b9f18b91f356b985fb5e90c4b3", + "polars_expr.str_contains": "polars_expr_v1:4c4cc5f6fe5bed1fbff4f59706145e216bb2597c2710c557d5ab0c39de476feb" +} diff --git a/tests/test_hashing/golden/data/v1/semantic-01.json b/tests/test_hashing/golden/data/v1/semantic-01.json index 0439e7c29..7af234ab8 100644 --- a/tests/test_hashing/golden/data/v1/semantic-01.json +++ b/tests/test_hashing/golden/data/v1/semantic-01.json @@ -25,16 +25,6 @@ "function.value.defaults": "semantic_v1:6fd6dab5cff8003986bad9ff2ff02a943fd2b857755ee7e3a5d26036709c05e0", "function.value.kinds": "semantic_v1:418c943f60795418e977f3baf27136be860052c190bb7bf4c41e3a7c3cf990d3", "function.value.plain": "semantic_v1:6ae9e5ea388c3c27a267f0e8532fe1086e1d050a19d4ba544ad4fdc9a988da87", - "polars_expr.and_two_columns": "polars_expr_v1:f8d1228cb5a4f9ca281f2918acc70ac9067052301c4784ec205ed71d6648afd7", - "polars_expr.eq_string": "polars_expr_v1:64fbdf549ede833eed74ee19dd1884292576534c215bfdb2521ce16df514132a", - "polars_expr.gt_datetime_utc": "polars_expr_v1:b791db9b35617d6296304cc8eda154da2fe7049565b9d0e209c9a39ac731b603", - "polars_expr.gt_dyn_float": "polars_expr_v1:4de5bd98bb5ddcf20b0bc3fef204586e995ac3a33b4a1617896897b26c2ebe12", - "polars_expr.gt_dyn_int": "polars_expr_v1:347941c97f28667e53bdfe3a2c02599987ec948b6c5e7a5f7339614cdf800e4e", - "polars_expr.gt_int8": "polars_expr_v1:20e3bc7bc11db8434b9f3aeb3cd27ae7450108ae8d308c394a8497727ec686ee", - "polars_expr.is_between_dates": "polars_expr_v1:1a2d2d7c79256108f22085b36703248fc21f9460a87027ba9285bb09b7cceb3c", - "polars_expr.is_in_strings": "polars_expr_v1:cd510b62f38f9e3e3f1b1aba775926ccc2b9b4345867936af59b1779d9c2fc72", - "polars_expr.is_not_null_cast": "polars_expr_v1:7a45a5e5c723df3d392600b79eb4dcfc3eb454b9f18b91f356b985fb5e90c4b3", - "polars_expr.str_contains": "polars_expr_v1:4c4cc5f6fe5bed1fbff4f59706145e216bb2597c2710c557d5ab0c39de476feb", "polars_type.dataframe": "semantic_v1:e924850e67f9d3c6f66abf7f72fe71b131e5e8afb9bc958c135f7b0f5a69636f", "polars_type.series": "semantic_v1:7c68ad68d1bf959a57401cee2679d3ab01b390d118e091c87f8346cec48ea05e", "polars_value.empty_with_schema": "polars_dataframe_v1:87cefe5fbb989beba6cda87cc1a334d643b878eccbc113d4daf8f0ba74192d81", diff --git a/tests/test_hashing/golden/generate_golden_hashes.py b/tests/test_hashing/golden/generate_golden_hashes.py index 1b7e8d6d3..15e5f8faf 100644 --- a/tests/test_hashing/golden/generate_golden_hashes.py +++ b/tests/test_hashing/golden/generate_golden_hashes.py @@ -6,6 +6,7 @@ uv run python tests/test_hashing/golden/generate_golden_hashes.py --check # report uv run python tests/test_hashing/golden/generate_golden_hashes.py --release 0.4.0 uv run python tests/test_hashing/golden/generate_golden_hashes.py --remove + uv run python tests/test_hashing/golden/generate_golden_hashes.py --relocate The golden values of the current data context live in ``tests/test_hashing/golden/data//-NN.json``, and the manifest of the data @@ -45,6 +46,16 @@ unreleased file and records that file again in the open revision (it refuses an entry a release lists), then ``--add`` writes the new value. +5. Provisional operators (``orcapod.operators.PROVISIONAL_OPERATORS``, ITL-1266) are outside + the freeze. A case that depends on one declares it (``@case(..., provisional=(...))``); + its entries go to ``/provisional/.json``, which the manifest lists in + ``golden.provisional``, outside the history, so no release freezes them. Their values + are checked like every other and change only with the developer's approval of the exact + list (``--remove`` then ``--add``), without a new data context. ``--relocate`` moves + pinned entries, values unchanged, to the kind of file their case declares: out of an + unreleased frozen file into the provisional file, or, when an operator leaves the group + before a release, into the open frozen file of its group. + Never edit a golden file to make a failing test pass. If a test fails and you did not mean to change a hash, the code change is the bug. @@ -125,16 +136,101 @@ def _require_sound(root: pathlib.Path, version: str) -> None: ) +def _provisional_listing(manifest: dict[str, Any]) -> dict[str, str]: + """A copy of the manifest's ``golden.provisional`` listing.""" + return dict(manifest["golden"][store.PROVISIONAL]) + + +def _set_provisional_listing(manifest: dict[str, Any], listing: dict[str, str]) -> None: + manifest["golden"][store.PROVISIONAL] = dict(sorted(listing.items())) + + +def _open_frozen_file( + folder: pathlib.Path, + files: dict[str, list[pathlib.Path]], + released: set[str], + group: str, +) -> pathlib.Path: + """The frozen file new entries of ``group`` go to. + + The newest file of the group when no release lists it, otherwise a new + ``-NN.json`` with the next counter. + """ + existing = files.get(group, []) + latest = existing[-1] if existing else None + if latest is not None and latest.name not in released: + return latest + number = 1 + if latest is not None: + match = store.FILE_NAME.match(latest.name) + assert match is not None + number = int(match["number"]) + 1 + if number > 99: + raise SystemExit(f"{group} has no free file counter") + return folder / f"{group}-{number:02d}.json" + + +class _Plan: + """Files to write and the history and listing that record them; nothing written yet.""" + + def __init__(self, folder: pathlib.Path, manifest: dict[str, Any]) -> None: + self.folder = folder + self.history: list[dict[str, Any]] = manifest["golden"]["history"] + self.listing = _provisional_listing(manifest) + self.contents: dict[pathlib.Path, dict[str, Any]] = {} + + def values(self, path: pathlib.Path) -> dict[str, Any]: + """The planned entries of a file, read from disk on first use.""" + if path not in self.contents: + self.contents[path] = dict(store.read_json(path)) if path.exists() else {} + return self.contents[path] + + def write(self, manifest: dict[str, Any], manifest_path: pathlib.Path) -> None: + """Write the planned files, record them, and write the manifest.""" + provisional = store.provisional_folder(self.folder) + writes: list[tuple[pathlib.Path, bytes | None]] = [] + for path, values in self.contents.items(): + if path.parent == provisional: + if values: + data = dump(values).encode("utf-8") + self.listing[path.name] = hashlib.sha256(data).hexdigest() + writes.append((path, data)) + else: + # An emptied provisional file is removed, with its listing. + self.listing.pop(path.name, None) + writes.append((path, None)) + else: + data = dump(values).encode("utf-8") + self.history = store.with_file( + self.history, path.name, hashlib.sha256(data).hexdigest(), _today() + ) + writes.append((path, data)) + for path, data in writes: + if data is None: + path.unlink(missing_ok=True) + print(f"removed {path}") + else: + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_bytes(data) + print(f"wrote {path}") + manifest["golden"]["history"] = self.history + _set_provisional_listing(manifest, self.listing) + store.write_json(manifest_path, manifest) + + def add(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: """Write the computed entries that no golden file pins yet. + An entry of a case that declares a provisional feature goes to the provisional file of + its group; any other to the group's open frozen file. + Args: root: Project root. version: Data context version. groups: Groups to compute. Returns: - Names of the files written. + Names of the files written, relative to the golden folder. """ _require_sound(root, version) path = store.manifest_path(root, version) @@ -143,13 +239,13 @@ def add(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: pinned = store.load_folder(folder) owner = {entry: group for group, values in pinned.items() for entry in values} files = store.group_files(folder) - history = manifest["golden"]["history"] - released = store.released_files(history) + released = store.released_files(manifest["golden"]["history"]) + plan = _Plan(folder, manifest) # Compute and plan every group before writing anything, so a failure leaves the files # and their history as they were. - planned: list[tuple[pathlib.Path, bytes, int]] = [] + counts: dict[pathlib.Path, int] = {} for group in groups: - live = cases.compute_group(group) + live, provisional = cases.compute_group_kinds(group) clash = sorted(e for e in live if owner.get(e, group) != group) if clash: raise SystemExit(f"{group} entries pinned under another group: {clash}") @@ -158,35 +254,32 @@ def add(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: continue # An entry name appears once in a folder: claim it before the next group plans. owner.update(dict.fromkeys(new, group)) - existing = files.get(group, []) - latest = existing[-1] if existing else None - if latest is not None and latest.name not in released: - target = latest - values = {**store.read_json(latest), **new} - else: - number = 1 - if latest is not None: - match = store.FILE_NAME.match(latest.name) - assert match is not None - number = int(match["number"]) + 1 - if number > 99: - raise SystemExit(f"{group} has no free file counter") - target = folder / f"{group}-{number:02d}.json" - values = new - data = dump(values).encode("utf-8") - history = store.with_file( - history, target.name, hashlib.sha256(data).hexdigest(), _today() - ) - planned.append((target, data, len(new))) - if not planned: + frozen_target = _open_frozen_file(folder, files, released, group) + provisional_target = store.provisional_folder(folder) / f"{group}.json" + for entry, value in new.items(): + target = provisional_target if entry in provisional else frozen_target + plan.values(target)[entry] = value + counts[target] = counts.get(target, 0) + 1 + if not plan.contents: print("nothing to add: every computed entry is pinned") return [] - for target, data, count in planned: - _ = target.write_bytes(data) - print(f"added {count} entries to {target}") - manifest["golden"]["history"] = history - store.write_json(path, manifest) - return [target.name for target, _, _ in planned] + for target, count in counts.items(): + print(f"adding {count} entries to {target}") + plan.write(manifest, path) + return [target.relative_to(folder).as_posix() for target in plan.contents] + + +def _misplaced( + pinned_provisional: set[str], live: dict[str, Any], provisional: frozenset[str] +) -> tuple[list[str], list[str]]: + """Entries pinned in the wrong kind of file: (to make provisional, to freeze).""" + to_provisional = sorted( + e for e in live if e in provisional and e not in pinned_provisional + ) + to_frozen = sorted( + e for e in pinned_provisional if e in live and e not in provisional + ) + return to_provisional, to_frozen def check(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: @@ -198,16 +291,19 @@ def check(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: groups: Groups to compute. Returns: - One sentence per difference or rule problem. + One sentence per difference or rule problem, including an entry pinned in a + frozen file whose case is provisional, or the other way round. """ problems = store.tree_problems(root, version) manifest = store.read_json(store.manifest_path(root, version)) + folder = store.golden_folder(root, manifest) try: - pinned = store.load_folder(store.golden_folder(root, manifest)) + pinned = store.load_folder(folder) + pinned_provisional = store.load_provisional(folder) except store.GoldenLayoutError as exc: return [*problems, str(exc)] for group in groups: - live = cases.compute_group(group) + live, provisional = cases.compute_group_kinds(group) pins = pinned.get(group, {}) skipped = cases.unavailable_entries(group) for entry in sorted(set(live) | set(pins)): @@ -221,6 +317,19 @@ def check(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: problems.append( f"{group}:{entry} moved: pinned {pins[entry]!r}, live {live[entry]!r}" ) + here = set(pinned_provisional.get(group, {})) + to_provisional, to_frozen = _misplaced(here, live, provisional) + problems.extend( + f"{group}:{e} is pinned in a frozen file, but its case is provisional: " + "run generate_golden_hashes.py --relocate" + for e in to_provisional + if e in pins + ) + problems.extend( + f"{group}:{e} is pinned in the provisional file, but its case is not " + "provisional: run generate_golden_hashes.py --relocate" + for e in to_frozen + ) return problems @@ -229,7 +338,8 @@ def remove(root: pathlib.Path, version: str, entries: list[str]) -> list[str]: The first step of an approved change to an unreleased value: remove the entry, then ``--add`` writes its new value. An entry in a file a release lists is refused, and - nothing is written unless every entry can be removed. + nothing is written unless every entry can be removed. A provisional file is never + listed by a release; one left without entries is removed. Args: root: Project root. @@ -237,47 +347,99 @@ def remove(root: pathlib.Path, version: str, entries: list[str]) -> list[str]: entries: Entry names to remove. Returns: - Names of the files rewritten. + Names of the files rewritten, relative to the golden folder. """ _require_sound(root, version) path = store.manifest_path(root, version) manifest = store.read_json(path) folder = store.golden_folder(root, manifest) - history = manifest["golden"]["history"] - released = store.released_files(history) - where = { - entry: file - for files in store.group_files(folder).values() - for file in files - for entry in store.read_json(file) - } + released = store.released_files(manifest["golden"]["history"]) + files = [ + *(file for paths in store.group_files(folder).values() for file in paths), + *store.provisional_files(folder).values(), + ] + where = {entry: file for file in files for entry in store.read_json(file)} unknown = sorted(e for e in entries if e not in where) if unknown: raise SystemExit(f"no golden file pins {unknown}; nothing written") - frozen = sorted(e for e in entries if where[e].name in released) + frozen = sorted( + e for e in entries if where[e].parent == folder and where[e].name in released + ) if frozen: raise SystemExit( f"{frozen} are in files a release lists; a released value never changes " "(a change needs a new data context version); nothing written" ) - planned: dict[pathlib.Path, dict[str, Any]] = {} + plan = _Plan(folder, manifest) for entry in dict.fromkeys(entries): - file = where[entry] - values = planned.setdefault(file, store.read_json(file)) - _ = values.pop(entry) - writes: list[tuple[pathlib.Path, bytes]] = [] - for file, values in planned.items(): - data = dump(values).encode("utf-8") - history = store.with_file( - history, file.name, hashlib.sha256(data).hexdigest(), _today() + _ = plan.values(where[entry]).pop(entry) + plan.write(manifest, path) + return [file.relative_to(folder).as_posix() for file in plan.contents] + + +def relocate(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: + """Move pinned entries to the kind of file their case declares; values unchanged. + + An entry of a provisional case pinned in a frozen file moves to the provisional file + of its group: refused, with nothing written, for a file a release lists, since a + frozen value never leaves. An entry pinned in a provisional file whose case no longer + declares a provisional feature (its operator left the group) moves to the open frozen + file of its group, to be frozen by the next release. Every value is copied as it is + pinned; nothing is computed into a file. + + Args: + root: Project root. + version: Data context version. + groups: Groups whose cases are computed to learn their kinds. + + Returns: + Names of the files rewritten, relative to the golden folder. + """ + _require_sound(root, version) + path = store.manifest_path(root, version) + manifest = store.read_json(path) + folder = store.golden_folder(root, manifest) + files = store.group_files(folder) + released = store.released_files(manifest["golden"]["history"]) + pinned_provisional = store.load_provisional(folder) + where = { + entry: file + for paths in files.values() + for file in paths + for entry in store.read_json(file) + } + plan = _Plan(folder, manifest) + for group in groups: + live, provisional = cases.compute_group_kinds(group) + here = pinned_provisional.get(group, {}) + to_provisional, to_frozen = _misplaced(set(here), live, provisional) + to_provisional = [e for e in to_provisional if e in where] + stuck = sorted(e for e in to_provisional if where[e].name in released) + if stuck: + raise SystemExit( + f"{stuck} are provisional but pinned in files a release lists; a " + "released value never leaves its file; nothing written" + ) + provisional_file = store.provisional_folder(folder) / f"{group}.json" + for entry in to_provisional: + value = plan.values(where[entry]).pop(entry) + plan.values(provisional_file)[entry] = value + if to_frozen: + target = _open_frozen_file(folder, files, released, group) + for entry in to_frozen: + plan.values(target)[entry] = plan.values(provisional_file).pop(entry) + if to_provisional or to_frozen: + print( + f"{group}: {len(to_provisional)} entries to the provisional file, " + f"{len(to_frozen)} to a frozen file" + ) + if not plan.contents: + print( + "nothing to relocate: every entry is in the kind of file its case declares" ) - writes.append((file, data)) - for file, data in writes: - _ = file.write_bytes(data) - print(f"removed entries from {file}") - manifest["golden"]["history"] = history - store.write_json(path, manifest) - return [file.name for file, _ in writes] + return [] + plan.write(manifest, path) + return [file.relative_to(folder).as_posix() for file in plan.contents] def release(root: pathlib.Path, version: str, release_text: str) -> str: @@ -360,6 +522,14 @@ def main(argv: list[str] | None = None) -> None: "developer's approval, the first step of changing an unreleased value" ), ) + _ = mode.add_argument( + "--relocate", + action="store_true", + help=( + "move pinned entries, values unchanged, between the frozen files and the " + "provisional files, as their cases declare (ITL-1266)" + ), + ) _ = mode.add_argument( "--stdout", action="store_true", @@ -407,6 +577,9 @@ def main(argv: list[str] | None = None) -> None: } _ = sys.stdout.write(dump(computed)) return + if args.relocate: + _ = relocate(args.root, version, groups) + return if args.check: problems = check(args.root, version, groups) if problems: diff --git a/tests/test_hashing/golden/store.py b/tests/test_hashing/golden/store.py index 0904e748a..db68349c0 100644 --- a/tests/test_hashing/golden/store.py +++ b/tests/test_hashing/golden/store.py @@ -31,6 +31,19 @@ ``scripts/check_golden_tags.py`` compares the record with the tag of that release, when the tag still exists, instead of comparing HEAD with it. +The provisional files (ITL-1266) hold the values of the golden cases that depend on a +provisional operator (``orcapod.operators.PROVISIONAL_OPERATORS``), whose identity is not +frozen. They are ``/provisional/.json``, one per group, and the manifest +lists them, with the sha256 of their bytes, in ``golden.provisional``, outside the history:: + + "golden": {"folder": "v1", "history": [...], + "provisional": {".json": ""}} + +No revision lists a provisional file, so no release freezes one: their values are computed +and checked like every other, and change with the developer's approval of the exact list, +without a new data context. The manifest pin excludes ``golden``, so a change to them never +touches a pin, and ``scripts/check_golden_tags.py`` does not compare them. + This module holds the rules: reading the folder, computing digests, and listing what in a tree breaks the rules. ``generate_golden_hashes.py`` writes the files and the history, ``test_golden_hashes.py`` and ``tests/test_contexts/test_data_context_v1.py`` check them, @@ -65,6 +78,15 @@ FILE_NAME = re.compile(r"^(?P[a-z][a-z0-9_]*)-(?P[0-9]{2})\.json$") """A golden file name: ``-NN.json``.""" +PROVISIONAL = "provisional" +"""The manifest field listing the provisional files, and the subfolder that holds them.""" + +PROVISIONAL_FILE_NAME = re.compile(r"^(?P[a-z][a-z0-9_]*)\.json$") +"""A provisional file name: ``.json``.""" + +GOLDEN_FIELDS = frozenset({"folder", "history", PROVISIONAL}) +"""The fields of a manifest's ``golden`` value.""" + REVISION_FIELDS = ("date", "release", "added", "previous") """The fields of a revision its digest covers, besides the digest itself.""" @@ -115,6 +137,11 @@ def golden_folder(root: Path, manifest: Mapping[str, Any]) -> Path: return root / GOLDEN_ROOT / manifest["golden"]["folder"] +def provisional_folder(folder: Path) -> Path: + """The subfolder of a golden folder that holds its provisional files.""" + return folder / PROVISIONAL + + def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: """Build a JSON object, refusing a key that appears twice (JSON keeps the last).""" out: dict[str, Any] = {} @@ -189,7 +216,11 @@ def reopen_tree(root: Path, version: str, date: str) -> None: revision: dict[str, Any] = { "date": date, "release": None, - "added": {p.name: file_digest(p) for p in sorted(folder.iterdir())}, + "added": { + p.name: file_digest(p) + for paths in group_files(folder).values() + for p in paths + }, "previous": None, } revision["digest"] = revision_digest(revision) @@ -307,14 +338,16 @@ def group_files(folder: Path) -> dict[str, list[Path]]: folder: A golden folder. Returns: - Group name to its files. + Group name to its files; the provisional files are not among them. Raises: GoldenLayoutError: The folder holds something that is not a ``-NN.json`` - file. + file or the ``provisional`` subfolder. """ files: dict[str, list[Path]] = {} for path in sorted(folder.iterdir()): + if path.name == PROVISIONAL and path.is_dir(): + continue match = FILE_NAME.match(path.name) if match is None or not path.is_file(): raise GoldenLayoutError( @@ -325,6 +358,35 @@ def group_files(folder: Path) -> dict[str, list[Path]]: return files +def provisional_files(folder: Path) -> dict[str, Path]: + """The provisional files of a golden folder, by group. + + Args: + folder: A golden folder. + + Returns: + Group name to its provisional file; empty when the folder has no ``provisional`` + subfolder. + + Raises: + GoldenLayoutError: The subfolder holds something that is not a ``.json`` + file. + """ + sub = provisional_folder(folder) + if not sub.is_dir(): + return {} + files: dict[str, Path] = {} + for path in sorted(sub.iterdir()): + match = PROVISIONAL_FILE_NAME.match(path.name) + if match is None or not path.is_file(): + raise GoldenLayoutError( + f"{path} is not a provisional golden file: the provisional folder holds " + "only files named .json" + ) + files[match["group"]] = path + return files + + def next_file_name(name: str) -> str: """The name of a group's next golden file, e.g. ``schema-02.json`` after ``schema-01.json``. @@ -344,7 +406,7 @@ def next_file_name(name: str) -> str: def load_folder(folder: Path) -> dict[str, dict[str, Any]]: - """Read every golden file of a folder. + """Read every golden file of a folder, the provisional files included. Args: folder: A golden folder. @@ -353,28 +415,56 @@ def load_folder(folder: Path) -> dict[str, dict[str, Any]]: Group name to its entries, merged over the group's files. Raises: - GoldenLayoutError: A file name is not ``-NN.json``, a file is not a JSON - object, or an entry name appears in two files of the folder. + GoldenLayoutError: A file name is not ``-NN.json`` (``.json`` in the + provisional folder), a file is not a JSON object, or an entry name appears in + two files of the folder. """ values: dict[str, dict[str, Any]] = {} where: dict[str, str] = {} - for group, paths in group_files(folder).items(): + paths_by_group: dict[str, list[Path]] = { + group: list(paths) for group, paths in group_files(folder).items() + } + for group, path in provisional_files(folder).items(): + paths_by_group.setdefault(group, []).append(path) + for group, paths in paths_by_group.items(): merged = values.setdefault(group, {}) for path in paths: entries = read_json(path) if not isinstance(entries, dict): raise GoldenLayoutError(f"{path} does not hold a JSON object") for entry, value in entries.items(): + name = path.relative_to(folder).as_posix() if entry in where: raise GoldenLayoutError( f"golden entry {entry!r} is pinned twice, in {where[entry]} and " - f"{path.name}; an entry name appears once in a folder" + f"{name}; an entry name appears once in a folder" ) - where[entry] = path.name + where[entry] = name merged[entry] = value return values +def load_provisional(folder: Path) -> dict[str, dict[str, Any]]: + """Read the provisional files of a golden folder. + + Args: + folder: A golden folder. + + Returns: + Group name to the entries of its provisional file. + + Raises: + GoldenLayoutError: A provisional file is misnamed or does not hold a JSON object. + """ + values: dict[str, dict[str, Any]] = {} + for group, path in provisional_files(folder).items(): + entries = read_json(path) + if not isinstance(entries, dict): + raise GoldenLayoutError(f"{path} does not hold a JSON object") + values[group] = entries + return values + + # --------------------------------------------------------------------------- # The history # --------------------------------------------------------------------------- @@ -582,6 +672,8 @@ def folder_problems(history: Sequence[Mapping[str, Any]], folder: Path) -> list[ if not folder.is_dir(): return [*problems, f"the golden folder {folder} does not exist"] for path in sorted(folder.iterdir()): + if path.name == PROVISIONAL and path.is_dir(): + continue if path.name not in listed: problems.append(f"{path.name} is in {folder} but no revision lists it") match = FILE_NAME.match(path.name) @@ -596,6 +688,54 @@ def folder_problems(history: Sequence[Mapping[str, Any]], folder: Path) -> list[ return problems +def provisional_problems(listing: Any, folder: Path) -> list[str]: + """What breaks the agreement between ``golden.provisional`` and the provisional files. + + The listing maps ``.json`` names, for groups of ``cases.GROUPS``, to the sha256 + of their bytes; every listed file exists in the provisional folder with that digest, + and every file there is listed. + + Args: + listing: The ``golden.provisional`` value of a manifest. + folder: The golden folder the manifest names. + + Returns: + One sentence per problem. + """ + if not isinstance(listing, dict) or not all( + isinstance(name, str) and isinstance(digest, str) + for name, digest in listing.items() + ): + return ["golden.provisional is not a mapping of file names to digests"] + problems: list[str] = [] + sub = provisional_folder(folder) + for name, digest in listing.items(): + match = PROVISIONAL_FILE_NAME.match(name) + if match is None or match["group"] not in GROUPS: + problems.append( + f"{name}, listed by golden.provisional, is not a provisional file name " + "(.json of a known group)" + ) + continue + path = sub / name + if not path.is_file(): + problems.append(f"{PROVISIONAL}/{name} is missing") + elif file_digest(path) != digest: + problems.append( + f"{PROVISIONAL}/{name} does not have the sha256 golden.provisional lists: " + "restore the file (git checkout); a value change needs approval, then " + "generate_golden_hashes.py --remove and --add on the restored tree" + ) + if sub.is_dir(): + for path in sorted(sub.iterdir()): + if path.name not in listing: + problems.append( + f"{PROVISIONAL}/{path.name} is in {folder} but golden.provisional " + "does not list it" + ) + return problems + + def withdrawn_releases(pin: Mapping[str, Any]) -> dict[str, Any]: """The records of a pin's withdrawn releases, by version; empty when it has none.""" withdrawn = pin.get("withdrawn", {}) @@ -778,13 +918,20 @@ def tree_problems(root: Path, version: str) -> list[str]: """ manifest = read_json(manifest_path(root, version)) golden = manifest.get("golden") - if not isinstance(golden, dict) or set(golden) != {"folder", "history"}: - return ['the manifest golden field is not {"folder": ..., "history": [...]}'] + if not isinstance(golden, dict) or set(golden) != GOLDEN_FIELDS: + return [ + 'the manifest golden field is not {"folder": ..., "history": [...], ' + '"provisional": {...}}' + ] history = golden["history"] problems = history_problems(history) if problems: return problems - problems = folder_problems(history, golden_folder(root, manifest)) + folder = golden_folder(root, manifest) + problems = [ + *folder_problems(history, folder), + *provisional_problems(golden[PROVISIONAL], folder), + ] pins = read_json(pins_path(root)) if version not in pins: return [*problems, f"{version} is not listed in released_manifests.json"] diff --git a/tests/test_hashing/golden/test_golden_hashes.py b/tests/test_hashing/golden/test_golden_hashes.py index 9c6c09c55..97056145d 100644 --- a/tests/test_hashing/golden/test_golden_hashes.py +++ b/tests/test_hashing/golden/test_golden_hashes.py @@ -47,18 +47,25 @@ FOLDER = store.golden_folder(store.PROJECT_ROOT, MANIFEST) -def _load_golden() -> tuple[dict[str, dict[str, Any]], str | None]: - """The pinned entries of every group, and the error that stopped reading them.""" +def _load_golden() -> tuple[dict[str, dict[str, Any]], dict[str, set[str]], str | None]: + """The pinned entries of every group (frozen and provisional files), the entries of + the provisional files, and the error that stopped reading them.""" try: loaded = store.load_folder(FOLDER) + provisional = store.load_provisional(FOLDER) except (store.GoldenLayoutError, OSError) as exc: - loaded, error = {}, str(exc) + loaded, provisional, error = {}, {}, str(exc) else: error = None - return {group: loaded.get(group, {}) for group in {*cases.GROUPS, *loaded}}, error + groups = {*cases.GROUPS, *loaded} + return ( + {group: loaded.get(group, {}) for group in groups}, + {group: set(provisional.get(group, {})) for group in groups}, + error, + ) -GOLDEN, LOAD_ERROR = _load_golden() +GOLDEN, PINNED_PROVISIONAL, LOAD_ERROR = _load_golden() GOLDEN_ENTRIES = [ @@ -81,9 +88,19 @@ def _moved_message(group: str, entry: str, expected: Any, actual: Any) -> str: @pytest.fixture(scope="module") -def live() -> dict[str, dict[str, Any]]: +def live_with_kinds() -> dict[str, tuple[dict[str, Any], frozenset[str]]]: + """Every golden group, computed in this process, with its provisional entries.""" + return { + group: cases.compute_group_kinds(group) for group in cases.available_groups() + } + + +@pytest.fixture(scope="module") +def live( + live_with_kinds: dict[str, tuple[dict[str, Any], frozenset[str]]], +) -> dict[str, dict[str, Any]]: """Every golden group, computed in this process.""" - return cases.compute_all() + return {group: values for group, (values, _) in live_with_kinds.items()} def test_golden_folder_reads() -> None: @@ -126,6 +143,28 @@ def test_no_unpinned_entries(live: dict[str, dict[str, Any]], group: str) -> Non ) +@pytest.mark.parametrize("group", GROUP_PARAMS) +def test_entries_are_in_the_files_their_cases_declare( + live_with_kinds: dict[str, tuple[dict[str, Any], frozenset[str]]], group: str +) -> None: + """A provisional case is pinned in the provisional file, any other in a frozen one. + + A value that depends on a provisional operator must never be in a file a release + freezes (ITL-1266); ``cases.compute_group_kinds`` already refuses a case that uses one + without declaring it. + """ + values, provisional = live_with_kinds[group] + here = PINNED_PROVISIONAL[group] + frozen = sorted(e for e in provisional if e in GOLDEN[group] and e not in here) + thawed = sorted(e for e in here if e in values and e not in provisional) + assert not frozen and not thawed, ( + f"{group} entries in the wrong kind of golden file; run " + "generate_golden_hashes.py --relocate (an entry in a released file cannot " + f"leave it). Provisional, in a frozen file: {frozen[:10]}. Not provisional, in " + f"the provisional file: {thawed[:10]}" + ) + + # --------------------------------------------------------------------------- # No golden entry is skipped or expected to fail, but for a missing optional package # --------------------------------------------------------------------------- @@ -599,10 +638,12 @@ def test_tensor_and_ndarray_pins_differ(name: str) -> None: store.copy_tree(store.PROJECT_ROOT, root, version) manifest = store.manifest_path(root, version) folder = store.golden_folder(root, store.read_json(manifest)) -before = {p.name: p.read_bytes() for p in [manifest, *sorted(folder.iterdir())]} +files = [manifest, *sorted(p for p in folder.rglob("*") if p.is_file())] +before = {str(p): p.read_bytes() for p in files} main(["--root", str(root), "--group", "torch"]) main(["--root", str(root), "--check", "--group", "torch"]) -after = {p.name: p.read_bytes() for p in [manifest, *sorted(folder.iterdir())]} +files = [manifest, *sorted(p for p in folder.rglob("*") if p.is_file())] +after = {str(p): p.read_bytes() for p in files} print("pins kept:", before == after) """ diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index a3e3292ac..4b2105c78 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -361,14 +361,14 @@ def test_add_changes_nothing_when_a_later_group_fails( """Every group is computed before anything is written: files and history stay in step.""" _ = _drop_entries(tree, NEWEST_SCHEMA, 1) before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() - compute = cases.compute_group + compute = cases.compute_group_kinds - def failing(group: str, **kwargs: Any) -> dict[str, Any]: + def failing(group: str, **kwargs: Any) -> tuple[dict[str, Any], frozenset[str]]: if group == "semantic": raise RuntimeError("semantic failed") return compute(group, **kwargs) - monkeypatch.setattr(cases, "compute_group", failing) + monkeypatch.setattr(cases, "compute_group_kinds", failing) with pytest.raises(RuntimeError, match="semantic failed"): main(["--root", str(tree), "--group", "schema", "--group", "semantic"]) after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() @@ -381,12 +381,15 @@ def test_add_refuses_one_new_entry_name_in_two_groups( ) -> None: """Two groups that compute the same new entry name in one run write nothing.""" before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() - compute = cases.compute_group + compute = cases.compute_group_kinds - def with_shared_entry(group: str, **kwargs: Any) -> dict[str, Any]: - return {**compute(group, **kwargs), "shared.new_entry": group} + def with_shared_entry( + group: str, **kwargs: Any + ) -> tuple[dict[str, Any], frozenset[str]]: + values, provisional = compute(group, **kwargs) + return {**values, "shared.new_entry": group}, provisional - monkeypatch.setattr(cases, "compute_group", with_shared_entry) + monkeypatch.setattr(cases, "compute_group_kinds", with_shared_entry) with pytest.raises(SystemExit, match="shared.new_entry"): main(["--root", str(tree), "--group", "schema", "--group", "arrow"]) after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() @@ -738,3 +741,343 @@ def test_a_withdrawn_release_left_in_the_history_is_reported(tree: Path) -> None store.write_json(store.pins_path(tree), pins) problems = store.tree_problems(tree, VERSION) assert any("is a release revision of the history" in p for p in problems), problems + + +# --------------------------------------------------------------------------- +# Provisional files: outside the history, never frozen (ITL-1266) +# --------------------------------------------------------------------------- + +NEWEST_SEMANTIC = _REPOSITORY_FILES["semantic"][-1].name +NEXT_SEMANTIC = store.next_file_name(NEWEST_SEMANTIC) + + +def _provisional(root: Path) -> Path: + return store.provisional_folder(_folder(root)) + + +def _listing(root: Path) -> dict[str, str]: + return _manifest(root)["golden"][store.PROVISIONAL] + + +def _everything(root: Path) -> dict[str, bytes]: + """Every file of the golden folder (provisional files included) and the manifest.""" + folder = _folder(root) + files = { + path.relative_to(folder).as_posix(): path.read_bytes() + for path in sorted(folder.rglob("*")) + if path.is_file() + } + files["manifest"] = store.manifest_path(root, VERSION).read_bytes() + return files + + +def _set_listing(root: Path, listing: dict[str, str]) -> None: + manifest = _manifest(root) + manifest["golden"][store.PROVISIONAL] = listing + store.write_json(store.manifest_path(root, VERSION), manifest) + + +def _freeze_provisional(root: Path, group: str, frozen_file: str) -> dict[str, Any]: + """Put a group's provisional entries into a frozen file, as before ITL-1266.""" + path = _provisional(root) / f"{group}.json" + entries = store.read_json(path) + path.unlink() + listing = dict(_listing(root)) + del listing[f"{group}.json"] + _set_listing(root, listing) + _record( + root, frozen_file, {**store.read_json(_folder(root) / frozen_file), **entries} + ) + return entries + + +def _without_declarations(group: str) -> list[tuple[str, Any, tuple[str, ...]]]: + """The cases of a group as if no provisional operator were left in it.""" + return [(name, fn, ()) for name, fn, _ in cases._CASES[group]] + + +def test_the_repository_pins_provisional_entries_outside_the_history() -> None: + """The provisional files are listed in golden.provisional, never in a revision.""" + manifest = store.read_json(store.manifest_path(store.PROJECT_ROOT, VERSION)) + listing = manifest["golden"][store.PROVISIONAL] + assert listing + revised = {name for r in manifest["golden"]["history"] for name in r["added"]} + assert not revised & set(listing) + + +def test_relocate_moves_provisional_entries_out_of_a_frozen_file(tree: Path) -> None: + """The ITL-1266 move on a copy: values unchanged, the frozen file only loses them.""" + original = _everything(tree) + entries = _freeze_provisional(tree, "semantic", NEWEST_SEMANTIC) + assert entries + main(["--root", str(tree), "--relocate", "--group", "semantic"]) + after = _everything(tree) + assert {k: v for k, v in after.items() if k != "manifest"} == { + k: v for k, v in original.items() if k != "manifest" + } + assert ( + _listing(tree) + == store.read_json(store.manifest_path(store.PROJECT_ROOT, VERSION))["golden"][ + store.PROVISIONAL + ] + ) + assert len(_history(tree)) == 1 + assert store.tree_problems(tree, VERSION) == [] + + +def test_relocate_writes_nothing_when_every_entry_is_in_place(tree: Path) -> None: + before = _everything(tree) + main(["--root", str(tree), "--relocate", "--group", "semantic"]) + assert _everything(tree) == before + + +def test_relocate_refuses_a_provisional_entry_in_a_released_file(tree: Path) -> None: + """A released value never leaves its file, not even to become provisional.""" + _ = _freeze_provisional(tree, "semantic", NEWEST_SEMANTIC) + main(["--root", str(tree), "--release", "0.3.0"]) + before = _everything(tree) + with pytest.raises(SystemExit, match="release lists"): + main(["--root", str(tree), "--relocate", "--group", "semantic"]) + assert _everything(tree) == before + + +@pytest.mark.parametrize("released", [False, True]) +def test_relocate_freezes_the_entries_of_an_operator_that_left_the_group( + tree: Path, monkeypatch: pytest.MonkeyPatch, released: bool +) -> None: + """Graduation: the entries go to the open frozen file, the provisional file goes.""" + entries = store.read_json(_provisional(tree) / "semantic.json") + if released: + main(["--root", str(tree), "--release", "0.3.0"]) + newest = store.read_json(_folder(tree) / NEWEST_SEMANTIC) + monkeypatch.setitem(cases._CASES, "semantic", _without_declarations("semantic")) + main(["--root", str(tree), "--relocate", "--group", "semantic"]) + assert not (_provisional(tree) / "semantic.json").exists() + assert "semantic.json" not in _listing(tree) + if released: + assert store.read_json(_folder(tree) / NEWEST_SEMANTIC) == newest + assert store.read_json(_folder(tree) / NEXT_SEMANTIC) == entries + assert list(_history(tree)[-1]["added"]) == [NEXT_SEMANTIC] + else: + assert store.read_json(_folder(tree) / NEWEST_SEMANTIC) == { + **newest, + **entries, + } + assert store.tree_problems(tree, VERSION) == [] + + +def test_check_reports_a_provisional_entry_in_a_frozen_file(tree: Path) -> None: + entries = _freeze_provisional(tree, "semantic", NEWEST_SEMANTIC) + entry = sorted(entries)[0] + with pytest.raises( + SystemExit, + match=re.escape(f"semantic:{entry} is pinned in a frozen file, but its case"), + ): + main(["--root", str(tree), "--check", "--group", "semantic"]) + + +def test_check_reports_an_entry_in_the_provisional_file_whose_case_is_not( + tree: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + entry = sorted(store.read_json(_provisional(tree) / "semantic.json"))[0] + monkeypatch.setitem(cases._CASES, "semantic", _without_declarations("semantic")) + with pytest.raises( + SystemExit, + match=re.escape(f"semantic:{entry} is pinned in the provisional file"), + ): + main(["--root", str(tree), "--check", "--group", "semantic"]) + + +@pytest.mark.parametrize("order", [("semantic", "pipeline"), ("pipeline", "semantic")]) +def test_add_writes_provisional_entries_to_their_provisional_files( + tree: Path, order: tuple[str, str] +) -> None: + """Two groups, in either order: each provisional file comes back byte for byte.""" + original = _everything(tree) + for name in ("semantic.json", "pipeline.json"): + (_provisional(tree) / name).unlink() + _set_listing(tree, {}) + args = ["--root", str(tree)] + for group in order: + args += ["--group", group] + main(args) + assert _everything(tree) == original + assert store.tree_problems(tree, VERSION) == [] + + +def test_a_release_never_lists_a_provisional_file(tree: Path) -> None: + """After a release a provisional value still changes by --remove and --add.""" + listing = dict(_listing(tree)) + main(["--root", str(tree), "--release", "0.3.0"]) + assert _history(tree)[-1]["added"] == {} + assert _listing(tree) == listing + released = store.released_files(_history(tree)) + assert not released & set(listing) + path = _provisional(tree) / "semantic.json" + values = store.read_json(path) + entry = sorted(values)[0] + main(["--root", str(tree), "--remove", entry]) + assert entry not in store.read_json(path) + assert store.tree_problems(tree, VERSION) == [] + main(["--root", str(tree), "--group", "semantic"]) + assert store.read_json(path) == values + assert _listing(tree) == listing + assert store.tree_problems(tree, VERSION) == [] + + +def test_removing_every_entry_of_a_provisional_file_removes_it(tree: Path) -> None: + path = _provisional(tree) / "semantic.json" + main(["--root", str(tree), *(f"--remove={e}" for e in store.read_json(path))]) + assert not path.exists() + assert "semantic.json" not in _listing(tree) + assert store.tree_problems(tree, VERSION) == [] + + +def test_an_unlisted_provisional_file_is_reported(tree: Path) -> None: + _write(_provisional(tree), "schema.json", {"x.y": "z"}) + assert any( + "golden.provisional does not list it" in p + for p in store.tree_problems(tree, VERSION) + ) + + +def test_an_edited_provisional_file_is_reported(tree: Path) -> None: + path = _provisional(tree) / "semantic.json" + _ = path.write_text(path.read_text() + " ") + assert any( + "does not have the sha256 golden.provisional lists" in p + for p in store.tree_problems(tree, VERSION) + ) + + +def test_a_missing_listed_provisional_file_is_reported(tree: Path) -> None: + (_provisional(tree) / "semantic.json").unlink() + assert any( + "provisional/semantic.json is missing" in p + for p in store.tree_problems(tree, VERSION) + ) + + +@pytest.mark.parametrize("name", ["unknown.json", "semantic-01.json", "Semantic.json"]) +def test_a_misnamed_provisional_listing_is_reported(tree: Path, name: str) -> None: + _set_listing(tree, {**_listing(tree), name: "0" * 64}) + assert any( + f"{name}, listed by golden.provisional, is not a provisional file name" in p + for p in store.tree_problems(tree, VERSION) + ) + + +def test_a_misnamed_file_in_the_provisional_folder_is_an_error(tree: Path) -> None: + _write(_provisional(tree), "semantic-01.json", {}) + with pytest.raises(store.GoldenLayoutError, match=".json"): + _ = store.load_folder(_folder(tree)) + + +def test_an_entry_in_a_frozen_and_a_provisional_file_is_reported(tree: Path) -> None: + entry, value = next( + iter(store.read_json(_provisional(tree) / "semantic.json").items()) + ) + _record( + tree, + NEWEST_SEMANTIC, + {**store.read_json(_folder(tree) / NEWEST_SEMANTIC), entry: value}, + ) + assert any("is pinned twice" in p for p in store.tree_problems(tree, VERSION)) + + +def test_a_manifest_without_the_provisional_listing_is_reported(tree: Path) -> None: + manifest = _manifest(tree) + del manifest["golden"][store.PROVISIONAL] + store.write_json(store.manifest_path(tree, VERSION), manifest) + assert store.tree_problems(tree, VERSION) == [ + 'the manifest golden field is not {"folder": ..., "history": [...], ' + '"provisional": {...}}' + ] + + +# --------------------------------------------------------------------------- +# Case declarations: a case using a provisional operator must declare it (ITL-1266) +# --------------------------------------------------------------------------- + +_A, _B = "ProvisionalA", "ProvisionalB" + + +def _using(*names: str) -> Any: + def fn() -> dict[str, Any]: + from orcapod.core.operators.provisional import warn_provisional_operator + + for name in names: + warn_provisional_operator(name) + return {"value": list(names)} + + return fn + + +@pytest.fixture +def two_provisional(monkeypatch: pytest.MonkeyPatch) -> None: + """Two provisional operators, so declarations can name several.""" + monkeypatch.setattr( + cases, "declarable_provisional_names", lambda: frozenset({_A, _B}) + ) + + +@pytest.mark.filterwarnings("ignore::orcapod.errors.ProvisionalHashWarning") +@pytest.mark.parametrize("used", [(_A, _B), (_B, _A)]) +@pytest.mark.parametrize("declared", [(_A, _B), (_B, _A)]) +def test_a_case_declaring_every_provisional_operator_it_uses_is_provisional( + monkeypatch: pytest.MonkeyPatch, + two_provisional: None, + used: tuple[str, ...], + declared: tuple[str, ...], +) -> None: + """Neither the order of use nor of declaration changes what is provisional.""" + monkeypatch.setitem( + cases._CASES, + "semantic", + [("both", _using(*used), declared), ("plain", _using(), ())], + ) + values, provisional = cases.compute_group_kinds("semantic") + assert values == {"both.value": list(used), "plain.value": []} + assert provisional == frozenset({"both.value"}) + + +@pytest.mark.filterwarnings("ignore::orcapod.errors.ProvisionalHashWarning") +@pytest.mark.parametrize( + ("declared", "missing"), + [((), [_A, _B]), ((_A,), [_B]), ((_B,), [_A])], +) +def test_a_case_using_an_undeclared_provisional_operator_is_refused( + monkeypatch: pytest.MonkeyPatch, + two_provisional: None, + declared: tuple[str, ...], + missing: list[str], +) -> None: + monkeypatch.setitem(cases._CASES, "semantic", [("both", _using(_B, _A), declared)]) + with pytest.raises(ValueError, match=re.escape(f"uses the provisional {missing}")): + _ = cases.compute_group_kinds("semantic") + + +def test_a_case_declaring_an_unknown_provisional_name_is_refused( + monkeypatch: pytest.MonkeyPatch, two_provisional: None +) -> None: + monkeypatch.setitem( + cases._CASES, "semantic", [("case", _using(), (_A, "PolarsFilter"))] + ) + with pytest.raises(ValueError, match=re.escape("declares ['PolarsFilter']")): + _ = cases.compute_group_kinds("semantic") + + +def test_the_repository_cases_declare_exactly_the_polars_filter_cases() -> None: + """The cases that depend on PolarsFilter, measured for ITL-1266, and no other.""" + declared = { + (group, name): names + for group, registered in cases._CASES.items() + for name, _, names in registered + if names + } + assert declared == { + ("semantic", "polars_expr"): ("PolarsFilter",), + ("pipeline", "polars_filter"): ("PolarsFilter",), + ("pipeline", "polars_filter_expr"): ("PolarsFilter",), + ("pipeline", "polars_filter_expr_job"): ("PolarsFilter",), + }