From 9eb790913a96682c48077ceb6620c6d119cb2d0b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:02:20 -0800 Subject: [PATCH 01/16] feat(golden): protect released golden hash files in CI (ITL-946) - Move the six golden files byte for byte to tests/test_hashing/golden/data/v1/-01.json; the golden test reads every -NN.json of the folder the manifest names. - v1.json golden = {folder, history}: an append-only chain of revisions listing each file with the sha256 of its bytes; one open revision for now. released_manifests.json pins {"manifest": null, "releases": {}} until the release step. - tests/test_hashing/golden/store.py holds the rules; test_data_context_v1.py checks files, chain, pins, released_in and the final-release line. - generate_golden_hashes.py: --add (default) writes only unpinned entries, never a released file; --check reports drift; --release X.Y.Z appends the release revision and the pins. - scripts/check_golden_tags.py, a Run Tests step: HEAD keeps every file, revision and pin a release tag of the current data context shipped; fails in CI without git, history or tags. - .gitattributes -text on the golden data; the golden test refuses skip/xfail except the optional-package rule; an entry name pinned twice in a folder is an error. - Rules in CLAUDE.md, .zed/rules, RELEASING.md (tag protection precondition) and docs/reference/golden-hashes.md; no exemption for a logical type's own change. No golden value changes: the files are pure renames. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 2 + .github/workflows/run-tests.yml | 5 + .zed/rules | 45 +- CLAUDE.md | 45 +- RELEASING.md | 44 +- changelog.d/ITL-944.md | 16 +- changelog.d/ITL-946.md | 14 + design_issues/ITL-944-DC1.md | 14 +- docs/concepts/logical-type-identifiers.md | 4 + docs/reference/golden-hashes.md | 357 +++++++++++ docs/reference/hashing.md | 9 +- mkdocs.yml | 1 + scripts/check_golden_tags.py | 235 ++++++++ .../contexts/data/schemas/context_schema.json | 60 +- src/orcapod/contexts/data/v1.json | 20 +- src/orcapod/contexts/released_manifests.json | 5 +- tests/test_check_golden_tags.py | 215 +++++++ tests/test_contexts/test_data_context_v1.py | 127 ++-- tests/test_hashing/golden/README.md | 26 + .../data/{arrow.json => v1/arrow-01.json} | 0 .../{datagram.json => v1/datagram-01.json} | 0 .../{pipeline.json => v1/pipeline-01.json} | 0 .../data/{schema.json => v1/schema-01.json} | 0 .../{semantic.json => v1/semantic-01.json} | 0 .../data/{torch.json => v1/torch-01.json} | 0 .../golden/generate_golden_hashes.py | 303 ++++++++-- tests/test_hashing/golden/store.py | 566 ++++++++++++++++++ .../test_hashing/golden/test_golden_hashes.py | 185 ++++-- .../test_hashing/golden/test_golden_store.py | 410 +++++++++++++ 29 files changed, 2485 insertions(+), 223 deletions(-) create mode 100644 .gitattributes create mode 100644 changelog.d/ITL-946.md create mode 100644 docs/reference/golden-hashes.md create mode 100644 scripts/check_golden_tags.py create mode 100644 tests/test_check_golden_tags.py create mode 100644 tests/test_hashing/golden/README.md rename tests/test_hashing/golden/data/{arrow.json => v1/arrow-01.json} (100%) rename tests/test_hashing/golden/data/{datagram.json => v1/datagram-01.json} (100%) rename tests/test_hashing/golden/data/{pipeline.json => v1/pipeline-01.json} (100%) rename tests/test_hashing/golden/data/{schema.json => v1/schema-01.json} (100%) rename tests/test_hashing/golden/data/{semantic.json => v1/semantic-01.json} (100%) rename tests/test_hashing/golden/data/{torch.json => v1/torch-01.json} (100%) create mode 100644 tests/test_hashing/golden/store.py create mode 100644 tests/test_hashing/golden/test_golden_store.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..4c240a5d6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Golden hash files are hashed byte for byte: never convert their line endings (ITL-946). +tests/test_hashing/golden/data/** -text diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index dc2c7f83a..379ef7e3e 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -33,6 +33,11 @@ jobs: - name: Install dependencies run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }} + - name: Check golden files against release tags + # Every golden file and pin a release tag of the current data context shipped must + # be unchanged in HEAD (ITL-946). Fails when git, the history or the tags are missing. + run: uv run --python ${{ matrix.python-version }} python scripts/check_golden_tags.py + - name: Run tests run: uv run --python ${{ matrix.python-version }} pytest -v -rsxX --durations=20 -m "not postgres" --cov=src --cov-report=term-missing --cov-report=xml diff --git a/.zed/rules b/.zed/rules index f8d0bb8ea..5835c75cb 100644 --- a/.zed/rules +++ b/.zed/rules @@ -287,7 +287,9 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension differently. It is not a package version. Built-in types use `version_metadata(N)` (`{"version":N}`); a new built-in type must write one, never empty metadata. - A changed identifier is a schema change: it moves stored paths and golden values of that - type. Treat it like any other hash change (developer approval, migration assessment). + type. Treat it like any other hash change (developer approval, migration assessment). There + is no exemption for a logical type's own change (developer, November 15, 2026, ITL-946): if + a golden value it moves has shipped in a release, it needs a new data context version. - The native containers `list[T]` / `set[T]` of a logical type (`ListLogicalType`) have no version of their own, by design: never add a `"version"` key to their metadata. Their identifier is the container kind plus the element's identifier; the type hash treats them @@ -386,14 +388,33 @@ A class a user defines is one of two kinds (developer, November 12, 2026, ITL-97 ## Golden hash values Hash values must stay identical across releases (developer, November 10, 2026). Every hash -Orcapod stores or embeds in a name is pinned in `tests/test_hashing/golden/data/*.json` and -checked by `tests/test_hashing/golden/test_golden_hashes.py`. +Orcapod stores or embeds in a name is pinned in the golden folder of the current data context, +`tests/test_hashing/golden/data//-NN.json`, and checked by +`tests/test_hashing/golden/test_golden_hashes.py`. The mechanism is +`docs/reference/golden-hashes.md` (developer, November 15, 2026, ITL-946). - A failing golden entry means a hash moved. Treat it as a bug in the change, not a stale - file. Never regenerate the golden files to make a test pass. -- Regenerate (`uv run python tests/test_hashing/golden/generate_golden_hashes.py`) only to add - entries (check the diff adds lines and changes none), or for a hash change the developer has - approved together with a migration assessment. + file. Never edit or regenerate a golden file to make a test pass. +- The unit of immutability is the golden file. The manifest's `golden.history` + (`src/orcapod/contexts/data/.json`) lists every file with the sha256 of its bytes; + a release appends a revision with `added: {}`, pinned in + `src/orcapod/contexts/released_manifests.json`. A file listed by a released revision never + changes and is never removed or renamed. +- Adding golden values is allowed in any release: + `uv run python tests/test_hashing/golden/generate_golden_hashes.py` writes only entries no + file pins yet (into a new `-NN.json` once the group's newest file is released) and + records them in the open revision. Check the diff adds and changes no value. `--check` + reports drift without writing. +- Any change to a released golden value needs a new data context version, with no exemption + (also not a logical type's own identifier change). A new data context version ships only in + a minor release (before 1.0) or a major one, never in a patch. When it starts, the old + context's golden files and code move to `orcapod-sanctuary`; this repository carries only + the current context's golden folder. +- Before a release, a value no release lists changes only with the developer's approval and + a migration assessment: remove the entry from its file and run the generator. +- The release pull request runs `generate_golden_hashes.py --release X.Y.Z`. CI checks HEAD + against every release tag (`scripts/check_golden_tags.py`). Never mark a golden entry + `skip` or `xfail`; the only skip is the optional-package rule (`torch`). - Never move or rename the fixtures in `tests/test_hashing/golden/cases.py`: function pods, dataclasses and pydantic models hash with their module path. @@ -685,10 +706,12 @@ generation of hashing; rows of the current data context are the current entries. proceed (developer, November 11, 2026). Operator caches and cached sources have no stable name; PipelineJob.run() reports one total for them. Every database class implements list_table_paths. -- A released data context manifest (contexts/data/.json) is frozen: its sha256 is - pinned in contexts/released_manifests.json. A change that moves an existing golden hash - value for a reason other than a logical type's own change needs a new data context version - with its own manifest file, never an edit of a released one. +- A released data context manifest (contexts/data/.json) is frozen: the digest of + its canonical JSON without golden is pinned in contexts/released_manifests.json + (manifest), with one releases line per library release (ITL-946). Only its golden + history grows, by appending. A change that moves a released golden hash value, for any + reason (a logical type's own change included), needs a new data context version with its + own manifest file, never an edit of a released one. - In memory the data context column is still _context_key (ColumnConfig context). - A writer refuses an input of another data context than its own (contexts/stored.require_same_data_context, InputValidationError naming both keys) before diff --git a/CLAUDE.md b/CLAUDE.md index d3006acd6..2e87cfa00 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -298,7 +298,9 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension differently. It is not a package version. Built-in types use `version_metadata(N)` (`{"version":N}`); a new built-in type must write one, never empty metadata. - A changed identifier is a schema change: it moves stored paths and golden values of that - type. Treat it like any other hash change (developer approval, migration assessment). + type. Treat it like any other hash change (developer approval, migration assessment). There + is no exemption for a logical type's own change (developer, November 15, 2026, ITL-946): if + a golden value it moves has shipped in a release, it needs a new data context version. - The native containers `list[T]` / `set[T]` of a logical type (`ListLogicalType`) have no version of their own, by design: never add a `"version"` key to their metadata. Their identifier is the container kind plus the element's identifier; the type hash treats them @@ -397,14 +399,33 @@ A class a user defines is one of two kinds (developer, November 12, 2026, ITL-97 ## Golden hash values Hash values must stay identical across releases (developer, November 10, 2026). Every hash -Orcapod stores or embeds in a name is pinned in `tests/test_hashing/golden/data/*.json` and -checked by `tests/test_hashing/golden/test_golden_hashes.py`. +Orcapod stores or embeds in a name is pinned in the golden folder of the current data context, +`tests/test_hashing/golden/data//-NN.json`, and checked by +`tests/test_hashing/golden/test_golden_hashes.py`. The mechanism is +`docs/reference/golden-hashes.md` (developer, November 15, 2026, ITL-946). - A failing golden entry means a hash moved. Treat it as a bug in the change, not a stale - file. Never regenerate the golden files to make a test pass. -- Regenerate (`uv run python tests/test_hashing/golden/generate_golden_hashes.py`) only to add - entries (check the diff adds lines and changes none), or for a hash change the developer has - approved together with a migration assessment. + file. Never edit or regenerate a golden file to make a test pass. +- The unit of immutability is the golden file. The manifest's `golden.history` + (`src/orcapod/contexts/data/.json`) lists every file with the sha256 of its bytes; + a release appends a revision with `added: {}`, pinned in + `src/orcapod/contexts/released_manifests.json`. A file listed by a released revision never + changes and is never removed or renamed. +- Adding golden values is allowed in any release: + `uv run python tests/test_hashing/golden/generate_golden_hashes.py` writes only entries no + file pins yet (into a new `-NN.json` once the group's newest file is released) and + records them in the open revision. Check the diff adds and changes no value. `--check` + reports drift without writing. +- Any change to a released golden value needs a new data context version, with no exemption + (also not a logical type's own identifier change). A new data context version ships only in + a minor release (before 1.0) or a major one, never in a patch. When it starts, the old + context's golden files and code move to `orcapod-sanctuary`; this repository carries only + the current context's golden folder. +- Before a release, a value no release lists changes only with the developer's approval and + a migration assessment: remove the entry from its file and run the generator. +- The release pull request runs `generate_golden_hashes.py --release X.Y.Z`. CI checks HEAD + against every release tag (`scripts/check_golden_tags.py`). Never mark a golden entry + `skip` or `xfail`; the only skip is the optional-package rule (`torch`). - Never move or rename the fixtures in `tests/test_hashing/golden/cases.py`: function pods, dataclasses and pydantic models hash with their module path. @@ -713,10 +734,12 @@ current data context are the current entries. proceed (developer, November 11, 2026). Operator caches and cached sources have no stable name; `PipelineJob.run()` reports one total for them. Every database class implements `list_table_paths`. -- A released data context manifest (`contexts/data/.json`) is frozen: its sha256 is - pinned in `contexts/released_manifests.json`. A change that moves an existing golden hash - value for a reason other than a logical type's own change needs a new data context version - with its own manifest file, never an edit of a released one. +- A released data context manifest (`contexts/data/.json`) is frozen: the digest of + its canonical JSON without `golden` is pinned in `contexts/released_manifests.json` + (`manifest`), with one `releases` line per library release (ITL-946). Only its `golden` + history grows, by appending. A change that moves a released golden hash value, for any + reason (a logical type's own change included), needs a new data context version with its + own manifest file, never an edit of a released one. - In memory the data context column is still `_context_key` (`ColumnConfig(context=True)`). - A writer refuses an input of another data context than its own (`contexts/stored.require_same_data_context`, `InputValidationError` naming both keys) diff --git a/RELEASING.md b/RELEASING.md index 3cad656d7..03dba449f 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -54,10 +54,23 @@ Do these on `metamorphic-labs` `main` before anything is pushed: The GitHub Release body is generated from PR titles; `CHANGELOG.md` is the curated record. -2. **Check that `main` is green** — the `Run Tests` workflow on the commit you are +2. **Record the release in the golden history**, in the same release pull request, as its + last change (see *Golden hash values* below): + + ``` + uv run python tests/test_hashing/golden/generate_golden_hashes.py --release X.Y.Z + ``` + + It appends the release revision to the `golden` history of the current data context's + manifest, writes its digest under `releases` in + `src/orcapod/contexts/released_manifests.json`, and at the data context's first release + sets `released_in` and the `manifest` pin. It computes no hash and refuses a version + already released. Commit both files with the release pull request; the release is tagged + on a commit that contains them. +3. **Check that `main` is green** — the `Run Tests` workflow on the commit you are releasing. The Release workflow re-runs the unit tests on Python 3.11, 3.12 and 3.13 and the license check, but not the PostgreSQL or SpiralDB integration jobs. -3. **Check the version is free** — the tag must not exist on origin, and the version must +4. **Check the version is free** — the tag must not exist on origin, and the version must not exist on PyPI or TestPyPI; neither index allows a version to be uploaded twice, even after it is deleted. @@ -71,12 +84,30 @@ All of these hold before the developer is asked for the go: 1. **The full suite is green**: `uv run pytest tests/` passes on the commit to be pushed. 2. **The release pull request is merged** on `metamorphic-labs` (`CHANGELOG.md` folded, section dated, `changelog.d/` fragments deleted). -3. **The data context freeze is done**: every released manifest is pinned in - `contexts/released_manifests.json` and the golden hash tests pass unchanged. +3. **The golden release step is done**: `contexts/released_manifests.json` has the current + data context's `releases` line for this version and a pinned `manifest`, and the golden + hash tests and the data context tests pass unchanged. 4. **`main` is green** on `metamorphic-labs`, and `git status` is clean. 5. **The version is free**: the tag does not exist on `nauticalab`, and the version is on neither PyPI nor TestPyPI. +## Golden hash values + +Developer decisions, November 15, 2026 (ITL-946); the mechanism and what each check protects +against are in `docs/reference/golden-hashes.md`. + +- A golden file that a release listed never changes, and is never removed or renamed. +- New golden values can be added in any release, a patch included. +- Any change to a released golden value, a logical type's own identifier change included, + needs a new data context version. A new data context version ships only in a minor release + (before 1.0) or a major one, never in a patch. +- When a new data context starts, the old context's golden files and code move to + `orcapod-sanctuary`; this repository carries only the current context's golden folder. +- Release tags (`v*`) are protected on both `metamorphic-labs/orcapod-python` and + `nauticalab/orcapod-python`. This is a precondition of the CI tag check + (`scripts/check_golden_tags.py`), which compares HEAD with every release tag of the current + data context. The developer sets the protection; no agent changes repository settings. + ## Cutting a Release Releases are triggered from the GitHub Actions UI — no manual `git tag` step required. @@ -142,7 +173,10 @@ environments: that name `metamorphic-labs`, if any. 3. Protect `main` on `nauticalab` (no force push, no deletion). It is written only by the fast-forward push above. -4. Optional: set `CODECOV_TOKEN` there. The Codecov, SpiralDB and Linear jobs run only on +4. Protect release tags (`v*`) on both `metamorphic-labs/orcapod-python` and + `nauticalab/orcapod-python` (no update, no deletion), so the golden tag check has a fixed + anchor (ITL-946). +5. Optional: set `CODECOV_TOKEN` there. The Codecov, SpiralDB and Linear jobs run only on `metamorphic-labs` and are skipped on `nauticalab`. ## Pre-releases diff --git a/changelog.d/ITL-944.md b/changelog.d/ITL-944.md index 8b801eec1..58ed4bbb6 100644 --- a/changelog.d/ITL-944.md +++ b/changelog.d/ITL-944.md @@ -8,18 +8,18 @@ read its tables as results, and computes every input again into its own tables. - **One data context key for the whole of Orcapod's hashing.** The default data context is now `std:v1:default` (manifest `orcapod/contexts/data/v1.json`). The version increments - when, and only when, an existing hash value changes for a reason other than a logical - type's own change. Everything stored before v0.3 (rows that carry `std:v0.1:default`, and - tables without the new column) is the earlier generation. A manifest is frozen at the + when, and only when, a released hash value changes, for any reason (ITL-946). Everything + stored before v0.3 (rows that carry `std:v0.1:default`, and tables without the new column) + is the earlier generation. A manifest is frozen at the release of the library version that contains it, and may be edited in place only while - that release is prepared: `orcapod/contexts/released_manifests.json` pins the sha256 of - each released one, and a test fails when a pinned file is edited. `v1.json` is pinned - when v0.3.0 is tagged. + that release is prepared: `orcapod/contexts/released_manifests.json` pins the digest of + each released one without its `golden` history, and a test fails when a pinned manifest is + edited (ITL-946). `v1.json` is pinned when v0.3.0 is tagged. - **Manifest contents.** Besides the wiring of the hashers, a manifest now has `released_in` (the library version that froze it; `null` while open), `previous` (the data context it follows), `definitions` (a name for each definition that lives in code), - `libraries` (the libraries the core hashes depend on, with versions), `golden` (a digest - of the golden hash files, set when the manifest is frozen) and `changes` (what differs + `libraries` (the libraries the core hashes depend on, with versions), `golden` (the folder + and the append-only history of the golden hash files, ITL-946) and `changes` (what differs from `previous`). - **Logical types moved out of the manifest.** The registered logical types and the value handlers that hash their Python values are now in diff --git a/changelog.d/ITL-946.md b/changelog.d/ITL-946.md new file mode 100644 index 000000000..b09500096 --- /dev/null +++ b/changelog.d/ITL-946.md @@ -0,0 +1,14 @@ +### Changed + +- **Released golden hash values are protected by CI (ITL-946).** The golden hash files of a + data context live in one folder per data context version + (`tests/test_hashing/golden/data/v1/-NN.json`), and the manifest's `golden` section + is an append-only history that lists every file with the sha256 of its bytes. A release + appends a revision whose digest is pinned under `releases` in + `orcapod/contexts/released_manifests.json`, which also pins the manifest without its + `golden` section. A golden file a release listed never changes; new golden values go into + new files and can be added in any release. A change to a released golden value, a logical + type's own identifier change included, needs a new data context version, shipped only in a + minor (before 1.0) or major release. A CI step (`scripts/check_golden_tags.py`) checks HEAD + against every release tag of the current data context. See + `docs/reference/golden-hashes.md`. diff --git a/design_issues/ITL-944-DC1.md b/design_issues/ITL-944-DC1.md index df13201b9..76aa6ab81 100644 --- a/design_issues/ITL-944-DC1.md +++ b/design_issues/ITL-944-DC1.md @@ -14,10 +14,11 @@ not a generation, and is missing from operator-cache and cached-source record id **Fix:** (developer's decisions of November 11, 2026, ITL-938 design document Parts 5.5 to 8) - The default data context is `std:v1:default`, manifest `contexts/data/v1.json`. - `contexts/released_manifests.json` pins the sha256 of every released manifest (`null` while - a manifest is open); `tests/test_contexts/test_data_context_v1.py` fails when a pinned file - changes, when more than the default manifest is open, or when a final release ships an open - manifest. `v1.json` is pinned once, in the v0.3.0 release-preparation commit. + `contexts/released_manifests.json` pins the digest of every released manifest without its + `golden` history (`null` while a manifest is open) and, since ITL-946, one revision digest + per release; `tests/test_contexts/test_data_context_v1.py` fails when a pinned manifest + changes, when more than the default manifest is open, or when a final release has no + releases line. `v1.json` is pinned once, by the v0.3.0 release step. - Every stored row carries `__data_context` (`large_string`): function pipeline and result tables, operator-cache tables, cached-source tables, side-effect invocation logs. It replaces `__input_data_context_key` and the result table's `_context_key`. The value is the @@ -52,8 +53,9 @@ Known limits, follow-ups: their value handlers moved to the unfrozen `contexts/logical_types.json`. The names under `definitions` are labels only: nothing in the code reads them or checks that a definition changed, and `changes` lists what is true on this branch (starfix format 2); the entries - for ITL-935 and ITL-942 are added when those land. `golden.digest` stays `null` until the - manifest is frozen, because the golden files still move. + for ITL-935 and ITL-942 are added when those land. `golden` holds the golden folder and the + append-only history of the golden files, and `released_manifests.json` pins the manifest + without `golden` and one revision digest per release (ITL-946). - `v0.1.json` is removed (to live in `orcapod-sanctuary`); `std:v0.1:default` raises `ContextResolutionError`, and a node saved under it loads under the current data context with a warning (`contexts.resolve_saved_context`). diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index 6cf9989f0..e96c39302 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -65,6 +65,10 @@ differently**. The identifier is therefore not a package version. A type can go through many releases under one identifier. +A built-in type's values are pinned as golden hashes. Once a release has shipped them, a new +identifier for that type moves released golden values, so it needs a new data context +version, like any other change to a released hash ([Golden Hashes](../reference/golden-hashes.md)). + ## Built-in types Orcapod's own types follow the same contract as yours. Their metadata is the JSON text diff --git a/docs/reference/golden-hashes.md b/docs/reference/golden-hashes.md new file mode 100644 index 000000000..98de80018 --- /dev/null +++ b/docs/reference/golden-hashes.md @@ -0,0 +1,357 @@ +# Golden Hashes + +Within one data context version, a golden hash value that has shipped in a release never +changes and is never removed. New golden values can be added at any time. Everything on this +page exists to keep that promise and to make CI fail when something breaks it (ITL-626, +ITL-946). + +The promise matters because Orcapod hashes are addresses, not just checksums. They name +stored table paths, record ids, memoization keys, default source ids and system-tag column +names (see the [Hashing Reference](hashing.md)). A hash that moves without notice strands +data: after an upgrade, a pipeline no longer finds its stored results and computes them +again, or writes a second copy next to the first. + +A **golden entry** is one named case and the exact value Orcapod computes for it, for +example the `semantic` entry `value.uuid.nil`, whose value is `semantic_v1:...`. The golden +test computes every entry again from the code on every CI run. A **data context version** +(today `v1`, key `std:v1:default`, first shipped in Orcapod 0.3) names one generation of +hashing. A hash change that is wanted on purpose starts a new data context version. It never +edits a released value. + +## The pieces at a glance + +The mechanism has five layers. Each one is checked against the layer below it, and the last +one is a set of records outside the repository that no pull request can change. Read an arrow +as "is checked against". The numbers refer to the scenarios in the +[threat model](#threat-model-who-catches-what). + +```text +Orcapod code computes every hash: stored paths, record ids, cache keys, source ids + | golden recompute: stops behaviour drift (1); no-skip rule (9) + v +Golden files data/v1/-NN.json: entry name -> pinned value + | file hashes: stops edited, deleted and unlisted files (2-7); duplicate names (8) + v +Chain golden.history in v1.json: revisions with the files they added (12) + | release pins: stops a recomputed chain (13) + v +Release pins released_manifests.json: one revision digest per release, manifest pin (10, 11) + | tag check in CI: stops a rewrite of files, chain and pins (14, 15) + v +Outside the tree a protected git tag per release; the PyPI wheel and sdist (16) +``` + +The first three checks stop accidental drift. A deliberate rewrite of the whole tree stops +only at the tags and on PyPI. + +## Layout and formats + +Three places hold the mechanism: the golden folder of the current data context, the +`golden` section of its manifest, and the release pins. The examples below show the state +after a hypothetical patch release 0.3.1 that added one file; digests are shortened. + +### Golden files: `tests/test_hashing/golden/data/v1/` + +```text +tests/test_hashing/golden/data/ + v1/ + arrow-01.json # the files of 0.3.0 + datagram-01.json + pipeline-01.json + schema-01.json + semantic-01.json + torch-01.json + semantic-02.json # added after 0.3.0: new semantic entries only +``` + +- A file is named after the group of cases it pins plus a two-digit counter: + `-NN.json`, where the group is one of `cases.GROUPS` (`semantic`, `schema`, + `arrow`, `datagram`, `pipeline`, `torch`). Names carry no date, issue or release; that + information lives in the manifest. The folder holds nothing else. +- A file maps entry names to values. An entry name appears in exactly one file of the + folder, whatever its group. +- A file listed by a released revision is never edited. New values go into the group's next + file. +- The folder holds only the current data context. The golden folders of older data + contexts live in `orcapod-sanctuary`. + +### The chain: `golden` in `src/orcapod/contexts/data/v1.json` + +```json +"golden": { + "folder": "v1", + "history": [ + {"date": "2026-11-15", "release": null, + "added": {"arrow-01.json": "d62d...", "datagram-01.json": "1151...", + "pipeline-01.json": "e4e4...", "schema-01.json": "315a...", + "semantic-01.json": "187c...", "torch-01.json": "99af..."}, + "previous": null, "digest": "2bc7..."}, + {"date": "2026-11-16", "release": "0.3.0", "added": {}, + "previous": "2bc7...", "digest": "d002..."}, + {"date": "2026-11-24", "release": null, + "added": {"semantic-02.json": "e810..."}, + "previous": "d002...", "digest": "d003..."}, + {"date": "2026-12-01", "release": "0.3.1", "added": {}, + "previous": "d003...", "digest": "d004..."} + ] +} +``` + +- `folder` names the golden folder, under `tests/test_hashing/golden/data/`. +- A revision either adds files or marks a release, never both. A revision that adds files + lists each with the sha256 of its bytes. A release revision has `added` empty and names the + library version in `release`. +- Revisions after the last release revision are open: they may still change, and their + digests are then computed again. The revisions up to the last release revision, and every + file they list, are fixed. +- `date` is the date the revision was last changed. It is informational: nothing can verify + it. + +### The pins: `src/orcapod/contexts/released_manifests.json` + +```json +{"v1": {"manifest": "b249...", + "releases": {"0.3.0": "d002...", "0.3.1": "d004..."}}} +``` + +- `manifest` pins the rest of `v1.json` (the hashers, `definitions`, `libraries`, + `previous`, `released_in`). It is `null` while the data context is open and is set once, + at the data context's first release. +- `releases` gains one line per library release: the digest of that release's revision. + +### Digest definitions + +| Value | Computed as | +| --- | --- | +| File hash in `added` | sha256 of the file's bytes, hex | +| Revision `digest` | sha256 of the canonical JSON (sorted keys, no spaces, `json.dumps(..., sort_keys=True, separators=(",", ":"))`) of `{date, release, added, previous}` | +| `manifest` pin | sha256 of the canonical JSON of the manifest without its `golden` key | +| `releases` pin | the `digest` of that release's revision | + +Each revision's digest covers the previous revision's digest, so pinning a release digest +also fixes every revision before it. + +## Lifecycle + +Three events touch the golden files: a pull request adds values, a release freezes them, and +a new data context version starts a new folder and chain. Nothing else writes to them. All +writes go through `tests/test_hashing/golden/generate_golden_hashes.py`. + +### Adding golden values (any pull request) + +1. Add the case to `tests/test_hashing/golden/cases.py`. Never move or rename an existing + fixture: function pods, dataclasses and pydantic models hash with their module path. +2. Run `uv run python tests/test_hashing/golden/generate_golden_hashes.py` (`--add` is the + default). It computes every case and writes only the entries that no golden file pins + yet. A group's new entries go into its newest file when no release lists that file yet, + and otherwise into a new file with the next counter (`semantic-02.json`). The file is + recorded in the open revision, dated today, and the open revisions' digests are computed + again; after a release revision, a new open revision is appended. It never changes an + existing value, and it writes nothing when the tree already breaks a rule. +3. The diff shows only added entries or a new file, and the edit to the open revision. +4. `--check` computes every case and reports moved, unpinned and no longer produced entries + without writing anything. + +Before a release, a value that no release lists can change only with the developer's +approval and a written migration assessment: remove the entry from its file and run the +generator again. + +### Releasing (the release pull request) + +1. The data context's `v*` release tags are protected on both `metamorphic-labs/orcapod-python` + and `nauticalab/orcapod-python` (developer decision, 2026-11-15). The developer sets + this; no agent changes repository settings. +2. Run `uv run python tests/test_hashing/golden/generate_golden_hashes.py --release X.Y.Z`. + It computes no hash. It appends a revision with `added: {}` and `release: "X.Y.Z"`, writes + its digest into `released_manifests.json` under `releases`, and at the data context's first + release sets `released_in` and the `manifest` pin. It refuses a version already released, + a development or local version, and a tree that breaks a rule. Try it first on a copy with + `--root ` (`store.copy_tree` writes one). +3. The Release workflow tags the commit it builds, which contains the release revision. From + then on every file listed up to that revision is frozen. +4. The tag and the PyPI wheel and sdist, which carry `v1.json` and `released_manifests.json`, + are records of what shipped, outside the repository. + +### Starting a new data context version + +Within `v1` a change to a released golden value is refused, with no exemption: also not for +a logical type's own identifier change. When such a change is wanted on purpose, it ships as +a new data context version, only in a minor release (before 1.0) or a major one, never in a +patch: + +1. The new manifest `v2.json` starts with `"golden": {"folder": "v2", "history": []}`, an + empty folder `data/v2/`, and the pin `"v2": {"manifest": null, "releases": {}}`. The + generator then writes `v2/-01.json` from the new code. +2. The `v1` code, manifest, golden folder and pins move to `orcapod-sanctuary`, which keeps + them computable. +3. This repository then carries only the `v2` folder and chain. + +## Threat model: who catches what + +Every drift scenario known to us fails at least one check, as long as git history and the +release tags are protected. Accidental drift is caught inside the tree; a deliberate rewrite +of the whole tree is caught only by records outside it, the tags and PyPI. + +| # | Scenario | Kind | Caught by | What fails | +| --- | --- | --- | --- | --- | +| 1 | A code change or a dependency bump in `uv.lock` moves a hash | accidental | Golden recompute | the entry's test names the case and both values | +| 2 | Someone regenerates or edits a released file to make the test pass | accidental | File hashes | the file's sha256 no longer matches its revision | +| 3 | An editor or formatter reformats a golden file | accidental | File hashes | as in 2, though no value moved | +| 4 | A Windows checkout converts line endings | accidental | `.gitattributes` (prevents it) | nothing: the bytes stay as committed | +| 5 | An entry is deleted from a released file | accidental | File hashes | the file's sha256 no longer matches | +| 6 | A released file is deleted | accidental | File hashes | a listed file is missing | +| 7 | A golden file is added but not listed | accidental | File hashes | an unlisted file is in the folder | +| 8 | A new file repeats a released entry name with another value | either | Duplicate-name check | the name is reported with both files | +| 9 | A golden entry is marked `skip` or `xfail` to silence it | either | No-skip rule | the golden test reports the mark | +| 10 | A hasher definition or library in a released `v1.json` is edited | accidental | Manifest pin | the manifest without `golden` no longer matches `manifest` | +| 11 | A release is cut without the release step | accidental | Final-release check | a final version has no `releases` line | +| 12 | A released file is edited and its hash in `added` updated to match | deliberate | Chain | that revision's digest no longer computes, or the next revision's `previous` no longer matches | +| 13 | As 12, with every later digest computed again | deliberate | Release pins | the pinned release digest is no longer in the chain | +| 14 | As 13, with the pins rewritten too | deliberate | Tag check, PyPI | the file and the history differ from the release tag; the published `v1.json` shows the original chain | +| 15 | A release line and its revision are removed | deliberate | Tag check | the pins at the tag are missing in HEAD | +| 16 | A release tag is moved or deleted | deliberate | Tag protection, PyPI | GitHub refuses it; PyPI never accepts a second upload of a version | +| 17 | The checks themselves are edited to pass | deliberate | Review only | nothing automatic: see the residual risks | +| 18 | `main` is force-pushed with rewritten history | deliberate | Branch protection | outside this mechanism | + +## Each component in detail + +The recompute protects behaviour against the files, the file hashes protect the files against +the manifest, the chain and the pins protect the manifest against itself, and the tags and +PyPI protect everything against a rewrite of the tree. + +### Golden recompute + +`tests/test_hashing/golden/test_golden_hashes.py` + +- **Checks:** every entry of every golden file of the current folder is computed again from + the code and compared with its value, one test per entry; computed entries that no file + pins fail too. The values are also computed in subprocesses with different + `PYTHONHASHSEED` values. +- **Protects against:** behaviour drift: a refactor, an interpreter difference (3.11 to + 3.13), or a library bump in `uv.lock` that moves a pinned hash (1). +- **Cannot catch:** a golden file edited to match the new behaviour. The file hashes catch + that. + +### File hashes (the `added` map) + +`tests/test_contexts/test_data_context_v1.py`, rules in `tests/test_hashing/golden/store.py` + +- **Checks:** every listed file exists with exactly the recorded sha256, and every file in + the folder is listed by exactly one revision. +- **Protects against:** a change to a golden file's bytes, a deleted released file, an + unlisted file (2, 3, 5, 6, 7). "The test passes again after regenerating" becomes a + failure. +- **Cannot catch:** an edit that also updates the hash in `added` (12). + +### Chain (`previous` and `digest`) + +`tests/test_contexts/test_data_context_v1.py` + +- **Checks:** every revision has exactly its five fields; a release revision adds no file + and any other revision adds at least one; each digest computes from the revision's fields; + each `previous` is the digest of the revision before; no version is released twice. +- **Protects against:** an edit to one revision without touching the rest (12). It also gives + every change a date and a place in order. +- **Cannot catch:** a chain computed again from the edit onwards (13). + +### Release pins (`releases`) + +`tests/test_contexts/test_data_context_v1.py` + +- **Checks:** each pinned digest is the digest of the release revision of that version, and + each release revision is pinned. +- **Protects against:** a chain computed again (13). A rewrite has to touch a second file, + visible in review. +- **Cannot catch:** a rewrite that edits the pins as well (14). + +### Manifest pin (`manifest`) + +`tests/test_contexts/test_data_context_v1.py` + +- **Checks:** the manifest without its `golden` section has the pinned digest; + `released_in` is set exactly when the manifest is pinned, which is exactly when the chain + has a release, and names the first release. +- **Protects against:** an edit to the hashers, `definitions`, `libraries` or `previous` of a + released data context (10). It is computed over parsed JSON, so a whitespace edit does not + trip it. + +### Final-release check + +`tests/test_contexts/test_data_context_v1.py` + +- **Checks:** a final version of the library (not a development, pre-release or local + version; the version comes from the git tag) has its own `releases` line and a pinned + manifest. +- **Protects against:** a release cut without the release step (11). + +### Tag check + +`scripts/check_golden_tags.py`, a step of the `Run Tests` workflow; tested by +`tests/test_check_golden_tags.py` + +- **Checks:** for each tag `vX.Y.Z` whose `released_manifests.json` pins release `X.Y.Z` of + the current data context (the one manifest at HEAD): every golden file listed by the + revisions up to that release exists in HEAD with the bytes it has at the tag; those + revisions are unchanged in HEAD's history; and the pins of the data context at the tag (the + manifest pin and every `releases` line) are unchanged in HEAD. +- **Protects against:** a full rewrite of files, chain and pins inside the tree (14, 15). + This is the anchor outside the tree. +- **Relies on:** protected tags and history. In CI (`CI=true`) it fails, never skips, when + git, the history (a shallow clone) or the tags are missing; outside CI it skips with a + message. With no release tag of the current data context it passes with nothing to check. + +### PyPI artifacts + +A record, not a check: the wheel and sdist of each release carry `v1.json` and +`released_manifests.json`, and PyPI never accepts a second upload of a version (16). + +### `.gitattributes` + +`tests/test_hashing/golden/data/** -text` keeps git from converting line endings on checkout, +so the bytes that are hashed are the bytes that were committed (4). + +### No-skip rule + +`tests/test_hashing/golden/test_golden_hashes.py` + +- **Checks:** no golden entry carries a `skip`, `skipif` or `xfail` mark, from its + parameters or added by any plugin, except the optional-package rule (ITL-871): the `torch` + group is skipped where torch is not installed, and an entry the installed torch cannot + compute (a newer dtype) is skipped. CI installs every extra. +- **Protects against:** a golden entry silenced instead of fixed (9). + +### Duplicate-name check + +`tests/test_hashing/golden/test_golden_hashes.py` and the generator + +- **Checks:** the folder holds only `-NN.json` files of known groups, and an entry name + appears in only one of them (8). + +## Residual risks and assumptions + +| Risk or assumption | Why it matters | What covers it | +| --- | --- | --- | +| Git history and `v*` tags are protected on both repositories | The tag check trusts the tags | Branch and tag protection on both repositories (developer decision, 2026-11-15) | +| The checks themselves can be edited (17) | A weakened check passes | Review: changes to these tests and scripts get explicit review | +| Golden values are computed only with the `uv.lock` versions | A user's newer pyarrow, polars or numpy could hash differently unnoticed | ITL-1151: CI against the lowest and highest allowed dependency versions | +| A hash with no golden entry is not protected | Coverage is only as wide as `cases.py` | Add cases whenever a new stored or embedded hash appears | +| Values on `main` after the last release are open | An unreleased commit has no guarantee | Use a released version for anything that must stay stable | +| Optional groups skip without their package | `torch` values are unchecked where torch is missing | CI installs every extra | +| The `date` field cannot be verified | Informational | Order and integrity come from the chain and the tags | + +A deliberate attacker with merge rights who also rewrites history and tags is beyond what any +check inside the repository can stop; the PyPI artifacts remain the record of what shipped. + +## Rules + +- A released golden file never changes, and is never removed or renamed. +- New golden values can be added in any release, a patch included, with + `generate_golden_hashes.py`. +- No exemption: any change to a released golden value, including one caused by a logical + type's own identifier change, needs a new data context version. +- A new data context version ships only in a minor release (before 1.0) or a major one, never + in a patch. +- The release pull request runs `generate_golden_hashes.py --release X.Y.Z`, and the release + is tagged on a commit that contains it. +- When a new data context starts, the old context's code and golden files move to + `orcapod-sanctuary`. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 87b5df395..bd16218a2 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -3,6 +3,8 @@ This document is a developer-facing reference for every hash computation site in the orcapod-python framework. For conceptual background on the two identity chains (`content_hash` and `pipeline_hash`), see [Identity & Hashing](../concepts/identity.md). +Every hash listed here that Orcapod stores or embeds in a name is pinned as a golden value; +how released values are protected is described in [Golden Hashes](golden-hashes.md). --- @@ -424,7 +426,8 @@ definition. Consequences: change to the encoding of a supported construct needs a new label (`polars_expr_v2`). The canonical trees of a corpus of expressions are pinned in -`tests/test_hashing/test_polars_expr_hashing.py`, and their hashes in the golden suite. +`tests/test_hashing/test_polars_expr_hashing.py`, and their hashes in the golden suite +([Golden Hashes](golden-hashes.md)). ### 1d. Function values and parameter defaults @@ -1273,7 +1276,9 @@ whenever any existing hash value can change (the hashers, the identity definitio record-id recipes, the libraries behind them). Orcapod v0.3 writes `std:v1:default`; everything stored before v0.3 is the earlier generation, `std:v0.1:default`. The manifest of the current data context is `orcapod/contexts/data/v1.json`. It is frozen at the release of -the library version that contains it and never edited afterwards. It names the hashers, the +the library version that contains it and never edited afterwards, except that its `golden` +history gains revisions as golden values are added and released +([Golden Hashes](golden-hashes.md)). It names the hashers, the definitions that live in code, the libraries the hashes depend on and what changed from the previous data context. Logical types are not part of it: they are registered in `orcapod/contexts/logical_types.json` and versioned by their own identifier. diff --git a/mkdocs.yml b/mkdocs.yml index 9ae18972c..412aa3a5a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,4 +70,5 @@ nav: - Types: api/types.md - Reference: - Hashing: reference/hashing.md + - Golden hashes: reference/golden-hashes.md - Troubleshooting: troubleshooting.md diff --git a/scripts/check_golden_tags.py b/scripts/check_golden_tags.py new file mode 100644 index 000000000..518f1e4fe --- /dev/null +++ b/scripts/check_golden_tags.py @@ -0,0 +1,235 @@ +"""Check HEAD against every release tag of the current data context (ITL-946). + +A golden file that a release listed never changes, and the pins of a release never change. +The tests in the tree check the golden files against the manifest's history and the history +against ``released_manifests.json``; this check anchors all three to the release tags, +which no pull request can move. For each tag ``v`` whose +``src/orcapod/contexts/released_manifests.json`` pins release ``X.Y.Z`` of the current data +context, it checks in HEAD: + +- every golden file listed by the revisions up to that release exists with the bytes it has + at the tag; +- the revisions up to that release are present unchanged in the golden history; +- the pins of the current data context at the tag (the manifest pin and every ``releases`` + line) are present unchanged. + +The current data context is the one manifest in ``src/orcapod/contexts/data/`` at HEAD. + +Usage: + + uv run python scripts/check_golden_tags.py [--repo DIR] + +It exits 1 on a difference. Without git, history or tags (not a repository, a shallow +clone, no tags) it exits 1 in CI (``CI=true``) and 0 with a message elsewhere. The script +uses the standard library only. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from collections.abc import Sequence +from pathlib import Path, PurePosixPath +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +CONTEXTS = PurePosixPath("src/orcapod/contexts") +PINS = CONTEXTS / "released_manifests.json" +GOLDEN_ROOT = PurePosixPath("tests/test_hashing/golden/data") + + +class GitUnavailableError(RuntimeError): + """git, the history or the tags cannot be read.""" + + +def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[bytes]: + try: + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, check=False + ) + except FileNotFoundError as exc: + raise GitUnavailableError("git is not installed") from exc + + +def _show(repo: Path, rev: str, path: PurePosixPath) -> bytes | None: + """The bytes of ``path`` at ``rev``, or ``None`` when it has no such file.""" + result = _git(repo, "cat-file", "blob", f"{rev}:{path}") + return result.stdout if result.returncode == 0 else None + + +def _show_json(repo: Path, rev: str, path: PurePosixPath) -> Any: + """The parsed JSON of ``path`` at ``rev``, or ``None`` when absent or not JSON.""" + data = _show(repo, rev, path) + if data is None: + return None + try: + return json.loads(data) + except ValueError: + return None + + +def _require_history(repo: Path) -> list[str]: + """The repository's tags; raises when git, its history or its tags are missing.""" + inside = _git(repo, "rev-parse", "--is-inside-work-tree") + if inside.returncode != 0 or inside.stdout.strip() != b"true": + raise GitUnavailableError(f"{repo} is not a git work tree") + shallow = _git(repo, "rev-parse", "--is-shallow-repository") + if shallow.stdout.strip() != b"false": + raise GitUnavailableError( + "the clone is shallow; check out with the full history and tags " + "(actions/checkout fetch-depth: 0)" + ) + tags = _git(repo, "tag", "--list") + names = tags.stdout.decode().split() + if tags.returncode != 0 or not names: + raise GitUnavailableError("the repository has no tags; fetch them") + return names + + +def _current_version(repo: Path) -> str: + """The version of the one data context manifest at HEAD.""" + listing = _git(repo, "ls-tree", "--name-only", "HEAD", f"{CONTEXTS / 'data'}/") + names = [ + PurePosixPath(line).name + for line in listing.stdout.decode().splitlines() + if line.endswith(".json") + ] + if listing.returncode != 0 or len(names) != 1: + raise SystemExit( + f"cannot tell the current data context: manifests at HEAD are {names}" + ) + return names[0].removesuffix(".json") + + +def check_tag(repo: Path, tag: str, version: str) -> list[str] | None: + """Check HEAD against one tag. + + Args: + repo: Repository root. + tag: Tag name. + version: Current data context version. + + Returns: + ``None`` when the tag is not a release of ``version``; otherwise one sentence per + difference (empty when HEAD keeps everything the tag released). + """ + release = tag.removeprefix("v") + tag_pins = _show_json(repo, tag, PINS) + pin = tag_pins.get(version) if isinstance(tag_pins, dict) else None + if not isinstance(pin, dict) or not isinstance(pin.get("releases"), dict): + return None + if release not in pin["releases"]: + return None + problems: list[str] = [] + head_pins = _show_json(repo, "HEAD", PINS) + head_pin = head_pins.get(version) if isinstance(head_pins, dict) else None + if not isinstance(head_pin, dict): + return [f"{tag}: HEAD has no pin of data context {version}"] + if head_pin.get("manifest") != pin.get("manifest"): + problems.append(f"{tag}: the manifest pin of {version} changed") + head_releases = head_pin.get("releases") + head_releases = head_releases if isinstance(head_releases, dict) else {} + for name, digest in pin["releases"].items(): + if head_releases.get(name) != digest: + problems.append(f"{tag}: the releases line {name} of {version} changed") + + manifest_path = CONTEXTS / "data" / f"{version}.json" + manifest = _show_json(repo, tag, manifest_path) + golden = manifest.get("golden") if isinstance(manifest, dict) else None + if not isinstance(golden, dict): + return [*problems, f"{tag}: {manifest_path} has no golden section"] + history = golden.get("history") + if not isinstance(history, list): + return [*problems, f"{tag}: {manifest_path} has no golden history"] + end = next( + ( + i + for i, revision in enumerate(history) + if isinstance(revision, dict) + and revision.get("release") == release + and revision.get("digest") == pin["releases"][release] + ), + None, + ) + if end is None: + return [*problems, f"{tag}: the history has no release revision {release}"] + released = history[: end + 1] + head_manifest = _show_json(repo, "HEAD", manifest_path) + head_golden = ( + head_manifest.get("golden") if isinstance(head_manifest, dict) else None + ) + head_history = head_golden.get("history") if isinstance(head_golden, dict) else None + if not isinstance(head_history, list) or head_history[: end + 1] != released: + problems.append( + f"{tag}: the golden history up to release {release} changed in HEAD" + ) + folder = GOLDEN_ROOT / str(golden.get("folder")) + for revision in released: + added = revision.get("added") if isinstance(revision, dict) else None + if not isinstance(added, dict): + problems.append(f"{tag}: a released revision lists no files") + continue + for name in added: + path = folder / name + at_tag = _show(repo, tag, path) + at_head = _show(repo, "HEAD", path) + if at_head is None: + problems.append(f"{tag}: {path} is missing in HEAD") + elif at_head != at_tag: + problems.append(f"{tag}: {path} differs from the released file") + return problems + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the check. + + Args: + argv: Command-line arguments; ``None`` reads ``sys.argv``. + + Returns: + The exit status. + """ + parser = argparse.ArgumentParser( + description="Check HEAD against every release tag of the current data context." + ) + _ = parser.add_argument("--repo", type=Path, default=REPO_ROOT) + args = parser.parse_args(argv) + in_ci = os.environ.get("CI", "").lower() == "true" + try: + tags = _require_history(args.repo) + except GitUnavailableError as exc: + if in_ci: + print(f"error: cannot check the release tags: {exc}", file=sys.stderr) + return 1 + print(f"skipped: cannot check the release tags: {exc}") + return 0 + version = _current_version(args.repo) + checked: list[str] = [] + problems: list[str] = [] + for tag in sorted(tags): + found = check_tag(args.repo, tag, version) + if found is None: + continue + checked.append(tag) + problems.extend(found) + if problems: + print( + "HEAD changes what a release of data context " + f"{version} shipped; a released golden file, revision or pin never changes:", + file=sys.stderr, + ) + for problem in problems: + print(f" {problem}", file=sys.stderr) + return 1 + if checked: + print(f"HEAD keeps what {', '.join(checked)} released (data context {version})") + else: + print(f"no release tag of data context {version}: nothing to check") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/orcapod/contexts/data/schemas/context_schema.json b/src/orcapod/contexts/data/schemas/context_schema.json index 739ba59cc..dded61b31 100644 --- a/src/orcapod/contexts/data/schemas/context_schema.json +++ b/src/orcapod/contexts/data/schemas/context_schema.json @@ -102,23 +102,59 @@ "golden": { "type": "object", "properties": { - "files": { - "type": "string" - }, - "algorithm": { - "type": "string" + "folder": { + "type": "string", + "description": "The golden folder of this data context, under tests/test_hashing/golden/data/" }, - "digest": { - "type": [ - "string", - "null" - ] + "history": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + }, + "release": { + "type": [ + "string", + "null" + ] + }, + "added": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "previous": { + "type": [ + "string", + "null" + ] + }, + "digest": { + "type": "string" + } + }, + "required": [ + "date", + "release", + "added", + "previous", + "digest" + ], + "additionalProperties": false + }, + "description": "Append-only chain of revisions. A revision lists the golden files it added, each with the sha256 of its bytes, or marks a release (added empty, release set); digest is the sha256 of the canonical JSON (sorted keys, no spaces) of date, release, added and previous" } }, "required": [ - "digest" + "folder", + "history" ], - "description": "Digest of the golden hash files this data context stands for; null while the manifest is open" + "additionalProperties": false, + "description": "The golden hash files of this data context (ITL-946). The revisions up to the last release revision, and the files they list, never change" }, "changes": { "type": "array", diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index 538faac3f..911295d6a 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -227,9 +227,23 @@ "polars": "1.41.2: serialisation of filter expressions, which is hashed" }, "golden": { - "files": "tests/test_hashing/golden/data/*.json", - "algorithm": "sha256 over the files in name order; each contributes its name, a NUL byte, its bytes and a NUL byte", - "digest": null + "folder": "v1", + "history": [ + { + "date": "2026-11-15", + "release": null, + "added": { + "arrow-01.json": "d62d1fd8ef374e4fd55a82d7eeb09f3ed5906b9de6fc96b6b1596edcb1579da8", + "datagram-01.json": "1151c8d391f6f7303657da83720e4fcfc8c5b5082e9cd670ef51ba8886575ca9", + "pipeline-01.json": "e4e4e71b4ef9bdd9211b5c5dc15db03ee333f66af606a832d1b5fccfa21f44da", + "schema-01.json": "315a5206497f86a69029810800045e542e8f85ef733c3f1108ef2e7526024a72", + "semantic-01.json": "187c1845303e90fd79f2f8afa1a4a54addfb0be91f087570e56bfc130e2cffcd", + "torch-01.json": "99af07350bc23a0200cecfd786ccd6c17985c2270ad5e3f44aca74273a1bcdc2" + }, + "previous": null, + "digest": "2bc703287b8342d9187c491123ca69665485d8cb0ded4f6bb814cdfb33037917" + } + ] }, "changes": [ { diff --git a/src/orcapod/contexts/released_manifests.json b/src/orcapod/contexts/released_manifests.json index 82c352d29..49ff26e31 100644 --- a/src/orcapod/contexts/released_manifests.json +++ b/src/orcapod/contexts/released_manifests.json @@ -1,3 +1,6 @@ { - "v1": null + "v1": { + "manifest": null, + "releases": {} + } } diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py new file mode 100644 index 000000000..e4c365277 --- /dev/null +++ b/tests/test_check_golden_tags.py @@ -0,0 +1,215 @@ +"""The release-tag check of the golden files (ITL-946): ``scripts/check_golden_tags.py``. + +Every test runs the script as a command against a temporary git repository holding a copy of +the golden tree, released with ``generate_golden_hashes.py --release`` and tagged. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from orcapod.contexts import get_default_context +from tests.test_hashing.golden import store +from tests.test_hashing.golden.generate_golden_hashes import dump, main + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "check_golden_tags.py" +VERSION = get_default_context().version + + +def _env(*, ci: bool, ceiling: Path | None = None) -> dict[str, str]: + env = { + k: v for k, v in os.environ.items() if k != "CI" and not k.startswith("GIT_") + } + env.update( + GIT_CONFIG_GLOBAL=os.devnull, + GIT_CONFIG_NOSYSTEM="1", + GIT_AUTHOR_NAME="test", + GIT_AUTHOR_EMAIL="test@example.invalid", + GIT_COMMITTER_NAME="test", + GIT_COMMITTER_EMAIL="test@example.invalid", + ) + if ci: + env["CI"] = "true" + if ceiling is not None: + env["GIT_CEILING_DIRECTORIES"] = str(ceiling) + return env + + +def _git(repo: Path, *args: str) -> None: + _ = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + env=_env(ci=False), + ) + + +def _commit(repo: Path, message: str) -> None: + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "--no-gpg-sign", "-m", message) + + +def _check(repo: Path, *, ci: bool = False) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo)], + capture_output=True, + text=True, + check=False, + env=_env(ci=ci, ceiling=repo.parent), + ) + + +def _folder(repo: Path) -> Path: + return store.golden_folder( + repo, store.read_json(store.manifest_path(repo, VERSION)) + ) + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + """A repository with an open commit tagged ``v0.2.0`` and a release tagged ``v0.3.0``.""" + root = tmp_path / "repo" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + _git(root.parent, "init", "-q", str(root)) + _commit(root, "open data context") + _git(root, "tag", "v0.2.0") + main(["--root", str(root), "--release", "0.3.0"]) + _commit(root, "release 0.3.0") + _git(root, "tag", "v0.3.0") + return root + + +def test_head_at_the_release_passes(repo: Path) -> None: + result = _check(repo, ci=True) + assert result.returncode == 0, result.stderr + assert "v0.3.0" in result.stdout + + +def test_a_tag_that_released_nothing_is_not_checked(tmp_path: Path) -> None: + root = tmp_path / "repo" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + _git(root.parent, "init", "-q", str(root)) + _commit(root, "open data context") + _git(root, "tag", "v0.2.0") + result = _check(root, ci=True) + assert result.returncode == 0, result.stderr + assert "nothing to check" in result.stdout + + +def test_a_golden_file_added_after_the_release_passes(repo: Path) -> None: + folder = _folder(repo) + _ = (folder / "schema-02.json").write_text(dump({"new.entry": "x"})) + manifest = store.read_json(store.manifest_path(repo, VERSION)) + manifest["golden"]["history"] = store.with_file( + manifest["golden"]["history"], + "schema-02.json", + store.file_digest(folder / "schema-02.json"), + "2026-11-20", + ) + store.write_json(store.manifest_path(repo, VERSION), manifest) + _commit(repo, "add schema-02") + assert store.tree_problems(repo, VERSION) == [] + result = _check(repo, ci=True) + assert result.returncode == 0, result.stderr + + +def test_an_edited_released_file_fails(repo: Path) -> None: + path = _folder(repo) / "semantic-01.json" + _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) + _commit(repo, "edit a released file") + result = _check(repo) + assert result.returncode == 1 + assert "semantic-01.json differs from the released file" in result.stderr + + +def test_a_released_file_rewritten_with_its_history_and_pins_fails(repo: Path) -> None: + """A rewrite of file, revision and pins inside the tree still differs from the tag.""" + folder = _folder(repo) + path = folder / "semantic-01.json" + _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) + manifest = store.read_json(store.manifest_path(repo, VERSION)) + history = manifest["golden"]["history"] + history[0]["added"]["semantic-01.json"] = store.file_digest(path) + for i, revision in enumerate(history): + revision["previous"] = history[i - 1]["digest"] if i else None + revision["digest"] = store.revision_digest(revision) + store.write_json(store.manifest_path(repo, VERSION), manifest) + pins = store.read_json(store.pins_path(repo)) + pins[VERSION]["releases"]["0.3.0"] = history[-1]["digest"] + store.write_json(store.pins_path(repo), pins) + assert store.tree_problems(repo, VERSION) == [] + _commit(repo, "rewrite the release") + result = _check(repo) + assert result.returncode == 1 + assert "semantic-01.json differs" in result.stderr + assert "the releases line 0.3.0" in result.stderr + assert "history up to release 0.3.0 changed" in result.stderr + + +def test_a_deleted_released_file_fails(repo: Path) -> None: + (_folder(repo) / "torch-01.json").unlink() + _commit(repo, "delete a released file") + result = _check(repo) + assert result.returncode == 1 + assert "torch-01.json is missing in HEAD" in result.stderr + + +def test_a_removed_releases_line_fails(repo: Path) -> None: + pins = store.read_json(store.pins_path(repo)) + pins[VERSION]["releases"].clear() + store.write_json(store.pins_path(repo), pins) + _commit(repo, "drop the releases line") + result = _check(repo) + assert result.returncode == 1 + assert "the releases line 0.3.0" in result.stderr + + +def test_a_changed_manifest_pin_fails(repo: Path) -> None: + pins = store.read_json(store.pins_path(repo)) + pins[VERSION]["manifest"] = "0" * 64 + store.write_json(store.pins_path(repo), pins) + _commit(repo, "change the manifest pin") + result = _check(repo) + assert result.returncode == 1 + assert "manifest pin" in result.stderr + + +def test_outside_a_repository_it_skips_locally_and_fails_in_ci(tmp_path: Path) -> None: + plain = tmp_path / "plain" + store.copy_tree(store.PROJECT_ROOT, plain, VERSION) + local = _check(plain) + assert local.returncode == 0 + assert "skipped" in local.stdout + ci = _check(plain, ci=True) + assert ci.returncode == 1 + assert "not a git work tree" in ci.stderr + + +def test_a_shallow_clone_fails_in_ci(repo: Path, tmp_path: Path) -> None: + clone = tmp_path / "clone" + _ = subprocess.run( + ["git", "clone", "-q", "--depth", "1", f"file://{repo}", str(clone)], + check=True, + capture_output=True, + env=_env(ci=False), + ) + result = _check(clone, ci=True) + assert result.returncode == 1 + assert "shallow" in result.stderr + assert _check(clone).returncode == 0 + + +def test_a_repository_without_tags_fails_in_ci(tmp_path: Path) -> None: + root = tmp_path / "repo" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + _git(root.parent, "init", "-q", str(root)) + _commit(root, "open data context") + result = _check(root, ci=True) + assert result.returncode == 1 + assert "no tags" in result.stderr diff --git a/tests/test_contexts/test_data_context_v1.py b/tests/test_contexts/test_data_context_v1.py index ad939c231..2ff80005f 100644 --- a/tests/test_contexts/test_data_context_v1.py +++ b/tests/test_contexts/test_data_context_v1.py @@ -1,21 +1,20 @@ """The data context of v0.3 is ``std:v1:default``, and released manifests are frozen (ITL-944). -A data context version names one generation of hashing. Its manifest file -(``src/orcapod/contexts/data/.json``) must not change once released, so -``released_manifests.json`` pins the sha256 of every released file. The manifest of the -version under development is listed with ``null`` until the release-preparation commit -sets its digest, once. +A data context version names one generation of hashing. ``released_manifests.json`` pins, +for each version, the sha256 of its manifest without the ``golden`` section (``null`` while +the manifest is open) and, under ``releases``, the digest of each library release's +revision in the manifest's golden history (ITL-946). The rules of the golden history and +its folder are stated in ``tests/test_hashing/golden/store.py``. """ from __future__ import annotations -import hashlib import json from importlib import resources from pathlib import Path +from typing import Any import pytest -from packaging.version import Version import orcapod from orcapod import contexts @@ -23,13 +22,14 @@ PythonTypeHandlerRegistry, ) from orcapod.semantic_types.universal_converter import UniversalTypeConverter +from tests.test_hashing.golden import store CONTEXTS_DIR = Path(str(resources.files("orcapod.contexts"))) MANIFEST_DIR = CONTEXTS_DIR / "data" -def _pins() -> dict[str, str | None]: - """The pinned sha256 of each manifest, by version; ``None`` for an open manifest.""" +def _pins() -> dict[str, dict[str, Any]]: + """The pin of each manifest, by version: ``{"manifest": ..., "releases": {...}}``.""" return json.loads((CONTEXTS_DIR / "released_manifests.json").read_text()) @@ -38,6 +38,16 @@ def _manifest_files() -> dict[str, Path]: return {path.stem: path for path in sorted(MANIFEST_DIR.glob("*.json"))} +def _manifest(version: str) -> dict[str, Any]: + """A shipped manifest as parsed JSON.""" + return json.loads(_manifest_files()[version].read_text()) + + +def _history(version: str) -> list[dict[str, Any]]: + """The golden history of a shipped manifest.""" + return _manifest(version)["golden"]["history"] + + def test_default_data_context_is_v1() -> None: """The default data context has key ``std:v1:default`` and version ``v1``.""" assert contexts.get_default_context_key() == "std:v1:default" @@ -56,60 +66,70 @@ def test_every_manifest_is_listed() -> None: assert set(_pins()) == set(_manifest_files()) -def test_released_manifests_are_unchanged() -> None: - """Each pinned manifest file still has the sha256 recorded for it.""" - files = _manifest_files() - for version, pinned in _pins().items(): - if pinned is None: - continue - found = hashlib.sha256(files[version].read_bytes()).hexdigest() - assert found == pinned, ( - f"The released data context manifest {files[version].name} was edited. " - "A released manifest is frozen: a change to hashing needs a new data " - "context version with its own manifest file." - ) +@pytest.mark.parametrize("version", sorted(_pins())) +def test_a_pinned_manifest_is_unchanged(version: str) -> None: + """A pinned manifest without its golden section still has the pinned sha256.""" + pinned = _pins()[version]["manifest"] + if pinned is None: + return + assert store.fixed_manifest_digest(_manifest(version)) == pinned, ( + f"The released data context manifest {version}.json was edited. A released " + "manifest is frozen: a change to hashing needs a new data context version with " + "its own manifest file." + ) def test_only_the_default_manifest_is_open() -> None: """At most one manifest has no pinned sha256, and it is the default version's.""" - open_versions = [version for version, pinned in _pins().items() if pinned is None] + open_versions = [v for v, pin in _pins().items() if pin["manifest"] is None] assert open_versions in ([], [contexts.get_default_context().version]) -def test_a_final_release_ships_no_open_manifest() -> None: - """A final release (not a prerelease or local build) pins every manifest.""" - version = Version(orcapod.__version__) - if version.is_prerelease or version.local is not None: - # A development build or a release candidate: the manifest may still be open. - return - assert None not in _pins().values(), ( - "A release must freeze its data context manifest: set its sha256 in " - "src/orcapod/contexts/released_manifests.json." - ) +def test_a_final_release_has_its_releases_line_and_a_pinned_manifest() -> None: + """A final release (not dev, rc or local) is recorded in the pins of its context.""" + pin = _pins()[contexts.get_default_context().version] + assert store.final_release_problems(pin, orcapod.__version__) == [] # --------------------------------------------------------------------------- -# The contents of the manifest (ITL-938 design document, section 7.4) +# The golden history (ITL-946) # --------------------------------------------------------------------------- -GOLDEN_DIR = Path(__file__).parents[1] / "test_hashing" / "golden" / "data" +@pytest.mark.parametrize("version", sorted(_pins())) +def test_the_golden_history_chain_recomputes(version: str) -> None: + """Each revision's digest recomputes and names the revision before it.""" + assert store.history_problems(_history(version)) == [] -def _v1() -> dict[str, object]: - """The v1 manifest as parsed JSON.""" - return json.loads((MANIFEST_DIR / "v1.json").read_text()) +@pytest.mark.parametrize("version", sorted(_pins())) +def test_every_golden_file_is_listed_once_with_its_hash(version: str) -> None: + """Every listed file exists with its sha256; every file is listed by one revision.""" + manifest = _manifest(version) + folder = store.golden_folder(store.PROJECT_ROOT, manifest) + assert store.folder_problems(manifest["golden"]["history"], folder) == [] -def golden_digest() -> str: - """The digest a manifest's ``golden`` field holds once the manifest is frozen. - sha256 over the golden hash files in name order; each file contributes its name, a - NUL byte, its bytes and a NUL byte. +@pytest.mark.parametrize("version", sorted(_pins())) +def test_the_pins_agree_with_the_golden_history(version: str) -> None: + """Each pinned release is a release revision with that version and digest, and the + reverse; ``released_in`` is set exactly when the manifest is pinned. """ - digest = hashlib.sha256() - for path in sorted(GOLDEN_DIR.glob("*.json")): - digest.update(path.name.encode() + b"\0" + path.read_bytes() + b"\0") - return digest.hexdigest() + manifest = _manifest(version) + problems = store.pin_problems( + manifest, _pins()[version], manifest["golden"]["history"] + ) + assert problems == [] + + +# --------------------------------------------------------------------------- +# The contents of the manifest (ITL-938 design document, section 7.4) +# --------------------------------------------------------------------------- + + +def _v1() -> dict[str, object]: + """The v1 manifest as parsed JSON.""" + return json.loads((MANIFEST_DIR / "v1.json").read_text()) def test_v1_follows_the_pre_v1_data_context() -> None: @@ -137,23 +157,6 @@ def test_v1_names_its_definitions_libraries_and_changes() -> None: ) -def test_released_in_is_set_exactly_when_the_manifest_is_pinned() -> None: - """An open manifest has no ``released_in`` and no golden digest; a frozen one has both.""" - pinned = _pins()["v1"] - manifest = _v1() - golden = manifest["golden"] - assert isinstance(golden, dict) - if pinned is None: - assert manifest["released_in"] is None - assert golden["digest"] is None - else: - assert isinstance(manifest["released_in"], str) and manifest["released_in"] - assert golden["digest"] == golden_digest(), ( - "The golden hash files differ from the ones this released data context " - "stands for. A released data context's values must not move." - ) - - def test_v1_lists_no_logical_types() -> None: """Logical types are outside the data context version: registering one edits no manifest.""" text = (MANIFEST_DIR / "v1.json").read_text() diff --git a/tests/test_hashing/golden/README.md b/tests/test_hashing/golden/README.md new file mode 100644 index 000000000..0ef17b3e3 --- /dev/null +++ b/tests/test_hashing/golden/README.md @@ -0,0 +1,26 @@ +# Golden hash values + +Every hash Orcapod stores or embeds in a name is pinned here and checked on every CI run. The +full description, with the threat model and what each check protects against, is +[`docs/reference/golden-hashes.md`](../../../docs/reference/golden-hashes.md). + +In short: + +- `data//-NN.json` holds the values of the current data context; the folder is + named by `golden.folder` in `src/orcapod/contexts/data/.json`. +- `golden.history` in that manifest lists every file with the sha256 of its bytes, in a chain + of revisions; a release appends a revision with `added: {}` and pins its digest in + `src/orcapod/contexts/released_manifests.json`. +- A file listed by a released revision never changes. New values go into new files, in any + release. A change to a released value needs a new data context version. + +| File | Role | +| --- | --- | +| `cases.py` | The cases. Never move or rename a fixture: they hash with their module path. | +| `generate_golden_hashes.py` | Adds new entries (default), reports drift (`--check`), records a release (`--release X.Y.Z`). Read its header first. | +| `store.py` | The rules: file names, digests, history, pins. | +| `test_golden_hashes.py` | Computes every entry again; refuses skips but the optional-package rule. | +| `test_golden_store.py` | The rules and the generator, on copies of the tree. | + +`tests/test_contexts/test_data_context_v1.py` checks the repository's history and pins, and +`scripts/check_golden_tags.py` (a CI step) checks HEAD against every release tag. diff --git a/tests/test_hashing/golden/data/arrow.json b/tests/test_hashing/golden/data/v1/arrow-01.json similarity index 100% rename from tests/test_hashing/golden/data/arrow.json rename to tests/test_hashing/golden/data/v1/arrow-01.json diff --git a/tests/test_hashing/golden/data/datagram.json b/tests/test_hashing/golden/data/v1/datagram-01.json similarity index 100% rename from tests/test_hashing/golden/data/datagram.json rename to tests/test_hashing/golden/data/v1/datagram-01.json diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/v1/pipeline-01.json similarity index 100% rename from tests/test_hashing/golden/data/pipeline.json rename to tests/test_hashing/golden/data/v1/pipeline-01.json diff --git a/tests/test_hashing/golden/data/schema.json b/tests/test_hashing/golden/data/v1/schema-01.json similarity index 100% rename from tests/test_hashing/golden/data/schema.json rename to tests/test_hashing/golden/data/v1/schema-01.json diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/v1/semantic-01.json similarity index 100% rename from tests/test_hashing/golden/data/semantic.json rename to tests/test_hashing/golden/data/v1/semantic-01.json diff --git a/tests/test_hashing/golden/data/torch.json b/tests/test_hashing/golden/data/v1/torch-01.json similarity index 100% rename from tests/test_hashing/golden/data/torch.json rename to tests/test_hashing/golden/data/v1/torch-01.json diff --git a/tests/test_hashing/golden/generate_golden_hashes.py b/tests/test_hashing/golden/generate_golden_hashes.py index fedbfb4f8..e27fa3733 100644 --- a/tests/test_hashing/golden/generate_golden_hashes.py +++ b/tests/test_hashing/golden/generate_golden_hashes.py @@ -1,11 +1,15 @@ -"""Regenerate the golden hash files checked by ``test_golden_hashes.py`` (ITL-626). +"""Add, check and release the golden hash files checked by ``test_golden_hashes.py``. Usage:: - uv run python tests/test_hashing/golden/generate_golden_hashes.py + uv run python tests/test_hashing/golden/generate_golden_hashes.py # add + uv run python tests/test_hashing/golden/generate_golden_hashes.py --check # report + uv run python tests/test_hashing/golden/generate_golden_hashes.py --release 0.3.0 -writes ``tests/test_hashing/golden/data/.json`` for every group in -``cases.GROUPS``. +The golden values of the current data context live in +``tests/test_hashing/golden/data//-NN.json``, and the manifest of the data +context (``src/orcapod/contexts/data/.json``) lists every file, with the sha256 of +its bytes, in its ``golden.history`` (see ``store.py`` and ``README.md`` in this directory). READ THIS BEFORE REGENERATING ============================= @@ -15,15 +19,25 @@ by an earlier release is no longer found, is keyed differently, or is read under another column name. **A changed value is a breaking change.** -Regenerating is legitimate only when: - -1. You ADD a case or entry (existing values must come out unchanged; check the diff), or -2. A hash change is intentional, has a written migration assessment (which stored data it - strands and how users recover it), and the developer has approved it. Name the approving - decision and the migration plan in the PR description. - -Never regenerate to make a failing test pass without that approval. If a test fails and you -did not mean to change a hash, the code change is the bug. +The rules (developer, November 15, 2026, ITL-946): + +1. A golden file listed by a released revision never changes and is never removed. This + script never rewrites one. +2. Adding values is allowed in any release. ``--add`` (the default) writes the entries the + cases compute and no file pins yet: into the newest file of their group when no release + lists that file yet, otherwise into a new ``-NN.json`` with the next counter. It + records the file in the open revision of the history (appending one after a release). + Existing values are never rewritten, so check that the diff only adds. +3. A change to a released value has no exemption, not even a logical type's own identifier + change: it needs a new data context version, with a new manifest and a new golden folder, + shipped only in a minor (before 1.0) or major release. The old context's golden files and + code move to ``orcapod-sanctuary``. +4. Before a release, a value no release lists may change only with the developer's approval + and a written migration assessment: remove the entry from its (unreleased) file and run + ``--add``. + +Never edit a golden file to make a failing test pass. If a test fails and you did not mean to +change a hash, the code change is the bug. Values must be identical on every supported Python version (3.11, 3.12, 3.13) and in every process, whatever ``PYTHONHASHSEED`` is. A value that differs by interpreter is a defect to @@ -34,6 +48,15 @@ columns, the canonical ``arrow.uuid`` and ``arrow.fixed_shape_tensor`` types, and fixed-size-list values are hashable and pinned. +Releasing +--------- + +``--release X.Y.Z`` is the golden step of a release pull request: it appends the release +revision (``added`` empty), pins its digest under ``releases`` in +``src/orcapod/contexts/released_manifests.json``, and, at the data context's first release, +sets ``released_in`` and pins the digest of the manifest without ``golden``. It computes no +hash. Try it on a copy first with ``--root`` (``store.copy_tree`` makes one). + Comparing releases ------------------ @@ -41,15 +64,16 @@ that checkout's environment and this checkout's test package on the path:: uv run --directory python /tests/test_hashing/golden/generate_golden_hashes.py \\ - --allow-errors --output-dir + --stdout --allow-errors > -then diff ```` against ``tests/test_hashing/golden/data``. ``--allow-errors`` -records ``ERROR: ...`` for a case the old API cannot express instead of stopping. +and compare ```` with the golden folder. ``--allow-errors`` records +``ERROR: ...`` for a case the old API cannot express instead of stopping. """ from __future__ import annotations import argparse +import datetime import json import pathlib import sys @@ -61,54 +85,219 @@ if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) -from tests.test_hashing.golden import cases # noqa: E402 +from orcapod.contexts import get_default_context # noqa: E402 +from tests.test_hashing.golden import cases, store # noqa: E402 + + +def dump(values: dict[str, Any]) -> str: + """Serialize one golden file's entries the way the golden files are stored. + + Args: + values: Entries of one file. + + Returns: + Indented JSON with sorted keys and a trailing newline. + """ + return json.dumps(values, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + -DATA_DIR = pathlib.Path(__file__).resolve().parent / "data" +def _today() -> str: + return datetime.date.today().isoformat() + + +def _require_sound(root: pathlib.Path, version: str) -> None: + """Stop when the tree already breaks the golden rules; nothing is written then.""" + problems = store.tree_problems(root, version) + if problems: + raise SystemExit( + "the golden history does not match the files; nothing written:\n " + + "\n ".join(problems) + ) -def golden_path(group: str, data_dir: pathlib.Path = DATA_DIR) -> pathlib.Path: - """Return the golden file path for ``group``. +def add(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: + """Write the computed entries that no golden file pins yet. Args: - group: One of ``cases.GROUPS``. - data_dir: Directory holding the golden files. + root: Project root. + version: Data context version. + groups: Groups to compute. Returns: - The path ``/.json``. + Names of the files written. """ - return data_dir / f"{group}.json" + _require_sound(root, version) + path = store.manifest_path(root, version) + manifest = store.read_json(path) + folder = store.golden_folder(root, manifest) + pinned = store.load_folder(folder) + owner = {entry: group for group, values in pinned.items() for entry in values} + files = store.group_files(folder) + history = manifest["golden"]["history"] + released = store.released_files(history) + written: list[str] = [] + for group in groups: + live = cases.compute_group(group) + clash = sorted(e for e in live if owner.get(e, group) != group) + if clash: + raise SystemExit(f"{group} entries pinned under another group: {clash}") + new = {e: v for e, v in live.items() if e not in owner} + if not new: + continue + existing = files.get(group, []) + latest = existing[-1] if existing else None + if latest is not None and latest.name not in released: + target = latest + values = {**store.read_json(latest), **new} + else: + number = 1 + if latest is not None: + match = store.FILE_NAME.match(latest.name) + assert match is not None + number = int(match["number"]) + 1 + if number > 99: + raise SystemExit(f"{group} has no free file counter") + target = folder / f"{group}-{number:02d}.json" + values = new + _ = target.write_text(dump(values), encoding="utf-8") + history = store.with_file( + history, target.name, store.file_digest(target), _today() + ) + written.append(target.name) + print(f"added {len(new)} {group} entries to {target}") + if written: + manifest["golden"]["history"] = history + store.write_json(path, manifest) + else: + print("nothing to add: every computed entry is pinned") + return written -def dump(values: dict[str, Any]) -> str: - """Serialize one group's entries the way the golden files are stored. +def check(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: + """Compare the computed values with the golden folder; write nothing. Args: - values: Entries of one group. + root: Project root. + version: Data context version. + groups: Groups to compute. Returns: - Indented JSON with sorted keys and a trailing newline. + One sentence per difference or rule problem. """ - return json.dumps(values, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + problems = store.tree_problems(root, version) + manifest = store.read_json(store.manifest_path(root, version)) + try: + pinned = store.load_folder(store.golden_folder(root, manifest)) + except store.GoldenLayoutError as exc: + return [*problems, str(exc)] + for group in groups: + live = cases.compute_group(group) + pins = pinned.get(group, {}) + skipped = cases.unavailable_entries(group) + for entry in sorted(set(live) | set(pins)): + if entry in skipped: + continue + if entry not in pins: + problems.append(f"{group}:{entry} is computed but not pinned") + elif entry not in live: + problems.append(f"{group}:{entry} is pinned but no longer produced") + elif live[entry] != pins[entry]: + problems.append( + f"{group}:{entry} moved: pinned {pins[entry]!r}, live {live[entry]!r}" + ) + return problems + + +def release(root: pathlib.Path, version: str, release_text: str) -> str: + """Record a library release in the golden history and the pins. + + Args: + root: Project root. + version: Data context version. + release_text: The library version, e.g. ``0.3.0``. + + Returns: + The digest of the release revision. + """ + try: + name = store.normalize_release(release_text) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + _require_sound(root, version) + path = store.manifest_path(root, version) + manifest = store.read_json(path) + pins = store.read_json(store.pins_path(root)) + pin = pins[version] + if name in pin["releases"]: + raise SystemExit(f"{name} is already released") + history = manifest["golden"]["history"] + if not history: + raise SystemExit("the golden history lists no file") + history = store.with_release(history, name, _today()) + manifest["golden"]["history"] = history + if manifest["released_in"] is None: + manifest["released_in"] = name + fixed = store.fixed_manifest_digest(manifest) + pins[version] = { + "manifest": fixed, + "releases": {**pin["releases"], name: history[-1]["digest"]}, + } + store.write_json(path, manifest) + store.write_json(store.pins_path(root), pins) + print(f"released {name}: revision {history[-1]['digest']}, manifest {fixed}") + return history[-1]["digest"] def main(argv: list[str] | None = None) -> None: - """Compute the golden groups and write (or print) them. + """Add, check or release the golden files, or print the computed values. Args: argv: Command-line arguments; ``None`` reads ``sys.argv``. """ parser = argparse.ArgumentParser( - description="Regenerate the golden hash files (ITL-626)." + description="Add, check or release the golden hash files (ITL-626, ITL-946)." + ) + mode = parser.add_mutually_exclusive_group() + _ = mode.add_argument( + "--add", + action="store_true", + help="write computed entries no golden file pins yet (the default)", + ) + _ = mode.add_argument( + "--check", + action="store_true", + help="report moved, unpinned and missing entries; write nothing", + ) + _ = mode.add_argument( + "--release", + metavar="X.Y.Z", + help="append the release revision and pin it; computes no hash", + ) + _ = mode.add_argument( + "--stdout", + action="store_true", + help="print all computed groups as one JSON object; write nothing", ) - _ = parser.add_argument("--output-dir", type=pathlib.Path, default=DATA_DIR) _ = parser.add_argument("--group", choices=cases.GROUPS, action="append") - _ = parser.add_argument("--allow-errors", action="store_true") _ = parser.add_argument( - "--stdout", + "--allow-errors", action="store_true", - help="print all groups as one JSON object instead of writing files", + help="with --stdout: record a failing case as ERROR instead of stopping", + ) + _ = parser.add_argument( + "--root", + type=pathlib.Path, + default=_PROJECT_ROOT, + help="project root holding the manifest, the pins and the golden folder", ) args = parser.parse_args(argv) + if args.allow_errors and not args.stdout: + parser.error("--allow-errors is only for --stdout: errors are never pinned") + + version = get_default_context().version + if args.release is not None: + _ = release(args.root, version, args.release) + return unavailable = [g for g in args.group or [] if not cases.group_available(g)] if unavailable: @@ -121,40 +310,22 @@ def main(argv: list[str] | None = None) -> None: g for g in cases.GROUPS if g not in groups and not cases.group_available(g) ] if skipped and not args.stdout: - # Never rewrite an optional group's file without its dependency (ITL-871). - print( - f"skipped {skipped}: optional dependency not installed; files left as they are" - ) - computed = { - g: cases.compute_group(g, allow_errors=args.allow_errors) for g in groups - } + print(f"skipped {skipped}: optional dependency not installed") if args.stdout: - sys.stdout.write(dump(computed)) + computed = { + g: cases.compute_group(g, allow_errors=args.allow_errors) for g in groups + } + _ = sys.stdout.write(dump(computed)) return - args.output_dir.mkdir(parents=True, exist_ok=True) - for group, values in computed.items(): - path = golden_path(group, args.output_dir) - missing = cases.unavailable_entries(group) - if missing: - # Keep the pins this environment cannot compute (an older optional - # dependency), from the target file or else the repository's, so a - # golden file is never written without them. - source = path if path.exists() else golden_path(group) - pinned = ( - json.loads(source.read_text(encoding="utf-8")) - if source.exists() - else {} + if args.check: + problems = check(args.root, version, groups) + if problems: + raise SystemExit( + f"{len(problems)} golden problem(s):\n " + "\n ".join(problems) ) - lost = sorted(e for e in missing if e not in pinned) - if lost: - raise SystemExit( - f"cannot write {path}: this environment cannot compute {lost} " - "and no existing golden file pins them; install the newer " - "optional dependency" - ) - values = {**values, **{e: pinned[e] for e in missing}} - path.write_text(dump(values), encoding="utf-8") - print(f"wrote {len(values)} entries to {path}") + print(f"golden values of {', '.join(groups)} match") + return + _ = add(args.root, version, groups) if __name__ == "__main__": diff --git a/tests/test_hashing/golden/store.py b/tests/test_hashing/golden/store.py new file mode 100644 index 000000000..499403223 --- /dev/null +++ b/tests/test_hashing/golden/store.py @@ -0,0 +1,566 @@ +"""The golden hash files of a data context and the history that pins them (ITL-946). + +The golden values of a data context version live in one folder, +``tests/test_hashing/golden/data//``, named by the ``golden.folder`` field of the +version's manifest (``src/orcapod/contexts/data/.json``). Each file is +``-NN.json``: the entries of one group of ``cases.GROUPS`` and a two-digit counter. +The unit of immutability is the file: a file listed by a released revision never changes, +and new values go into a new file. + +The manifest's ``golden.history`` is an append-only chain of revisions:: + + {"date": "YYYY-MM-DD", "release": "X.Y.Z" or null, + "added": {"": ""}, + "previous": , + "digest": } + +A release appends a revision with ``added`` empty and ``release`` set; its digest is pinned +in ``src/orcapod/contexts/released_manifests.json``. Revisions after the last release +revision may still be edited (their digests are recomputed); the ones up to it may not. + +This module holds the rules: reading the folder, computing digests, and listing what in a +tree breaks the rules. ``generate_golden_hashes.py`` writes the files and the history, +``test_golden_hashes.py`` and ``tests/test_contexts/test_data_context_v1.py`` check them, +and ``scripts/check_golden_tags.py`` compares them with release tags. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import shutil +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +from packaging.version import InvalidVersion, Version + +PROJECT_ROOT = Path(__file__).resolve().parents[3] + +CONTEXTS_DIR = Path("src/orcapod/contexts") +"""The data context directory, relative to a project root.""" + +PINS_FILE = CONTEXTS_DIR / "released_manifests.json" +"""The pins of released manifests, relative to a project root.""" + +GOLDEN_ROOT = Path("tests/test_hashing/golden/data") +"""The directory holding one golden folder per data context version, relative to a root.""" + +FILE_NAME = re.compile(r"^(?P[a-z][a-z0-9_]*)-(?P[0-9]{2})\.json$") +"""A golden file name: ``-NN.json``.""" + +REVISION_FIELDS = ("date", "release", "added", "previous") +"""The fields of a revision its digest covers, besides the digest itself.""" + +_DATE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$") + + +class GoldenLayoutError(ValueError): + """The golden folder breaks the file rules (an unknown file name, a duplicate entry).""" + + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- + + +def manifest_path(root: Path, version: str) -> Path: + """The manifest file of a data context version. + + Args: + root: Project root (the repository, or a copy of its golden tree). + version: Data context version, e.g. ``v1``. + + Returns: + ``/src/orcapod/contexts/data/.json``. + """ + return root / CONTEXTS_DIR / "data" / f"{version}.json" + + +def pins_path(root: Path) -> Path: + """The ``released_manifests.json`` file under ``root``.""" + return root / PINS_FILE + + +def golden_folder(root: Path, manifest: Mapping[str, Any]) -> Path: + """The golden folder a manifest names. + + Args: + root: Project root. + manifest: Parsed manifest. + + Returns: + ``/tests/test_hashing/golden/data/``. + """ + return root / GOLDEN_ROOT / manifest["golden"]["folder"] + + +def read_json(path: Path) -> Any: + """Parse a JSON file.""" + return json.loads(path.read_text(encoding="utf-8")) + + +def write_json(path: Path, value: Any) -> None: + """Write a manifest or pin file the way the repository stores it. + + Four-space indentation, keys in their given order, a trailing newline. + """ + path.write_text( + json.dumps(value, indent=4, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + +def copy_tree(source_root: Path, target_root: Path, version: str) -> None: + """Copy what the golden rules cover to another root, for a trial run. + + Copies the version's manifest, ``released_manifests.json`` and the version's golden + folder, keeping their paths relative to the root. ``generate_golden_hashes.py --root`` + can then add or release on the copy without touching the repository. + + Args: + source_root: Project root to copy from. + target_root: Root to copy to; created if missing. + version: Data context version whose files are copied. + """ + manifest = read_json(manifest_path(source_root, version)) + for path in (manifest_path(source_root, version), pins_path(source_root)): + target = target_root / path.relative_to(source_root) + target.parent.mkdir(parents=True, exist_ok=True) + _ = shutil.copyfile(path, target) + folder = golden_folder(source_root, manifest) + _ = shutil.copytree(folder, target_root / folder.relative_to(source_root)) + + +# --------------------------------------------------------------------------- +# Digests +# --------------------------------------------------------------------------- + + +def canonical_json(value: Any) -> bytes: + """The canonical JSON bytes of a value: sorted keys, no spaces.""" + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def revision_digest(revision: Mapping[str, Any]) -> str: + """The digest of a revision: sha256 of the canonical JSON of its four fields. + + Args: + revision: A revision; only ``date``, ``release``, ``added`` and ``previous`` are + read. + + Returns: + The hex digest. + """ + body = {name: revision[name] for name in REVISION_FIELDS} + return hashlib.sha256(canonical_json(body)).hexdigest() + + +def fixed_manifest_digest(manifest: Mapping[str, Any]) -> str: + """The digest pinned for a released manifest: everything but ``golden``. + + Args: + manifest: Parsed manifest. + + Returns: + sha256 of the canonical JSON of the manifest without its ``golden`` field. + """ + fixed = {name: value for name, value in manifest.items() if name != "golden"} + return hashlib.sha256(canonical_json(fixed)).hexdigest() + + +def file_digest(path: Path) -> str: + """sha256 of a file's bytes.""" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +# --------------------------------------------------------------------------- +# The folder +# --------------------------------------------------------------------------- + + +def group_files(folder: Path) -> dict[str, list[Path]]: + """The golden files of a folder by group, each group's files in counter order. + + Args: + folder: A golden folder. + + Returns: + Group name to its files. + + Raises: + GoldenLayoutError: The folder holds something that is not a ``-NN.json`` + file. + """ + files: dict[str, list[Path]] = {} + for path in sorted(folder.iterdir()): + match = FILE_NAME.match(path.name) + if match is None or not path.is_file(): + raise GoldenLayoutError( + f"{path} is not a golden file: a golden folder holds only files named " + "-NN.json" + ) + files.setdefault(match["group"], []).append(path) + return files + + +def load_folder(folder: Path) -> dict[str, dict[str, Any]]: + """Read every golden file of a folder. + + Args: + folder: A golden folder. + + Returns: + Group name to its entries, merged over the group's files. + + Raises: + GoldenLayoutError: A file name is not ``-NN.json``, a file is not a JSON + object, or an entry name appears in two files of the folder. + """ + values: dict[str, dict[str, Any]] = {} + where: dict[str, str] = {} + for group, paths in group_files(folder).items(): + merged = values.setdefault(group, {}) + for path in paths: + entries = read_json(path) + if not isinstance(entries, dict): + raise GoldenLayoutError(f"{path} does not hold a JSON object") + for entry, value in entries.items(): + if entry in where: + raise GoldenLayoutError( + f"golden entry {entry!r} is pinned twice, in {where[entry]} and " + f"{path.name}; an entry name appears once in a folder" + ) + where[entry] = path.name + merged[entry] = value + return values + + +# --------------------------------------------------------------------------- +# The history +# --------------------------------------------------------------------------- + + +def last_release_index(history: Sequence[Mapping[str, Any]]) -> int | None: + """Index of the last release revision, or ``None`` when nothing was released.""" + releases = [i for i, revision in enumerate(history) if revision["release"]] + return releases[-1] if releases else None + + +def released_files(history: Sequence[Mapping[str, Any]]) -> set[str]: + """Names of the files listed by revisions up to the last release revision.""" + end = last_release_index(history) + if end is None: + return set() + return {name for revision in history[: end + 1] for name in revision["added"]} + + +def rechain(history: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Recompute ``previous`` and ``digest`` of the revisions after the last release. + + The revisions up to the last release revision are returned unchanged. + + Args: + history: Revisions, oldest first. + + Returns: + A new list of revisions. + """ + end = last_release_index(history) + start = 0 if end is None else end + 1 + out = [dict(revision) for revision in history] + for i in range(start, len(out)): + out[i]["previous"] = out[i - 1]["digest"] if i else None + out[i]["digest"] = revision_digest(out[i]) + return out + + +def with_file( + history: Sequence[Mapping[str, Any]], name: str, digest: str, date: str +) -> list[dict[str, Any]]: + """Record a new or rewritten golden file in the open revision. + + The open revision is the last one when it is not a release; otherwise a new revision + is appended. A file listed by an open revision has its digest updated in place. + + Args: + history: Revisions, oldest first. + name: File name, ``-NN.json``. + digest: sha256 of the file's bytes. + date: The date of the change, ``YYYY-MM-DD``; it becomes the revision's date. + + Returns: + The new history, rechained. + + Raises: + ValueError: ``name`` is listed by a released revision. + """ + if name in released_files(history): + raise ValueError(f"{name} is listed by a released revision and never changes") + out = [dict(revision) for revision in history] + owner = next((r for r in out if name in r["added"]), None) + if owner is None: + if not out or out[-1]["release"]: + out.append({"date": date, "release": None, "added": {}, "previous": None}) + owner = out[-1] + owner["added"] = dict(sorted({**owner["added"], name: digest}.items())) + owner["date"] = date + return rechain(out) + + +def with_release( + history: Sequence[Mapping[str, Any]], release: str, date: str +) -> list[dict[str, Any]]: + """Append the release revision of a library version. + + Args: + history: Revisions, oldest first. + release: The library version, normalized. + date: The release date, ``YYYY-MM-DD``. + + Returns: + The new history. + """ + out = [dict(revision) for revision in history] + revision: dict[str, Any] = { + "date": date, + "release": release, + "added": {}, + "previous": out[-1]["digest"] if out else None, + } + revision["digest"] = revision_digest(revision) + return [*out, revision] + + +# --------------------------------------------------------------------------- +# The rules +# --------------------------------------------------------------------------- + + +def history_problems(history: Any) -> list[str]: + """What breaks the shape or the chain of a history. + + Every revision has exactly the fields ``date``, ``release``, ``added``, ``previous`` + and ``digest``; a release revision adds no file and any other revision adds at least + one; ``previous`` is the digest of the revision before (``null`` for the first); the + digest recomputes; and no library version is released twice. + + Args: + history: The ``golden.history`` value of a manifest. + + Returns: + One sentence per problem; empty when the history is sound. + """ + if not isinstance(history, list): + return ["golden.history is not a list"] + problems: list[str] = [] + expected_fields = {*REVISION_FIELDS, "digest"} + previous: str | None = None + seen_releases: set[str] = set() + for i, revision in enumerate(history): + label = f"revision {i}" + if not isinstance(revision, dict) or set(revision) != expected_fields: + problems.append( + f"{label} does not have exactly the fields {sorted(expected_fields)}" + ) + previous = None + continue + if not isinstance(revision["date"], str) or not _DATE.match(revision["date"]): + problems.append(f"{label} has no YYYY-MM-DD date") + added = revision["added"] + if not isinstance(added, dict) or not all( + isinstance(name, str) and isinstance(digest, str) + for name, digest in added.items() + ): + problems.append(f"{label}: added is not a mapping of file names to digests") + added = {} + release = revision["release"] + if release is not None: + if not isinstance(release, str) or not release: + problems.append(f"{label}: release is neither null nor a version") + elif release in seen_releases: + problems.append(f"{label}: release {release} appears twice") + else: + seen_releases.add(release) + if added: + problems.append(f"{label} is a release revision and adds files") + elif not added: + problems.append(f"{label} adds no file and is not a release revision") + if revision["previous"] != previous: + problems.append( + f"{label}: previous is not the digest of the revision before" + ) + if revision["digest"] != revision_digest(revision): + problems.append(f"{label}: digest does not recompute") + previous = revision["digest"] + return problems + + +def folder_problems(history: Sequence[Mapping[str, Any]], folder: Path) -> list[str]: + """What breaks the agreement between a history and its golden folder. + + Every listed file exists and has the listed sha256; every file in the folder is listed + by exactly one revision. + + Args: + history: A history without ``history_problems``. + folder: The golden folder the manifest names. + + Returns: + One sentence per problem. + """ + problems: list[str] = [] + listed: dict[str, int] = {} + end = last_release_index(history) + for i, revision in enumerate(history): + for name, digest in revision["added"].items(): + if name in listed: + problems.append(f"{name} is listed by revisions {listed[name]} and {i}") + continue + listed[name] = i + path = folder / name + if not path.is_file(): + problems.append(f"{name}, listed by revision {i}, is missing") + elif file_digest(path) != digest: + problems.append( + f"{name} does not have the sha256 revision {i} lists: " + + ( + "a released golden file never changes" + if end is not None and i <= end + else "run generate_golden_hashes.py to record it" + ) + ) + if folder.is_dir(): + for path in sorted(folder.iterdir()): + if path.name not in listed: + problems.append(f"{path.name} is in {folder} but no revision lists it") + else: + problems.append(f"the golden folder {folder} does not exist") + return problems + + +def pin_problems( + manifest: Mapping[str, Any], pin: Any, history: Sequence[Mapping[str, Any]] +) -> list[str]: + """What breaks the agreement between a manifest, its history and its pin. + + The pin is ``{"manifest": , "releases": {: }}``. The + pinned releases are exactly the release revisions, each with its digest; the manifest + pin is set exactly when ``released_in`` is, and then equals the digest of the manifest + without ``golden``; ``released_in`` is the first release; and a manifest with releases + is pinned. + + Args: + manifest: Parsed manifest. + pin: The version's value in ``released_manifests.json``. + history: The manifest's history, without ``history_problems``. + + Returns: + One sentence per problem. + """ + if ( + not isinstance(pin, dict) + or set(pin) != {"manifest", "releases"} + or not isinstance(pin["releases"], dict) + ): + return ['the pin is not {"manifest": ..., "releases": {...}}'] + problems: list[str] = [] + revisions = {r["release"]: r["digest"] for r in history if r["release"]} + for release, digest in pin["releases"].items(): + if release not in revisions: + problems.append(f"pinned release {release} is not a release revision") + elif revisions[release] != digest: + problems.append( + f"pinned release {release} is not the digest of its release revision" + ) + for release in revisions: + if release not in pin["releases"]: + problems.append(f"release revision {release} is not pinned") + released_in = manifest.get("released_in") + if (pin["manifest"] is None) != (released_in is None): + problems.append("released_in is set exactly when the manifest is pinned") + if (pin["manifest"] is None) != (not revisions): + problems.append("the manifest is pinned exactly when it has a release") + if pin["manifest"] is not None and pin["manifest"] != fixed_manifest_digest( + manifest + ): + problems.append( + "the manifest without golden differs from its pin: a released manifest is " + "frozen; a hashing change needs a new data context version" + ) + if released_in is not None and revisions and released_in != next(iter(revisions)): + problems.append("released_in is not the first release revision") + return problems + + +def final_release_problems(pin: Mapping[str, Any], package_version: str) -> list[str]: + """What a final release of the library lacks in its data context's pin. + + A final release (not a development, pre-release or local version) has its own + ``releases`` line and a pinned manifest. + + Args: + pin: The current data context's value in ``released_manifests.json``. + package_version: The library version, e.g. ``orcapod.__version__``. + + Returns: + One sentence per problem; empty for a version that is not a final release. + """ + version = Version(package_version) + if version.is_prerelease or version.is_devrelease or version.local is not None: + return [] + problems: list[str] = [] + if pin["manifest"] is None: + problems.append(f"release {version} ships an open manifest") + if str(version) not in pin["releases"]: + problems.append( + f"release {version} has no releases line: run " + f"generate_golden_hashes.py --release {version}" + ) + return problems + + +def normalize_release(text: str) -> str: + """The normalized public version of a release, e.g. ``0.3.0``. + + Args: + text: A version, optionally with a leading ``v``. + + Returns: + The normalized version. + + Raises: + ValueError: ``text`` is not a version, or is a local or development version. + """ + try: + version = Version(text.removeprefix("v")) + except InvalidVersion as exc: + raise ValueError(f"{text!r} is not a version") from exc + if version.local is not None or version.is_devrelease: + raise ValueError(f"{text!r} is a local or development version, never released") + return str(version) + + +def tree_problems(root: Path, version: str) -> list[str]: + """Every problem of a tree's golden history, folder and pin for one version. + + Args: + root: Project root. + version: Data context version. + + Returns: + One sentence per problem. + """ + manifest = read_json(manifest_path(root, version)) + golden = manifest.get("golden") + if not isinstance(golden, dict) or set(golden) != {"folder", "history"}: + return ['the manifest golden field is not {"folder": ..., "history": [...]}'] + history = golden["history"] + problems = history_problems(history) + if problems: + return problems + problems = folder_problems(history, golden_folder(root, manifest)) + pins = read_json(pins_path(root)) + if version not in pins: + return [*problems, f"{version} is not listed in released_manifests.json"] + return [*problems, *pin_problems(manifest, pins[version], history)] diff --git a/tests/test_hashing/golden/test_golden_hashes.py b/tests/test_hashing/golden/test_golden_hashes.py index 7c5f4acbd..2f562e87f 100644 --- a/tests/test_hashing/golden/test_golden_hashes.py +++ b/tests/test_hashing/golden/test_golden_hashes.py @@ -1,12 +1,18 @@ -"""Golden hash regression tests (ITL-626). +"""Golden hash regression tests (ITL-626, ITL-946). Every hash Orcapod stores, keys data on, or embeds in a column name or a database table path -is pinned in ``data/.json``. These tests recompute each value and compare it with the -pinned one, one test per entry, so a failure names exactly what moved. +is pinned in the golden folder of the current data context, +``data//-NN.json`` (the folder is named by the manifest's ``golden.folder``). +These tests recompute each value and compare it with the pinned one, one test per entry, so +a failure names exactly what moved. A failing entry is a BREAKING change: stored data keyed on the old value is stranded. Do not -regenerate the golden files to make a failure go away; read the header of -``generate_golden_hashes.py`` first. +edit the golden files to make a failure go away; read the header of +``generate_golden_hashes.py`` and ``README.md`` first. + +A golden entry is never skipped or expected to fail, except for the optional-package rule +(ITL-871): a group whose optional package is not installed, and an entry the installed +version of that package cannot compute. """ from __future__ import annotations @@ -16,6 +22,7 @@ import subprocess import sys import uuid +from collections.abc import Iterable from pathlib import Path from typing import Any @@ -25,23 +32,28 @@ from orcapod.contexts import get_default_context from orcapod.types import Schema -from tests.test_hashing.golden import cases -from tests.test_hashing.golden.generate_golden_hashes import golden_path +from tests.test_hashing.golden import cases, store GENERATOR = Path(__file__).with_name("generate_golden_hashes.py") +MANIFEST = store.read_json( + store.manifest_path(store.PROJECT_ROOT, get_default_context().version) +) +FOLDER = store.golden_folder(store.PROJECT_ROOT, MANIFEST) -def _load_golden() -> dict[str, dict[str, Any]]: - golden: dict[str, dict[str, Any]] = {} - for group in cases.GROUPS: - path = golden_path(group) - golden[group] = ( - json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} - ) - return golden + +def _load_golden() -> tuple[dict[str, dict[str, Any]], str | None]: + """The pinned entries of every group, and the error that stopped reading them.""" + try: + loaded = store.load_folder(FOLDER) + except (store.GoldenLayoutError, OSError) as exc: + loaded, error = {}, str(exc) + else: + error = None + return {group: loaded.get(group, {}) for group in {*cases.GROUPS, *loaded}}, error -GOLDEN = _load_golden() +GOLDEN, LOAD_ERROR = _load_golden() def _needs(group: str) -> Any: @@ -53,8 +65,13 @@ def _needs(group: str) -> Any: ) +def _entry_marks(group: str) -> list[Any]: + """The only marks a golden entry may carry: the optional-package skip (ITL-871).""" + return [_needs(group)] if group in cases.OPTIONAL_GROUPS else [] + + GOLDEN_ENTRIES = [ - pytest.param(group, entry, id=f"{group}:{entry}", marks=_needs(group)) + pytest.param(group, entry, id=f"{group}:{entry}", marks=_entry_marks(group)) for group in cases.GROUPS for entry in GOLDEN[group] ] @@ -78,16 +95,30 @@ def live() -> dict[str, dict[str, Any]]: return cases.compute_all() +def test_golden_folder_reads() -> None: + """The golden folder holds only ``-NN.json`` files, no entry pinned twice.""" + assert LOAD_ERROR is None, LOAD_ERROR + + +def test_golden_folder_names_only_known_groups() -> None: + """Every golden file belongs to a group of ``cases.GROUPS``.""" + unknown = sorted(set(GOLDEN) - set(cases.GROUPS)) + assert not unknown, f"golden files of unknown groups in {FOLDER}: {unknown}" + + @pytest.mark.parametrize("group", cases.GROUPS) -def test_golden_file_present(group: str) -> None: - assert golden_path(group).exists(), ( - f"missing golden file {golden_path(group)}; run generate_golden_hashes.py" +def test_golden_group_has_a_file(group: str) -> None: + assert GOLDEN[group], ( + f"no {group}-NN.json in {FOLDER}; run generate_golden_hashes.py" ) @pytest.mark.parametrize(("group", "entry"), GOLDEN_ENTRIES) def test_golden_entry(live: dict[str, dict[str, Any]], group: str, entry: str) -> None: if entry in cases.unavailable_entries(group): + # The optional-package rule only: an entry the installed version of an optional + # package cannot compute (ITL-871). + assert group in cases.OPTIONAL_GROUPS, f"{group}:{entry} may not be skipped" pytest.skip(f"{group}:{entry} needs a newer version of an optional dependency") actual = live[group].get(entry, "") expected = GOLDEN[group][entry] @@ -99,10 +130,94 @@ def test_no_unpinned_entries(live: dict[str, dict[str, Any]], group: str) -> Non unpinned = sorted(set(live[group]) - set(GOLDEN[group])) assert not unpinned, ( f"{len(unpinned)} {group} entries are computed but not pinned (new cases need " - f"the generator run; check the diff adds entries only): {unpinned[:10]}" + f"generate_golden_hashes.py --add; check the diff adds entries only): " + f"{unpinned[:10]}" ) +# --------------------------------------------------------------------------- +# No golden entry is skipped or expected to fail, but for a missing optional package +# --------------------------------------------------------------------------- + +_SKIP_OR_XFAIL = frozenset({"skip", "skipif", "xfail"}) + + +def disallowed_marks(group: str, marks: Iterable[Any]) -> list[str]: + """The skip and xfail marks on a golden entry of ``group`` that the rules forbid. + + The one allowed mark is the optional-package skip of an optional group, exactly as + ``_needs`` makes it. + + Args: + group: The entry's group. + marks: The entry's marks, as ``pytest.Mark`` or ``pytest.MarkDecorator`` objects. + + Returns: + A description of each forbidden mark. + """ + allowed = [mark.mark for mark in _entry_marks(group)] + found = [getattr(mark, "mark", mark) for mark in marks] + return [ + f"{mark.name}{mark.args}{mark.kwargs}" + for mark in found + if mark.name in _SKIP_OR_XFAIL and mark not in allowed + ] + + +def test_golden_entry_params_carry_only_the_optional_package_skip() -> None: + """No parametrized golden entry is marked skip or xfail but by the ITL-871 rule.""" + found = { + param.id: disallowed_marks(str(param.values[0]), param.marks) + for param in GOLDEN_ENTRIES + } + assert not {k: v for k, v in found.items() if v} + + +def test_collected_golden_entries_carry_only_the_optional_package_skip( + request: pytest.FixtureRequest, +) -> None: + """The collected golden entry tests carry no other skip or xfail, whoever added it.""" + found: dict[str, list[str]] = {} + for item in request.session.items: + if getattr(item, "originalname", None) != "test_golden_entry": + continue + group = item.callspec.params["group"] # pyright: ignore[reportAttributeAccessIssue] + bad = disallowed_marks(group, item.iter_markers()) + if bad: + found[item.nodeid] = bad + assert not found, f"golden entries marked skip or xfail: {found}" + + +def test_golden_test_module_marks_nothing() -> None: + """Neither the module nor the entry test function carries a mark of its own.""" + assert "pytestmark" not in globals() + own = [ + mark.name + for mark in getattr(test_golden_entry, "pytestmark", []) + if mark.name != "parametrize" + ] + assert not own + + +@pytest.mark.parametrize( + ("group", "mark"), + [ + ("semantic", pytest.mark.xfail(reason="moved")), + ("schema", pytest.mark.skip(reason="later")), + ("torch", pytest.mark.skipif(True, reason="another reason")), + ("arrow", _needs("torch")), + ], +) +def test_a_forbidden_mark_is_reported(group: str, mark: Any) -> None: + """``disallowed_marks`` names a skip or xfail other than the optional-package skip.""" + assert disallowed_marks(group, [mark.mark]) + + +def test_the_optional_package_skip_is_allowed() -> None: + """The optional-package skip of an optional group is not reported.""" + assert disallowed_marks("torch", [_needs("torch").mark]) == [] + + GENERATOR_TIMEOUT_S = 600 @@ -396,13 +511,12 @@ def test_tensor_and_ndarray_pins_differ(name: str) -> None: OLDER_TORCH_GOLDEN = """ -import json import pathlib -import shutil import sys -from tests.test_hashing.golden import cases -from tests.test_hashing.golden.generate_golden_hashes import golden_path, main +from orcapod.contexts import get_default_context +from tests.test_hashing.golden import cases, store +from tests.test_hashing.golden.generate_golden_hashes import main # Stand in for a supported PyTorch that predates float8_e8m0fnu (added in torch # 2.7). The attribute itself cannot be removed: torch.save refers to it. @@ -414,16 +528,16 @@ def test_tensor_and_ndarray_pins_differ(name: str) -> None: live = cases.compute_group("torch") print("produced:", "value.float8_e8m0fnu" in live) print("unavailable:", sorted(cases.unavailable_entries("torch"))) -out = pathlib.Path(sys.argv[1]) -shutil.copy(golden_path("torch"), out / "torch.json") -main(["--group", "torch", "--output-dir", str(out)]) -kept = json.loads((out / "torch.json").read_text()) -pinned = json.loads(golden_path("torch").read_text()) -print("pins kept:", kept == pinned) -empty = out / "empty" -main(["--group", "torch", "--output-dir", str(empty)]) -fresh = json.loads((empty / "torch.json").read_text()) -print("pins kept in an empty output dir:", fresh == pinned) +root = pathlib.Path(sys.argv[1]) +version = get_default_context().version +store.copy_tree(store.PROJECT_ROOT, root, version) +manifest = store.manifest_path(root, version) +folder = store.golden_folder(root, store.read_json(manifest)) +before = {p.name: p.read_bytes() for p in [manifest, *sorted(folder.iterdir())]} +main(["--root", str(root), "--group", "torch"]) +main(["--root", str(root), "--check", "--group", "torch"]) +after = {p.name: p.read_bytes() for p in [manifest, *sorted(folder.iterdir())]} +print("pins kept:", before == after) """ @@ -444,4 +558,3 @@ def test_torch_group_runs_on_a_torch_without_a_newer_dtype(tmp_path: Path) -> No assert "produced: False" in lines assert "unavailable: ['value.float8_e8m0fnu']" in lines assert "pins kept: True" in lines - assert "pins kept in an empty output dir: True" in lines diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py new file mode 100644 index 000000000..39c194eb0 --- /dev/null +++ b/tests/test_hashing/golden/test_golden_store.py @@ -0,0 +1,410 @@ +"""The golden folder, its history and the generator's add, check and release (ITL-946). + +The generator runs on a copy of the repository's golden tree (``--root``), never on the +repository itself. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from orcapod.contexts import get_default_context +from tests.test_hashing.golden import store +from tests.test_hashing.golden.generate_golden_hashes import dump, main + +VERSION = get_default_context().version + + +def _write(folder: Path, name: str, values: dict[str, Any]) -> None: + folder.mkdir(parents=True, exist_ok=True) + _ = (folder / name).write_text(dump(values), encoding="utf-8") + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + """A copy of the repository's golden tree.""" + root = tmp_path / "tree" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + return root + + +def _manifest(root: Path) -> dict[str, Any]: + return store.read_json(store.manifest_path(root, VERSION)) + + +def _folder(root: Path) -> Path: + return store.golden_folder(root, _manifest(root)) + + +def _history(root: Path) -> list[dict[str, Any]]: + return _manifest(root)["golden"]["history"] + + +def _pin(root: Path) -> dict[str, Any]: + return store.read_json(store.pins_path(root))[VERSION] + + +def _record(root: Path, name: str, values: dict[str, Any]) -> None: + """Write an unreleased golden file and record it, as an earlier ``--add`` would.""" + folder = _folder(root) + _write(folder, name, values) + manifest = _manifest(root) + manifest["golden"]["history"] = store.with_file( + manifest["golden"]["history"], + name, + store.file_digest(folder / name), + "2026-11-15", + ) + store.write_json(store.manifest_path(root, VERSION), manifest) + + +def _drop_entries(root: Path, name: str, count: int) -> dict[str, Any]: + """Remove the last ``count`` entries of an unreleased file; return them.""" + values = store.read_json(_folder(root) / name) + dropped = dict(list(values.items())[-count:]) + _record(root, name, {k: v for k, v in values.items() if k not in dropped}) + return dropped + + +def _snapshot(folder: Path) -> dict[str, bytes]: + return { + path.name: path.read_bytes() + for path in sorted(folder.iterdir()) + if path.is_file() + } + + +# --------------------------------------------------------------------------- +# The folder +# --------------------------------------------------------------------------- + + +def test_the_repository_tree_is_sound() -> None: + """The repository's manifest, pins and golden folder agree.""" + assert store.tree_problems(store.PROJECT_ROOT, VERSION) == [] + + +@pytest.mark.parametrize( + ("first", "second"), + [("arrow-01.json", "arrow-02.json"), ("arrow-01.json", "schema-01.json")], +) +def test_an_entry_pinned_in_two_files_is_an_error( + tmp_path: Path, first: str, second: str +) -> None: + """An entry name appears once in a folder, across the files of every group.""" + _write(tmp_path, first, {"a": 1, "b": 2}) + _write(tmp_path, second, {"b": 2}) + with pytest.raises(store.GoldenLayoutError, match="'b' is pinned twice"): + _ = store.load_folder(tmp_path) + + +def test_entries_of_a_group_merge_over_its_files(tmp_path: Path) -> None: + _write(tmp_path, "arrow-01.json", {"a": 1}) + _write(tmp_path, "arrow-02.json", {"b": 2}) + _write(tmp_path, "schema-01.json", {"c": 3}) + assert store.load_folder(tmp_path) == { + "arrow": {"a": 1, "b": 2}, + "schema": {"c": 3}, + } + + +@pytest.mark.parametrize( + "name", ["arrow.json", "arrow-1.json", "arrow-001.json", "notes.txt"] +) +def test_a_file_not_named_group_and_counter_is_an_error( + tmp_path: Path, name: str +) -> None: + _ = (tmp_path / name).write_text("{}") + with pytest.raises(store.GoldenLayoutError, match="-NN.json"): + _ = store.load_folder(tmp_path) + + +def test_an_edited_listed_file_is_reported(tree: Path) -> None: + path = _folder(tree) / "arrow-01.json" + _ = path.write_bytes(path.read_bytes().replace(b"\n", b"\r\n")) + assert any( + "arrow-01.json does not have the sha256" in p + for p in store.tree_problems(tree, VERSION) + ) + + +def test_an_unlisted_file_is_reported(tree: Path) -> None: + _write(_folder(tree), "arrow-02.json", {"new": 1}) + assert "arrow-02.json is in" in " ".join(store.tree_problems(tree, VERSION)) + + +def test_a_missing_listed_file_is_reported(tree: Path) -> None: + (_folder(tree) / "torch-01.json").unlink() + assert any( + "torch-01.json" in p and "missing" in p + for p in store.tree_problems(tree, VERSION) + ) + + +def test_a_file_listed_twice_is_reported(tree: Path) -> None: + history = _history(tree) + added = history[0]["added"] + history = store.with_release(history, "0.3.0", "2026-11-16") + history.append( + {"date": "2026-11-17", "release": None, "added": dict(added), "previous": None} + ) + history = store.rechain(history) + assert store.history_problems(history) == [] + assert any( + "listed by revisions 0 and 2" in p + for p in store.folder_problems(history, _folder(tree)) + ) + + +# --------------------------------------------------------------------------- +# The history +# --------------------------------------------------------------------------- + + +def _released_history(tree: Path) -> list[dict[str, Any]]: + return store.with_release(_history(tree), "0.3.0", "2026-11-16") + + +def test_a_rechained_history_is_sound(tree: Path) -> None: + assert store.history_problems(_released_history(tree)) == [] + + +@pytest.mark.parametrize( + ("change", "problem"), + [ + ( + lambda h: h[0].update(date="2026-11-14"), + "revision 0: digest does not recompute", + ), + (lambda h: h[1].update(previous=None), "revision 1: previous is not"), + ( + lambda h: h[1].update(added={"x-01.json": "00"}), + "release revision and adds files", + ), + (lambda h: h[0].update(added={}), "adds no file and is not a release"), + (lambda h: h[1].pop("digest"), "does not have exactly the fields"), + ( + lambda h: h.append(dict(h[1], previous=h[1]["digest"])), + "release 0.3.0 appears twice", + ), + (lambda h: h[0].update(date="15 November"), "no YYYY-MM-DD date"), + ], +) +def test_a_broken_history_is_reported(tree: Path, change: Any, problem: str) -> None: + history = _released_history(tree) + change(history) + assert problem in " ".join(store.history_problems(history)) + + +def test_a_released_file_cannot_be_recorded_again(tree: Path) -> None: + with pytest.raises(ValueError, match="never changes"): + _ = store.with_file( + _released_history(tree), "arrow-01.json", "00", "2026-11-17" + ) + + +def test_rechain_leaves_released_revisions_alone(tree: Path) -> None: + history = _released_history(tree) + tampered = [dict(history[0], date="2000-01-01"), *history[1:]] + assert store.rechain(tampered)[0]["digest"] == history[0]["digest"] + + +# --------------------------------------------------------------------------- +# The pins +# --------------------------------------------------------------------------- + + +def test_a_final_release_needs_its_releases_line_and_a_pinned_manifest() -> None: + pin = {"manifest": None, "releases": {}} + problems = store.final_release_problems(pin, "0.3.0") + assert "release 0.3.0 ships an open manifest" in problems + assert any("no releases line" in p for p in problems) + + +@pytest.mark.parametrize("version", ["0.3.1.dev4+g1234abc", "0.3.0rc1", "0.3.0+local"]) +def test_a_build_that_is_not_a_final_release_needs_no_releases_line( + version: str, +) -> None: + assert ( + store.final_release_problems({"manifest": None, "releases": {}}, version) == [] + ) + + +def test_the_open_repository_pin_is_consistent() -> None: + """While the manifest is open: no manifest pin, no releases, no released_in.""" + pin = _pin(store.PROJECT_ROOT) + manifest = _manifest(store.PROJECT_ROOT) + assert store.pin_problems(manifest, pin, manifest["golden"]["history"]) == [] + + +# --------------------------------------------------------------------------- +# The generator on a copy: add, check, release +# --------------------------------------------------------------------------- + + +def test_add_writes_new_entries_into_the_open_file_of_their_group(tree: Path) -> None: + """A group's newest file that no release lists takes the new entries.""" + dropped = _drop_entries(tree, "schema-01.json", 3) + before = _snapshot(_folder(tree)) + main(["--root", str(tree), "--group", "schema"]) + after = _snapshot(_folder(tree)) + assert set(after) == set(before) + schema = json.loads(after["schema-01.json"]) + assert dropped.items() <= schema.items() + assert {n: b for n, b in after.items() if n != "schema-01.json"} == { + n: b for n, b in before.items() if n != "schema-01.json" + } + assert len(_history(tree)) == 1 + assert store.tree_problems(tree, VERSION) == [] + + +def test_add_after_a_release_writes_a_new_file_and_a_new_revision(tree: Path) -> None: + """A released file never changes: new entries go to the next ``-NN.json``.""" + dropped = _drop_entries(tree, "schema-01.json", 2) + main(["--root", str(tree), "--release", "0.3.0"]) + before = _snapshot(_folder(tree)) + main(["--root", str(tree), "--group", "schema"]) + after = _snapshot(_folder(tree)) + assert {n: after[n] for n in before} == before + assert set(after) - set(before) == {"schema-02.json"} + assert json.loads(after["schema-02.json"]) == dropped + history = _history(tree) + assert [r["release"] for r in history] == [None, "0.3.0", None] + assert history[2]["previous"] == history[1]["digest"] + assert list(history[2]["added"]) == ["schema-02.json"] + assert store.tree_problems(tree, VERSION) == [] + + +def test_add_writes_nothing_when_every_entry_is_pinned(tree: Path) -> None: + before = _snapshot(tree / store.CONTEXTS_DIR / "data"), _snapshot(_folder(tree)) + main(["--root", str(tree), "--group", "schema"]) + assert ( + _snapshot(tree / store.CONTEXTS_DIR / "data"), + _snapshot(_folder(tree)), + ) == before + + +def test_add_refuses_a_tree_that_breaks_the_rules(tree: Path) -> None: + path = _folder(tree) / "schema-01.json" + _ = path.write_text(path.read_text() + " ") + manifest = store.manifest_path(tree, VERSION).read_bytes() + with pytest.raises(SystemExit, match="nothing written"): + main(["--root", str(tree), "--group", "schema"]) + assert store.manifest_path(tree, VERSION).read_bytes() == manifest + + +def test_check_reports_a_moved_value_and_writes_nothing(tree: Path) -> None: + name = "schema-01.json" + values = store.read_json(_folder(tree) / name) + entry = next(iter(values)) + _record(tree, name, {**values, entry: "moved"}) + before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + with pytest.raises(SystemExit, match=f"schema:{entry} moved"): + main(["--root", str(tree), "--check", "--group", "schema"]) + after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + assert after == before + + +def test_check_reports_an_unpinned_entry(tree: Path) -> None: + dropped = _drop_entries(tree, "schema-01.json", 1) + with pytest.raises(SystemExit, match=f"schema:{next(iter(dropped))} is computed"): + main(["--root", str(tree), "--check", "--group", "schema"]) + + +def test_check_passes_on_the_copy(tree: Path) -> None: + main(["--root", str(tree), "--check", "--group", "schema"]) + + +def test_release_freezes_the_manifest_and_pins_the_release(tree: Path) -> None: + """The freeze step, on a copy: release revision, pins and ``released_in``.""" + files = _snapshot(_folder(tree)) + main(["--root", str(tree), "--release", "v0.3.0"]) + manifest = _manifest(tree) + history = manifest["golden"]["history"] + pin = _pin(tree) + assert history[-1]["release"] == "0.3.0" + assert history[-1]["added"] == {} + assert manifest["released_in"] == "0.3.0" + assert pin == { + "manifest": store.fixed_manifest_digest(manifest), + "releases": {"0.3.0": history[-1]["digest"]}, + } + assert _snapshot(_folder(tree)) == files + assert store.tree_problems(tree, VERSION) == [] + assert store.final_release_problems(pin, "0.3.0") == [] + + +def test_a_later_release_keeps_released_in_and_the_manifest_pin(tree: Path) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + first = _pin(tree) + main(["--root", str(tree), "--release", "0.3.1"]) + pin = _pin(tree) + assert _manifest(tree)["released_in"] == "0.3.0" + assert pin["manifest"] == first["manifest"] + assert pin["releases"]["0.3.0"] == first["releases"]["0.3.0"] + assert set(pin["releases"]) == {"0.3.0", "0.3.1"} + assert store.tree_problems(tree, VERSION) == [] + + +def test_a_released_manifest_edited_afterwards_is_reported(tree: Path) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + manifest = _manifest(tree) + manifest["description"] += " (edited)" + store.write_json(store.manifest_path(tree, VERSION), manifest) + assert any("differs from its pin" in p for p in store.tree_problems(tree, VERSION)) + + +def test_a_released_golden_file_edited_afterwards_is_reported(tree: Path) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + path = _folder(tree) / "semantic-01.json" + _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) + assert any( + "a released golden file never changes" in p + for p in store.tree_problems(tree, VERSION) + ) + + +@pytest.mark.parametrize( + "change", + [ + lambda pin, h: pin["releases"].update({"0.3.1": h[-1]["digest"]}), + lambda pin, h: pin["releases"].clear(), + lambda pin, h: pin["releases"].update({"0.3.0": h[0]["digest"]}), + lambda pin, h: pin.update(manifest=None), + ], +) +def test_a_pin_that_disagrees_with_the_history_is_reported( + tree: Path, change: Any +) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + pins = store.read_json(store.pins_path(tree)) + change(pins[VERSION], _history(tree)) + store.write_json(store.pins_path(tree), pins) + assert store.tree_problems(tree, VERSION) + + +@pytest.mark.parametrize( + "version", ["0.3.0", "0.3.1+local", "0.3.1.dev1", "not-a-version"] +) +def test_release_refuses_a_version_it_cannot_record(tree: Path, version: str) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + before = ( + _snapshot(tree / store.CONTEXTS_DIR), + _snapshot(tree / store.CONTEXTS_DIR / "data"), + ) + with pytest.raises(SystemExit): + main(["--root", str(tree), "--release", version]) + after = ( + _snapshot(tree / store.CONTEXTS_DIR), + _snapshot(tree / store.CONTEXTS_DIR / "data"), + ) + assert after == before + + +def test_allow_errors_is_only_for_stdout(tree: Path) -> None: + with pytest.raises(SystemExit): + main(["--root", str(tree), "--allow-errors", "--group", "schema"]) From eb28445781223e2a28e12f860c3121a33590e998 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:06:49 -0800 Subject: [PATCH 02/16] docs(golden): make every golden description match the per-context folder (ITL-946) The v1 manifest description, contexts.get_context_info example, the golden cases docstrings, the hashing reference's pinned file name and the ITL-626 and ITL-1043 design log entries now describe data/v1/-NN.json and the append-only golden history. Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-1043-LJ1.md | 4 ++-- design_issues/ITL-626-HS1.md | 11 +++++++---- docs/reference/hashing.md | 2 +- src/orcapod/contexts/__init__.py | 2 +- src/orcapod/contexts/data/v1.json | 2 +- tests/test_hashing/golden/cases.py | 12 +++++++----- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/design_issues/ITL-1043-LJ1.md b/design_issues/ITL-1043-LJ1.md index 7734770f5..2141bddcb 100644 --- a/design_issues/ITL-1043-LJ1.md +++ b/design_issues/ITL-1043-LJ1.md @@ -22,8 +22,8 @@ Found November 12, 2026. Evidence and scripts: `superpowers/specs/itl-1043/FINDI - The operator's `cache_mode` is not saved (ITL-977). The running node's hashes are the intended ones: nullability is part of the schema hash -(`docs/reference/hashing.md`; golden `datagram.json` `arrow_source.int64` against -`arrow_source.int64_nullable`, `schema.json` `multi_field.nullable_type`). No live hash, +(`docs/reference/hashing.md`; golden `v1/datagram-01.json` `arrow_source.int64` against +`arrow_source.int64_nullable`, `v1/schema-01.json` `multi_field.nullable_type`). No live hash, stored path or golden value is wrong; only what a loaded job reconstructs. **Fix:** (ITL-1043; developer decision, November 12, 2026: record the information in the diff --git a/design_issues/ITL-626-HS1.md b/design_issues/ITL-626-HS1.md index 63787590b..7d438419b 100644 --- a/design_issues/ITL-626-HS1.md +++ b/design_issues/ITL-626-HS1.md @@ -14,7 +14,7 @@ and v0.2, for example, the semantic hashes of the `Path`, `UUID`, `DataFrame`, ` carrying those types. That change was noticed only by comparing releases. **Fix:** `tests/test_hashing/golden/` pins about 600 values in committed JSON files, one -generator regenerates them, and every entry is its own test, so a failure names the value that +generator writes them, and every entry is its own test, so a failure names the value that moved. Coverage: semantic hashes of scalars, collections, type objects and logical-type values; `Schema` hashes and stored Arrow types for scalar, nested, nullable and logical types (`Path`, `UUID`, `ndarray`, `DataFrame`, dataclass, pydantic, `list[Path]`, `set[UUID]`); @@ -24,6 +24,9 @@ nullability, nesting, column and row order, extension columns); and end-to-end i (content and pipeline hashes, system-tag columns, record ids, per-row data hashes, database table paths) of representative pipelines and a `PipelineJob`. Values are checked in subprocesses with different `PYTHONHASHSEED` values and were verified identical on Python -3.11, 3.12 and 3.13. The generator's header says when regeneration is legitimate: a changed -value needs a migration assessment and the developer's approval. Not pinned yet: the -`StarfixArrowHasher` gaps tracked in ITL-848. +3.11, 3.12 and 3.13. Not pinned yet: the `StarfixArrowHasher` gaps tracked in ITL-848. +Since ITL-946 the files live in one folder per data context version +(`data/v1/-NN.json`), the manifest's `golden` history pins each file's bytes, a file a +release listed never changes (new values go into new files), a change to a released value +needs a new data context version, and CI checks HEAD against every release tag; see +`docs/reference/golden-hashes.md`. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index b26f5499d..29fb94d3f 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -729,7 +729,7 @@ topology suffix, encoding the full join lineage into each column name. | **Known exclusions** | When `record_id_column` is not specified, the row index is used as the provenance token; `record_id` changes if rows are **reordered** within the source table | The recipe recomputes a record id from the source id and the token alone. The value below is -the one pinned for this row in the golden hash file (`pipeline.json`): +the one pinned for this row in the golden hash file (`v1/pipeline-01.json`): ```python import uuid diff --git a/src/orcapod/contexts/__init__.py b/src/orcapod/contexts/__init__.py index ffdbdc2ad..f86d17dcb 100644 --- a/src/orcapod/contexts/__init__.py +++ b/src/orcapod/contexts/__init__.py @@ -148,7 +148,7 @@ def get_context_info(version: str) -> dict[str, Any]: Example: >>> info = get_context_info("v1") >>> print(info['description']) - 'Data context of Orcapod v0.3: the hashing generation H1. Frozen when v0.3.0 is released; editable in place while that release is prepared.' + 'Data context of Orcapod v0.3: the hashing generation H1. Frozen when v0.3.0 is released, except its golden history, which only grows; editable in place while that release is prepared.' """ registry = _get_registry() return registry.get_context_info(version) diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index 911295d6a..8bc8af711 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -1,7 +1,7 @@ { "context_key": "std:v1:default", "version": "v1", - "description": "Data context of Orcapod v0.3: the hashing generation H1. Frozen when v0.3.0 is released; editable in place while that release is prepared.", + "description": "Data context of Orcapod v0.3: the hashing generation H1. Frozen when v0.3.0 is released, except its golden history, which only grows; editable in place while that release is prepared.", "released_in": null, "previous": "std:v0.1:default", "file_hasher": { diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 132923b0b..d319757d8 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -2,8 +2,9 @@ Each case computes a flat ``{entry: value}`` map. ``compute_group`` prefixes each entry with its case name, so a golden file entry reads ``.``. The values are pinned in -``data/.json`` by ``generate_golden_hashes.py`` and checked by -``test_golden_hashes.py``. +the golden folder of the current data context, ``data//-NN.json``, by +``generate_golden_hashes.py`` and checked by ``test_golden_hashes.py`` (ITL-946; see +``README.md``). DO NOT MOVE OR RENAME anything defined in this module. Function pods, dataclasses and pydantic models hash together with their defining module and qualified name, so moving them @@ -46,8 +47,8 @@ OPTIONAL_GROUPS: dict[str, str] = {"torch": "torch"} """Groups that need an optional dependency, mapped to the module they need (ITL-871). -Such a group is computed, generated and checked only where the module is installed; its -golden file is never rewritten without it. +Such a group is computed, added to and checked only where the module is installed; a +golden file is never rewritten, with the module or without it (ITL-946). """ @@ -78,7 +79,8 @@ def case(group: str, name: str) -> Callable[[CaseFn], CaseFn]: """Register a golden case under ``group``. Args: - group: One of ``GROUPS``; selects the golden file the entries are written to. + group: One of ``GROUPS``; selects the golden files (``-NN.json``) that + pin the entries. name: Case name, used as the prefix of every entry the case returns. Returns: From 3590a453189e44bf9f2fa1f87f4e58f50537730b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:11:18 -0800 Subject: [PATCH 03/16] fix(golden): trial trees start open; --add plans every group before writing (ITL-946) CodeRabbit CLI review of 9eb79091: - The store and tag-check tests copied the repository's tree as is, so after the v0.3.0 release step their --release 0.3.0 and edits of released files failed. store.reopen_tree gives a copy one open revision and empty pins. - --add wrote each group's file before computing the next group, so a later failure left files without their history. It now computes and plans every group first, then writes exact bytes and the manifest. Co-Authored-By: Claude Opus 5.5 --- tests/test_check_golden_tags.py | 4 +++ .../golden/generate_golden_hashes.py | 25 +++++++------ tests/test_hashing/golden/store.py | 35 +++++++++++++++++-- .../test_hashing/golden/test_golden_store.py | 30 +++++++++++++--- 4 files changed, 78 insertions(+), 16 deletions(-) diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py index e4c365277..c40de586e 100644 --- a/tests/test_check_golden_tags.py +++ b/tests/test_check_golden_tags.py @@ -76,6 +76,7 @@ def repo(tmp_path: Path) -> Path: """A repository with an open commit tagged ``v0.2.0`` and a release tagged ``v0.3.0``.""" root = tmp_path / "repo" store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") _git(root.parent, "init", "-q", str(root)) _commit(root, "open data context") _git(root, "tag", "v0.2.0") @@ -94,6 +95,7 @@ def test_head_at_the_release_passes(repo: Path) -> None: def test_a_tag_that_released_nothing_is_not_checked(tmp_path: Path) -> None: root = tmp_path / "repo" store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") _git(root.parent, "init", "-q", str(root)) _commit(root, "open data context") _git(root, "tag", "v0.2.0") @@ -183,6 +185,7 @@ def test_a_changed_manifest_pin_fails(repo: Path) -> None: def test_outside_a_repository_it_skips_locally_and_fails_in_ci(tmp_path: Path) -> None: plain = tmp_path / "plain" store.copy_tree(store.PROJECT_ROOT, plain, VERSION) + store.reopen_tree(plain, VERSION, "2026-11-15") local = _check(plain) assert local.returncode == 0 assert "skipped" in local.stdout @@ -208,6 +211,7 @@ def test_a_shallow_clone_fails_in_ci(repo: Path, tmp_path: Path) -> None: def test_a_repository_without_tags_fails_in_ci(tmp_path: Path) -> None: root = tmp_path / "repo" store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") _git(root.parent, "init", "-q", str(root)) _commit(root, "open data context") result = _check(root, ci=True) diff --git a/tests/test_hashing/golden/generate_golden_hashes.py b/tests/test_hashing/golden/generate_golden_hashes.py index e27fa3733..0413d5b92 100644 --- a/tests/test_hashing/golden/generate_golden_hashes.py +++ b/tests/test_hashing/golden/generate_golden_hashes.py @@ -74,6 +74,7 @@ import argparse import datetime +import hashlib import json import pathlib import sys @@ -135,7 +136,9 @@ def add(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: files = store.group_files(folder) history = manifest["golden"]["history"] released = store.released_files(history) - written: list[str] = [] + # Compute and plan every group before writing anything, so a failure leaves the files + # and their history as they were. + planned: list[tuple[pathlib.Path, bytes, int]] = [] for group in groups: live = cases.compute_group(group) clash = sorted(e for e in live if owner.get(e, group) != group) @@ -159,18 +162,20 @@ def add(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: raise SystemExit(f"{group} has no free file counter") target = folder / f"{group}-{number:02d}.json" values = new - _ = target.write_text(dump(values), encoding="utf-8") + data = dump(values).encode("utf-8") history = store.with_file( - history, target.name, store.file_digest(target), _today() + history, target.name, hashlib.sha256(data).hexdigest(), _today() ) - written.append(target.name) - print(f"added {len(new)} {group} entries to {target}") - if written: - manifest["golden"]["history"] = history - store.write_json(path, manifest) - else: + planned.append((target, data, len(new))) + if not planned: print("nothing to add: every computed entry is pinned") - return written + return [] + for target, data, count in planned: + _ = target.write_bytes(data) + print(f"added {count} entries to {target}") + manifest["golden"]["history"] = history + store.write_json(path, manifest) + return [target.name for target, _, _ in planned] def check(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: diff --git a/tests/test_hashing/golden/store.py b/tests/test_hashing/golden/store.py index 499403223..7eaa97eae 100644 --- a/tests/test_hashing/golden/store.py +++ b/tests/test_hashing/golden/store.py @@ -106,8 +106,8 @@ def write_json(path: Path, value: Any) -> None: Four-space indentation, keys in their given order, a trailing newline. """ - path.write_text( - json.dumps(value, indent=4, ensure_ascii=False) + "\n", encoding="utf-8" + _ = path.write_bytes( + (json.dumps(value, indent=4, ensure_ascii=False) + "\n").encode("utf-8") ) @@ -132,6 +132,37 @@ def copy_tree(source_root: Path, target_root: Path, version: str) -> None: _ = shutil.copytree(folder, target_root / folder.relative_to(source_root)) +def reopen_tree(root: Path, version: str, date: str) -> None: + """Give a copied tree a fixed open starting point, whatever the repository released. + + The history becomes one open revision listing every file of the golden folder with its + sha256; ``released_in`` is cleared and the pin becomes ``{"manifest": null, + "releases": {}}``. The golden files are not touched. Tests that add or release on a + copy start from this state, so they keep working after the repository's own releases. + + Args: + root: Root of a copy made by ``copy_tree``; never the repository. + version: Data context version. + date: Date of the open revision, ``YYYY-MM-DD``. + """ + path = manifest_path(root, version) + manifest = read_json(path) + folder = golden_folder(root, manifest) + revision: dict[str, Any] = { + "date": date, + "release": None, + "added": {p.name: file_digest(p) for p in sorted(folder.iterdir())}, + "previous": None, + } + revision["digest"] = revision_digest(revision) + manifest["golden"]["history"] = [revision] + manifest["released_in"] = None + write_json(path, manifest) + pins = read_json(pins_path(root)) + pins[version] = {"manifest": None, "releases": {}} + write_json(pins_path(root), pins) + + # --------------------------------------------------------------------------- # Digests # --------------------------------------------------------------------------- diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index 39c194eb0..f9166e167 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -13,7 +13,7 @@ import pytest from orcapod.contexts import get_default_context -from tests.test_hashing.golden import store +from tests.test_hashing.golden import cases, store from tests.test_hashing.golden.generate_golden_hashes import dump, main VERSION = get_default_context().version @@ -26,9 +26,10 @@ def _write(folder: Path, name: str, values: dict[str, Any]) -> None: @pytest.fixture def tree(tmp_path: Path) -> Path: - """A copy of the repository's golden tree.""" + """A copy of the repository's golden tree, reopened: one open revision, no release.""" root = tmp_path / "tree" store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") return root @@ -234,8 +235,8 @@ def test_a_build_that_is_not_a_final_release_needs_no_releases_line( ) -def test_the_open_repository_pin_is_consistent() -> None: - """While the manifest is open: no manifest pin, no releases, no released_in.""" +def test_the_repository_pin_is_consistent() -> None: + """The repository's pin, manifest and history agree, open or released.""" pin = _pin(store.PROJECT_ROOT) manifest = _manifest(store.PROJECT_ROOT) assert store.pin_problems(manifest, pin, manifest["golden"]["history"]) == [] @@ -279,6 +280,27 @@ def test_add_after_a_release_writes_a_new_file_and_a_new_revision(tree: Path) -> assert store.tree_problems(tree, VERSION) == [] +def test_add_changes_nothing_when_a_later_group_fails( + tree: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Every group is computed before anything is written: files and history stay in step.""" + _ = _drop_entries(tree, "schema-01.json", 1) + before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + compute = cases.compute_group + + def failing(group: str, **kwargs: Any) -> dict[str, Any]: + if group == "semantic": + raise RuntimeError("semantic failed") + return compute(group, **kwargs) + + monkeypatch.setattr(cases, "compute_group", failing) + with pytest.raises(RuntimeError, match="semantic failed"): + main(["--root", str(tree), "--group", "schema", "--group", "semantic"]) + after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + assert after == before + assert store.tree_problems(tree, VERSION) == [] + + def test_add_writes_nothing_when_every_entry_is_pinned(tree: Path) -> None: before = _snapshot(tree / store.CONTEXTS_DIR / "data"), _snapshot(_folder(tree)) main(["--root", str(tree), "--group", "schema"]) From 2189c24e61abb2c4ce165b4359a6c1528cf8e11d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:17:15 -0800 Subject: [PATCH 04/16] fix(golden): refuse one new entry name planned by two groups in one --add run (ITL-946) Co-Authored-By: Claude Opus 5.5 --- .../golden/generate_golden_hashes.py | 2 ++ tests/test_hashing/golden/test_golden_store.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/tests/test_hashing/golden/generate_golden_hashes.py b/tests/test_hashing/golden/generate_golden_hashes.py index 0413d5b92..6e5374959 100644 --- a/tests/test_hashing/golden/generate_golden_hashes.py +++ b/tests/test_hashing/golden/generate_golden_hashes.py @@ -147,6 +147,8 @@ def add(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: new = {e: v for e, v in live.items() if e not in owner} if not new: continue + # An entry name appears once in a folder: claim it before the next group plans. + owner.update(dict.fromkeys(new, group)) existing = files.get(group, []) latest = existing[-1] if existing else None if latest is not None and latest.name not in released: diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index f9166e167..837b0ad77 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -301,6 +301,23 @@ def failing(group: str, **kwargs: Any) -> dict[str, Any]: assert store.tree_problems(tree, VERSION) == [] +def test_add_refuses_one_new_entry_name_in_two_groups( + tree: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two groups that compute the same new entry name in one run write nothing.""" + before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + compute = cases.compute_group + + def with_shared_entry(group: str, **kwargs: Any) -> dict[str, Any]: + return {**compute(group, **kwargs), "shared.new_entry": group} + + monkeypatch.setattr(cases, "compute_group", with_shared_entry) + with pytest.raises(SystemExit, match="shared.new_entry"): + main(["--root", str(tree), "--group", "schema", "--group", "arrow"]) + after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + assert after == before + + def test_add_writes_nothing_when_every_entry_is_pinned(tree: Path) -> None: before = _snapshot(tree / store.CONTEXTS_DIR / "data"), _snapshot(_folder(tree)) main(["--root", str(tree), "--group", "schema"]) From bd11c842955226bebafb60b489804bcef76dd6e5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:24:52 -0800 Subject: [PATCH 05/16] feat(golden): --remove takes an entry out of an unreleased file (ITL-946) The documented approved-change procedure (remove the entry, run the generator) stopped at the soundness check, because the edited file no longer matched its recorded sha256. --remove (repeatable) removes entries from files no release lists and records those files again in the open revision; it refuses a released entry or an unknown one and then writes nothing. Docs, CLAUDE.md and .zed/rules describe --remove then --add, and the release-tag ruleset now in place on both repositories. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 3 +- CLAUDE.md | 3 +- RELEASING.md | 18 +++-- docs/reference/golden-hashes.md | 17 +++-- tests/test_hashing/golden/README.md | 2 +- .../golden/generate_golden_hashes.py | 74 ++++++++++++++++++- .../test_hashing/golden/test_golden_store.py | 34 +++++++++ 7 files changed, 133 insertions(+), 18 deletions(-) diff --git a/.zed/rules b/.zed/rules index 5835c75cb..a646e3419 100644 --- a/.zed/rules +++ b/.zed/rules @@ -411,7 +411,8 @@ Orcapod stores or embeds in a name is pinned in the golden folder of the current context's golden files and code move to `orcapod-sanctuary`; this repository carries only the current context's golden folder. - Before a release, a value no release lists changes only with the developer's approval and - a migration assessment: remove the entry from its file and run the generator. + a migration assessment: `generate_golden_hashes.py --remove ` (refuses an entry a + release lists), then the generator writes the new value. - The release pull request runs `generate_golden_hashes.py --release X.Y.Z`. CI checks HEAD against every release tag (`scripts/check_golden_tags.py`). Never mark a golden entry `skip` or `xfail`; the only skip is the optional-package rule (`torch`). diff --git a/CLAUDE.md b/CLAUDE.md index 2e87cfa00..a2b085b31 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -422,7 +422,8 @@ Orcapod stores or embeds in a name is pinned in the golden folder of the current context's golden files and code move to `orcapod-sanctuary`; this repository carries only the current context's golden folder. - Before a release, a value no release lists changes only with the developer's approval and - a migration assessment: remove the entry from its file and run the generator. + a migration assessment: `generate_golden_hashes.py --remove ` (refuses an entry a + release lists), then the generator writes the new value. - The release pull request runs `generate_golden_hashes.py --release X.Y.Z`. CI checks HEAD against every release tag (`scripts/check_golden_tags.py`). Never mark a golden entry `skip` or `xfail`; the only skip is the optional-package rule (`torch`). diff --git a/RELEASING.md b/RELEASING.md index 03dba449f..d7668a122 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -103,10 +103,13 @@ against are in `docs/reference/golden-hashes.md`. (before 1.0) or a major one, never in a patch. - When a new data context starts, the old context's golden files and code move to `orcapod-sanctuary`; this repository carries only the current context's golden folder. -- Release tags (`v*`) are protected on both `metamorphic-labs/orcapod-python` and - `nauticalab/orcapod-python`. This is a precondition of the CI tag check - (`scripts/check_golden_tags.py`), which compares HEAD with every release tag of the current - data context. The developer sets the protection; no agent changes repository settings. +- Release tags are protected on both `metamorphic-labs/orcapod-python` and + `nauticalab/orcapod-python`: the repository ruleset `protect-release-tags` (target + `refs/tags/v*`; rules deletion, update and non-fast-forward; no bypass actors) is in place + since 2026-11-15. A `v*` tag can be created, as the Release workflow does, but never moved + or deleted. This is a precondition of the CI tag check (`scripts/check_golden_tags.py`), + which compares HEAD with every release tag of the current data context. Agents never change + repository settings. ## Cutting a Release @@ -173,9 +176,10 @@ environments: that name `metamorphic-labs`, if any. 3. Protect `main` on `nauticalab` (no force push, no deletion). It is written only by the fast-forward push above. -4. Protect release tags (`v*`) on both `metamorphic-labs/orcapod-python` and - `nauticalab/orcapod-python` (no update, no deletion), so the golden tag check has a fixed - anchor (ITL-946). +4. Protect release tags on both `metamorphic-labs/orcapod-python` and + `nauticalab/orcapod-python` with the ruleset `protect-release-tags` (target + `refs/tags/v*`; deletion, update, non-fast-forward; no bypass actors), so the golden tag + check has a fixed anchor (ITL-946). Done on 2026-11-15. 5. Optional: set `CODECOV_TOKEN` there. The Codecov, SpiralDB and Linear jobs run only on `metamorphic-labs` and are skipped on `nauticalab`. diff --git a/docs/reference/golden-hashes.md b/docs/reference/golden-hashes.md index 98de80018..420940411 100644 --- a/docs/reference/golden-hashes.md +++ b/docs/reference/golden-hashes.md @@ -153,14 +153,19 @@ writes go through `tests/test_hashing/golden/generate_golden_hashes.py`. without writing anything. Before a release, a value that no release lists can change only with the developer's -approval and a written migration assessment: remove the entry from its file and run the -generator again. +approval and a written migration assessment. `--remove ` (repeatable) takes the entry +out of its file and records the file's new sha256 in the open revision; it refuses an entry +in a file a release lists and writes nothing unless every entry can be removed. `--add` then +writes the new value. Editing the file by hand does not work: the file hashes report the +edit and the generator refuses to write over it. ### Releasing (the release pull request) -1. The data context's `v*` release tags are protected on both `metamorphic-labs/orcapod-python` - and `nauticalab/orcapod-python` (developer decision, 2026-11-15). The developer sets - this; no agent changes repository settings. +1. Precondition, in place since 2026-11-15 (developer decision): the repository ruleset + `protect-release-tags` on both `metamorphic-labs/orcapod-python` and + `nauticalab/orcapod-python` targets `refs/tags/v*` with the rules deletion, update and + non-fast-forward and no bypass actors. A `v*` tag can be created (the Release workflow + pushes it) but never moved or deleted. Agents never change repository settings. 2. Run `uv run python tests/test_hashing/golden/generate_golden_hashes.py --release X.Y.Z`. It computes no hash. It appends a revision with `added: {}` and `release: "X.Y.Z"`, writes its digest into `released_manifests.json` under `releases`, and at the data context's first @@ -331,7 +336,7 @@ so the bytes that are hashed are the bytes that were committed (4). | Risk or assumption | Why it matters | What covers it | | --- | --- | --- | -| Git history and `v*` tags are protected on both repositories | The tag check trusts the tags | Branch and tag protection on both repositories (developer decision, 2026-11-15) | +| Git history and `v*` tags are protected on both repositories | The tag check trusts the tags | The `protect-release-tags` ruleset on both repositories (in place since 2026-11-15) and branch protection | | The checks themselves can be edited (17) | A weakened check passes | Review: changes to these tests and scripts get explicit review | | Golden values are computed only with the `uv.lock` versions | A user's newer pyarrow, polars or numpy could hash differently unnoticed | ITL-1151: CI against the lowest and highest allowed dependency versions | | A hash with no golden entry is not protected | Coverage is only as wide as `cases.py` | Add cases whenever a new stored or embedded hash appears | diff --git a/tests/test_hashing/golden/README.md b/tests/test_hashing/golden/README.md index 0ef17b3e3..a1749c496 100644 --- a/tests/test_hashing/golden/README.md +++ b/tests/test_hashing/golden/README.md @@ -17,7 +17,7 @@ In short: | File | Role | | --- | --- | | `cases.py` | The cases. Never move or rename a fixture: they hash with their module path. | -| `generate_golden_hashes.py` | Adds new entries (default), reports drift (`--check`), records a release (`--release X.Y.Z`). Read its header first. | +| `generate_golden_hashes.py` | Adds new entries (default), reports drift (`--check`), removes an unreleased entry (`--remove`), records a release (`--release X.Y.Z`). Read its header first. | | `store.py` | The rules: file names, digests, history, pins. | | `test_golden_hashes.py` | Computes every entry again; refuses skips but the optional-package rule. | | `test_golden_store.py` | The rules and the generator, on copies of the tree. | diff --git a/tests/test_hashing/golden/generate_golden_hashes.py b/tests/test_hashing/golden/generate_golden_hashes.py index 6e5374959..7d233e58c 100644 --- a/tests/test_hashing/golden/generate_golden_hashes.py +++ b/tests/test_hashing/golden/generate_golden_hashes.py @@ -5,6 +5,7 @@ uv run python tests/test_hashing/golden/generate_golden_hashes.py # add uv run python tests/test_hashing/golden/generate_golden_hashes.py --check # report uv run python tests/test_hashing/golden/generate_golden_hashes.py --release 0.3.0 + uv run python tests/test_hashing/golden/generate_golden_hashes.py --remove The golden values of the current data context live in ``tests/test_hashing/golden/data//-NN.json``, and the manifest of the data @@ -33,8 +34,9 @@ shipped only in a minor (before 1.0) or major release. The old context's golden files and code move to ``orcapod-sanctuary``. 4. Before a release, a value no release lists may change only with the developer's approval - and a written migration assessment: remove the entry from its (unreleased) file and run - ``--add``. + and a written migration assessment: ``--remove `` takes the entry out of its + unreleased file and records that file again in the open revision (it refuses an entry a + release lists), then ``--add`` writes the new value. Never edit a golden file to make a failing test pass. If a test fails and you did not mean to change a hash, the code change is the bug. @@ -215,6 +217,62 @@ def check(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: return problems +def remove(root: pathlib.Path, version: str, entries: list[str]) -> list[str]: + """Remove entries from golden files no release lists, recording the files again. + + The first step of an approved change to an unreleased value: remove the entry, then + ``--add`` writes its new value. An entry in a file a release lists is refused, and + nothing is written unless every entry can be removed. + + Args: + root: Project root. + version: Data context version. + entries: Entry names to remove. + + Returns: + Names of the files rewritten. + """ + _require_sound(root, version) + path = store.manifest_path(root, version) + manifest = store.read_json(path) + folder = store.golden_folder(root, manifest) + history = manifest["golden"]["history"] + released = store.released_files(history) + where = { + entry: file + for files in store.group_files(folder).values() + for file in files + for entry in store.read_json(file) + } + unknown = sorted(e for e in entries if e not in where) + if unknown: + raise SystemExit(f"no golden file pins {unknown}; nothing written") + frozen = sorted(e for e in entries if where[e].name in released) + if frozen: + raise SystemExit( + f"{frozen} are in files a release lists; a released value never changes " + "(a change needs a new data context version); nothing written" + ) + planned: dict[pathlib.Path, dict[str, Any]] = {} + for entry in dict.fromkeys(entries): + file = where[entry] + values = planned.setdefault(file, store.read_json(file)) + _ = values.pop(entry) + writes: list[tuple[pathlib.Path, bytes]] = [] + for file, values in planned.items(): + data = dump(values).encode("utf-8") + history = store.with_file( + history, file.name, hashlib.sha256(data).hexdigest(), _today() + ) + writes.append((file, data)) + for file, data in writes: + _ = file.write_bytes(data) + print(f"removed entries from {file}") + manifest["golden"]["history"] = history + store.write_json(path, manifest) + return [file.name for file, _ in writes] + + def release(root: pathlib.Path, version: str, release_text: str) -> str: """Record a library release in the golden history and the pins. @@ -280,6 +338,15 @@ def main(argv: list[str] | None = None) -> None: metavar="X.Y.Z", help="append the release revision and pin it; computes no hash", ) + _ = mode.add_argument( + "--remove", + metavar="ENTRY", + action="append", + help=( + "remove an entry from a file no release lists (repeatable); with the " + "developer's approval, the first step of changing an unreleased value" + ), + ) _ = mode.add_argument( "--stdout", action="store_true", @@ -302,6 +369,9 @@ def main(argv: list[str] | None = None) -> None: parser.error("--allow-errors is only for --stdout: errors are never pinned") version = get_default_context().version + if args.remove: + _ = remove(args.root, version, args.remove) + return if args.release is not None: _ = release(args.root, version, args.release) return diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index 837b0ad77..53b56382d 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -336,6 +336,40 @@ def test_add_refuses_a_tree_that_breaks_the_rules(tree: Path) -> None: assert store.manifest_path(tree, VERSION).read_bytes() == manifest +def test_remove_then_add_changes_an_unreleased_value(tree: Path) -> None: + """The approved-change procedure: ``--remove`` the entry, then ``--add`` writes it.""" + name = "schema-01.json" + values = store.read_json(_folder(tree) / name) + entry = next(iter(values)) + _record(tree, name, {**values, entry: "old value"}) + main(["--root", str(tree), "--remove", entry]) + assert entry not in store.read_json(_folder(tree) / name) + assert store.tree_problems(tree, VERSION) == [] + main(["--root", str(tree), "--group", "schema"]) + assert store.read_json(_folder(tree) / name) == values + assert len(_history(tree)) == 1 + assert store.tree_problems(tree, VERSION) == [] + + +def test_remove_refuses_an_entry_a_release_lists(tree: Path) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + entry = next(iter(store.read_json(_folder(tree) / "schema-01.json"))) + before = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") + with pytest.raises(SystemExit, match="released"): + main(["--root", str(tree), "--remove", entry]) + after = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") + assert after == before + + +def test_remove_refuses_an_unknown_entry_and_writes_nothing(tree: Path) -> None: + entry = next(iter(store.read_json(_folder(tree) / "schema-01.json"))) + before = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") + with pytest.raises(SystemExit, match="no.such.entry"): + main(["--root", str(tree), "--remove", entry, "--remove", "no.such.entry"]) + after = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") + assert after == before + + def test_check_reports_a_moved_value_and_writes_nothing(tree: Path) -> None: name = "schema-01.json" values = store.read_json(_folder(tree) / name) From d52f2f3dcd635b19e4c96261a3efdcfed4b2a5d5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:34:49 -0800 Subject: [PATCH 06/16] test(golden): take the newest file and next counter from the copied inventory (ITL-946) CodeRabbit CLI review of bd11c842: the trial-tree tests named schema-01.json and schema-02.json, so a later committed schema file would break them. They now derive the newest schema file and its successor (store.next_file_name) from the repository's inventory, and escape entry names used as patterns. Verified with a simulated released tree holding a schema-02.json. Co-Authored-By: Claude Opus 5.5 --- tests/test_check_golden_tags.py | 9 ++-- tests/test_hashing/golden/store.py | 18 +++++++ .../test_hashing/golden/test_golden_store.py | 47 ++++++++++++------- 3 files changed, 53 insertions(+), 21 deletions(-) diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py index c40de586e..130df288b 100644 --- a/tests/test_check_golden_tags.py +++ b/tests/test_check_golden_tags.py @@ -106,16 +106,17 @@ def test_a_tag_that_released_nothing_is_not_checked(tmp_path: Path) -> None: def test_a_golden_file_added_after_the_release_passes(repo: Path) -> None: folder = _folder(repo) - _ = (folder / "schema-02.json").write_text(dump({"new.entry": "x"})) + name = store.next_file_name(store.group_files(folder)["schema"][-1].name) + _ = (folder / name).write_text(dump({"new.entry": "x"})) manifest = store.read_json(store.manifest_path(repo, VERSION)) manifest["golden"]["history"] = store.with_file( manifest["golden"]["history"], - "schema-02.json", - store.file_digest(folder / "schema-02.json"), + name, + store.file_digest(folder / name), "2026-11-20", ) store.write_json(store.manifest_path(repo, VERSION), manifest) - _commit(repo, "add schema-02") + _commit(repo, f"add {name}") assert store.tree_problems(repo, VERSION) == [] result = _check(repo, ci=True) assert result.returncode == 0, result.stderr diff --git a/tests/test_hashing/golden/store.py b/tests/test_hashing/golden/store.py index 7eaa97eae..5757335af 100644 --- a/tests/test_hashing/golden/store.py +++ b/tests/test_hashing/golden/store.py @@ -235,6 +235,24 @@ def group_files(folder: Path) -> dict[str, list[Path]]: return files +def next_file_name(name: str) -> str: + """The name of a group's next golden file, e.g. ``schema-02.json`` after ``schema-01.json``. + + Args: + name: A golden file name, ``-NN.json``. + + Returns: + The same group with the counter one higher. + + Raises: + GoldenLayoutError: ``name`` is not a golden file name. + """ + match = FILE_NAME.match(name) + if match is None: + raise GoldenLayoutError(f"{name} is not a golden file name") + return f"{match['group']}-{int(match['number']) + 1:02d}.json" + + def load_folder(folder: Path) -> dict[str, dict[str, Any]]: """Read every golden file of a folder. diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index 53b56382d..98c150019 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -7,6 +7,7 @@ from __future__ import annotations import json +import re from pathlib import Path from typing import Any @@ -19,6 +20,18 @@ VERSION = get_default_context().version +# The trial trees copy the repository's golden folder, so the newest file of a group and the +# next counter come from that inventory, never from fixed names. +_REPOSITORY_FILES = store.group_files( + store.golden_folder( + store.PROJECT_ROOT, + store.read_json(store.manifest_path(store.PROJECT_ROOT, VERSION)), + ) +) +NEWEST_SCHEMA = _REPOSITORY_FILES["schema"][-1].name +NEXT_SCHEMA = store.next_file_name(NEWEST_SCHEMA) + + def _write(folder: Path, name: str, values: dict[str, Any]) -> None: folder.mkdir(parents=True, exist_ok=True) _ = (folder / name).write_text(dump(values), encoding="utf-8") @@ -249,15 +262,15 @@ def test_the_repository_pin_is_consistent() -> None: def test_add_writes_new_entries_into_the_open_file_of_their_group(tree: Path) -> None: """A group's newest file that no release lists takes the new entries.""" - dropped = _drop_entries(tree, "schema-01.json", 3) + dropped = _drop_entries(tree, NEWEST_SCHEMA, 3) before = _snapshot(_folder(tree)) main(["--root", str(tree), "--group", "schema"]) after = _snapshot(_folder(tree)) assert set(after) == set(before) - schema = json.loads(after["schema-01.json"]) + schema = json.loads(after[NEWEST_SCHEMA]) assert dropped.items() <= schema.items() - assert {n: b for n, b in after.items() if n != "schema-01.json"} == { - n: b for n, b in before.items() if n != "schema-01.json" + assert {n: b for n, b in after.items() if n != NEWEST_SCHEMA} == { + n: b for n, b in before.items() if n != NEWEST_SCHEMA } assert len(_history(tree)) == 1 assert store.tree_problems(tree, VERSION) == [] @@ -265,18 +278,18 @@ def test_add_writes_new_entries_into_the_open_file_of_their_group(tree: Path) -> def test_add_after_a_release_writes_a_new_file_and_a_new_revision(tree: Path) -> None: """A released file never changes: new entries go to the next ``-NN.json``.""" - dropped = _drop_entries(tree, "schema-01.json", 2) + dropped = _drop_entries(tree, NEWEST_SCHEMA, 2) main(["--root", str(tree), "--release", "0.3.0"]) before = _snapshot(_folder(tree)) main(["--root", str(tree), "--group", "schema"]) after = _snapshot(_folder(tree)) assert {n: after[n] for n in before} == before - assert set(after) - set(before) == {"schema-02.json"} - assert json.loads(after["schema-02.json"]) == dropped + assert set(after) - set(before) == {NEXT_SCHEMA} + assert json.loads(after[NEXT_SCHEMA]) == dropped history = _history(tree) assert [r["release"] for r in history] == [None, "0.3.0", None] assert history[2]["previous"] == history[1]["digest"] - assert list(history[2]["added"]) == ["schema-02.json"] + assert list(history[2]["added"]) == [NEXT_SCHEMA] assert store.tree_problems(tree, VERSION) == [] @@ -284,7 +297,7 @@ def test_add_changes_nothing_when_a_later_group_fails( tree: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Every group is computed before anything is written: files and history stay in step.""" - _ = _drop_entries(tree, "schema-01.json", 1) + _ = _drop_entries(tree, NEWEST_SCHEMA, 1) before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() compute = cases.compute_group @@ -328,7 +341,7 @@ def test_add_writes_nothing_when_every_entry_is_pinned(tree: Path) -> None: def test_add_refuses_a_tree_that_breaks_the_rules(tree: Path) -> None: - path = _folder(tree) / "schema-01.json" + path = _folder(tree) / NEWEST_SCHEMA _ = path.write_text(path.read_text() + " ") manifest = store.manifest_path(tree, VERSION).read_bytes() with pytest.raises(SystemExit, match="nothing written"): @@ -338,7 +351,7 @@ def test_add_refuses_a_tree_that_breaks_the_rules(tree: Path) -> None: def test_remove_then_add_changes_an_unreleased_value(tree: Path) -> None: """The approved-change procedure: ``--remove`` the entry, then ``--add`` writes it.""" - name = "schema-01.json" + name = NEWEST_SCHEMA values = store.read_json(_folder(tree) / name) entry = next(iter(values)) _record(tree, name, {**values, entry: "old value"}) @@ -353,7 +366,7 @@ def test_remove_then_add_changes_an_unreleased_value(tree: Path) -> None: def test_remove_refuses_an_entry_a_release_lists(tree: Path) -> None: main(["--root", str(tree), "--release", "0.3.0"]) - entry = next(iter(store.read_json(_folder(tree) / "schema-01.json"))) + entry = next(iter(store.read_json(_folder(tree) / NEWEST_SCHEMA))) before = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") with pytest.raises(SystemExit, match="released"): main(["--root", str(tree), "--remove", entry]) @@ -362,7 +375,7 @@ def test_remove_refuses_an_entry_a_release_lists(tree: Path) -> None: def test_remove_refuses_an_unknown_entry_and_writes_nothing(tree: Path) -> None: - entry = next(iter(store.read_json(_folder(tree) / "schema-01.json"))) + entry = next(iter(store.read_json(_folder(tree) / NEWEST_SCHEMA))) before = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") with pytest.raises(SystemExit, match="no.such.entry"): main(["--root", str(tree), "--remove", entry, "--remove", "no.such.entry"]) @@ -371,20 +384,20 @@ def test_remove_refuses_an_unknown_entry_and_writes_nothing(tree: Path) -> None: def test_check_reports_a_moved_value_and_writes_nothing(tree: Path) -> None: - name = "schema-01.json" + name = NEWEST_SCHEMA values = store.read_json(_folder(tree) / name) entry = next(iter(values)) _record(tree, name, {**values, entry: "moved"}) before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() - with pytest.raises(SystemExit, match=f"schema:{entry} moved"): + with pytest.raises(SystemExit, match=re.escape(f"schema:{entry} moved")): main(["--root", str(tree), "--check", "--group", "schema"]) after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() assert after == before def test_check_reports_an_unpinned_entry(tree: Path) -> None: - dropped = _drop_entries(tree, "schema-01.json", 1) - with pytest.raises(SystemExit, match=f"schema:{next(iter(dropped))} is computed"): + dropped = _drop_entries(tree, NEWEST_SCHEMA, 1) + with pytest.raises(SystemExit, match=re.escape(f"schema:{next(iter(dropped))} is computed")): main(["--root", str(tree), "--check", "--group", "schema"]) From c05293b87b1851cf87a5e77651c5c0505952b260 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:34:58 -0800 Subject: [PATCH 07/16] style(golden): ruff format (ITL-946) Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/golden/test_golden_store.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index 98c150019..47bc6002a 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -397,7 +397,9 @@ def test_check_reports_a_moved_value_and_writes_nothing(tree: Path) -> None: def test_check_reports_an_unpinned_entry(tree: Path) -> None: dropped = _drop_entries(tree, NEWEST_SCHEMA, 1) - with pytest.raises(SystemExit, match=re.escape(f"schema:{next(iter(dropped))} is computed")): + with pytest.raises( + SystemExit, match=re.escape(f"schema:{next(iter(dropped))} is computed") + ): main(["--root", str(tree), "--check", "--group", "schema"]) From 0e15bf07e51e679074686b7b62d7877b58e69fec Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:41:57 -0800 Subject: [PATCH 08/16] test(golden): name the unlisted trial file after the inventory (ITL-946) CodeRabbit CLI review of c05293b8: test_an_unlisted_file_is_reported wrote arrow-02.json, which a later committed arrow file would already list. Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/golden/test_golden_store.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index 47bc6002a..8604b607f 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -147,8 +147,9 @@ def test_an_edited_listed_file_is_reported(tree: Path) -> None: def test_an_unlisted_file_is_reported(tree: Path) -> None: - _write(_folder(tree), "arrow-02.json", {"new": 1}) - assert "arrow-02.json is in" in " ".join(store.tree_problems(tree, VERSION)) + name = store.next_file_name(_REPOSITORY_FILES["arrow"][-1].name) + _write(_folder(tree), name, {"new": 1}) + assert f"{name} is in" in " ".join(store.tree_problems(tree, VERSION)) def test_a_missing_listed_file_is_reported(tree: Path) -> None: From 2dcb6d64aa159a580e50bd05b88e8e0d5423f6de Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 13:50:18 -0800 Subject: [PATCH 09/16] fix(golden): the soundness check covers duplicate names and unknown groups (ITL-946) CodeRabbit CLI review of 0e15bf07: tree_problems did not read the folder, so a recorded file repeating an entry name, or a file of a group outside cases.GROUPS, passed it and --release could pin it. folder_problems now reports both. Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/golden/store.py | 25 +++++++++++++------ .../test_hashing/golden/test_golden_store.py | 18 +++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/tests/test_hashing/golden/store.py b/tests/test_hashing/golden/store.py index 5757335af..855be911e 100644 --- a/tests/test_hashing/golden/store.py +++ b/tests/test_hashing/golden/store.py @@ -36,6 +36,8 @@ from packaging.version import InvalidVersion, Version +from tests.test_hashing.golden.cases import GROUPS + PROJECT_ROOT = Path(__file__).resolve().parents[3] CONTEXTS_DIR = Path("src/orcapod/contexts") @@ -449,7 +451,8 @@ def folder_problems(history: Sequence[Mapping[str, Any]], folder: Path) -> list[ """What breaks the agreement between a history and its golden folder. Every listed file exists and has the listed sha256; every file in the folder is listed - by exactly one revision. + by exactly one revision, is named ``-NN.json`` for a group of ``cases.GROUPS``, + and holds a JSON object; and no entry name appears in two files. Args: history: A history without ``history_problems``. @@ -479,12 +482,20 @@ def folder_problems(history: Sequence[Mapping[str, Any]], folder: Path) -> list[ else "run generate_golden_hashes.py to record it" ) ) - if folder.is_dir(): - for path in sorted(folder.iterdir()): - if path.name not in listed: - problems.append(f"{path.name} is in {folder} but no revision lists it") - else: - problems.append(f"the golden folder {folder} does not exist") + if not folder.is_dir(): + return [*problems, f"the golden folder {folder} does not exist"] + for path in sorted(folder.iterdir()): + if path.name not in listed: + problems.append(f"{path.name} is in {folder} but no revision lists it") + match = FILE_NAME.match(path.name) + if match is not None and match["group"] not in GROUPS: + problems.append( + f"{path.name} belongs to no known group (cases.GROUPS: {list(GROUPS)})" + ) + try: + _ = load_folder(folder) + except GoldenLayoutError as exc: + problems.append(str(exc)) return problems diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index 8604b607f..105b0be3f 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -152,6 +152,24 @@ def test_an_unlisted_file_is_reported(tree: Path) -> None: assert f"{name} is in" in " ".join(store.tree_problems(tree, VERSION)) +def test_a_recorded_file_repeating_an_entry_name_is_reported(tree: Path) -> None: + """A duplicate entry name in correctly recorded files is a problem, and no release.""" + entry, value = next(iter(store.read_json(_folder(tree) / NEWEST_SCHEMA).items())) + _record(tree, NEXT_SCHEMA, {entry: value}) + assert any("pinned twice" in p for p in store.tree_problems(tree, VERSION)) + with pytest.raises(SystemExit, match="pinned twice"): + main(["--root", str(tree), "--release", "0.3.0"]) + + +def test_a_recorded_file_of_an_unknown_group_is_reported(tree: Path) -> None: + """A file whose group is not one of ``cases.GROUPS`` is a problem, even when recorded.""" + _record(tree, "obsolete-01.json", {"obsolete.entry": "x"}) + assert any( + "obsolete-01.json" in p and "known group" in p + for p in store.tree_problems(tree, VERSION) + ) + + def test_a_missing_listed_file_is_reported(tree: Path) -> None: (_folder(tree) / "torch-01.json").unlink() assert any( From c1289181bd51916978bb8f49ed14cf59137fdbc8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 14:04:12 -0800 Subject: [PATCH 10/16] test(golden): pin the hash_samples cases no golden case covered (ITL-946) Developer decision, 2026-11-15 (option A): new golden cases for what the older sample files pinned and no case did, added with --add (55 entries, additions only, no existing value changes): - semantic sample_value: 26 plain Python values of the hash sample file (bytes values follow with ITL-1152) - semantic annotation: 7 annotations over File, Directory and int | str - schema semantic_schema: 8 Schema semantic hashes (equal to hash_samples/schema_hash_golden.json) - semantic annotated_function: signature info and value hashes of 7 annotated functions defined in cases.py The hash_samples tests are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/contexts/data/v1.json | 6 +- tests/test_hashing/golden/cases.py | 156 ++++++++++++++++++ .../golden/data/v1/schema-01.json | 8 + .../golden/data/v1/semantic-01.json | 47 ++++++ 4 files changed, 214 insertions(+), 3 deletions(-) diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index 8bc8af711..1e88c54d2 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -236,12 +236,12 @@ "arrow-01.json": "d62d1fd8ef374e4fd55a82d7eeb09f3ed5906b9de6fc96b6b1596edcb1579da8", "datagram-01.json": "1151c8d391f6f7303657da83720e4fcfc8c5b5082e9cd670ef51ba8886575ca9", "pipeline-01.json": "e4e4e71b4ef9bdd9211b5c5dc15db03ee333f66af606a832d1b5fccfa21f44da", - "schema-01.json": "315a5206497f86a69029810800045e542e8f85ef733c3f1108ef2e7526024a72", - "semantic-01.json": "187c1845303e90fd79f2f8afa1a4a54addfb0be91f087570e56bfc130e2cffcd", + "schema-01.json": "f1bfda8bc4da3ecfc7973998c5579b5a3f151f7ec1f9c75c20ac5ad5244856ae", + "semantic-01.json": "f47a5ca0597163ea30bfb86e6ef79a3d657b524a4b67a477c6ff7a9160007fc1", "torch-01.json": "99af07350bc23a0200cecfd786ccd6c17985c2270ad5e3f44aca74273a1bcdc2" }, "previous": null, - "digest": "2bc703287b8342d9187c491123ca69665485d8cb0ded4f6bb814cdfb33037917" + "digest": "af319bff27032545759de7aba90d02366f826c14b7592e36f269de1a0dd3abc6" } ] }, diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index d319757d8..10f52e394 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -37,6 +37,7 @@ import pyarrow as pa import pydantic +import orcapod as op from orcapod import dataclass as structural_dataclass from orcapod import function_pod @@ -2062,3 +2063,158 @@ def stream(ks: list[list[int]], xs: list[int]) -> ArrowTableStream: "as_given": stream([[2], [1]], [20, 10]).content_hash().to_string(), "reversed": stream([[1], [2]], [10, 20]).content_hash().to_string(), } + + +# --------------------------------------------------------------------------- +# Cases brought in from tests/test_hashing/hash_samples (ITL-946) +# +# The values, annotations, schemas and annotated functions that the older sample files +# pinned and no case above did. The functions are defined here, so their pins carry this +# module's path. DO NOT MOVE OR RENAME them. +# --------------------------------------------------------------------------- + +SAMPLE_VALUES: dict[str, Any] = { + "int_42": 42, + "float_pi": 3.14159, + "float_negative_e": -2.71828, + "str_hello": "hello", + "str_hello_world": "Hello, World!", + "str_special_chars": "Special chars: !@#$%^&*()", + "str_unicode": "Unicode: 你好, Привет, こんにちは", + "list_strs": ["a", "b", "c"], + "list_mixed": [1, "a", True], + "set_empty": set(), + "set_strs": {"a", "b", "c"}, + "frozenset_empty": frozenset(), + "tuple_empty": (), + "dict_one": {"a": 1}, + "dict_other_order": {"b": 1, "a": 2}, + "dict_nested": {"nested": {"a": 1, "b": 2}}, + "nested_lists": [1, [2, [3, [4, [5]]]]], + "nested_dicts": {"a": {"b": {"c": {"d": {"e": 42}}}}}, + "nested_dict_list": {"a": [1, 2, {"b": [3, 4, {"c": 5}]}]}, + "list_of_dicts": [{"a": 1}, {"b": 2}, {"c": [3, 4, 5]}], + "dict_keys_values": {"keys": ["a", "b", "c"], "values": [1, 2, 3]}, + "list_of_dicts_with_lists": [{"a": 1, "b": [2, 3]}, {"c": 4, "d": [5, 6]}], + "dict_users": {"users": [{"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}]}, + "dict_points_labels": { + "data": {"points": [[1, 2], [3, 4], [5, 6]], "labels": ["A", "B", "C"]} + }, + "ordered_dict": collections.OrderedDict([("a", 1), ("b", 2), ("c", 3)]), + "lists_and_dict": [[1, 2], [3, 4], {"a": [5, 6], "b": [7, 8]}], +} + + +@case("semantic", "sample_value") +def semantic_sample_values() -> dict[str, Any]: + """Semantic hashes of the plain Python values of the older hash sample file.""" + hasher = _ctx().semantic_hasher + return {k: hasher.hash_object(v).to_string() for k, v in SAMPLE_VALUES.items()} + + +def _annotations() -> dict[str, Any]: + import typing + + import orcapod as op + + return { + "file": op.File, + "directory": op.Directory, + "list_file": list[op.File], + "dict_str_file": dict[str, op.File], + "int_or_str": int | str, + "file_or_none": op.File | None, + "optional_file": typing.Optional[op.File], # noqa: UP045 (the typing form is the case) + } + + +@case("semantic", "annotation") +def semantic_annotations() -> dict[str, Any]: + """Semantic hashes of type annotations over Orcapod's logical types (ITL-638).""" + hasher = _ctx().semantic_hasher + return {k: hasher.hash_object(t).to_string() for k, t in _annotations().items()} + + +@case("schema", "semantic_schema") +def schema_semantic_hashes() -> dict[str, Any]: + """Semantic hashes of ``Schema`` objects with primitive and logical types.""" + import orcapod as op + from orcapod.types import Schema + + schemas = { + "x_int": Schema({"x": int}), + "x_int_y_str": Schema({"x": int, "y": str}), + "f_file": Schema({"f": op.File}), + "d_directory": Schema({"d": op.Directory}), + "p_path": Schema({"p": op.Path}), + "x_int_f_file": Schema({"x": int, "f": op.File}), + "f_file_d_directory": Schema({"f": op.File, "d": op.Directory}), + "f_optional_file": Schema({"f": op.File | None}), + } + hasher = _ctx().semantic_hasher + return {k: hasher.hash_object(s).to_string() for k, s in schemas.items()} + + +def golden_annotated_none(): + """Function without annotations (ITL-946). DO NOT MOVE.""" + return None + + +def golden_annotated_builtin(x: int, y: str) -> float: + """Function annotated with builtins (ITL-946). DO NOT MOVE.""" + return float(x) + + +def golden_annotated_file_param(f: op.File) -> str: + """Function taking a ``File`` (ITL-946). DO NOT MOVE.""" + return str(f) + + +def golden_annotated_file_return(s: str) -> op.File: + """Function returning a ``File`` (ITL-946). DO NOT MOVE.""" + return op.File(s) # pyright: ignore[reportArgumentType] (never called; only hashed) + + +def golden_annotated_file_list(files: list[op.File]) -> list[str]: + """Function taking a list of ``File`` (ITL-946). DO NOT MOVE.""" + return [] + + +def golden_annotated_optional_file(f: op.File | None) -> op.File | None: + """Function taking and returning an optional ``File`` (ITL-946). DO NOT MOVE.""" + return f + + +def golden_annotated_mixed(f: op.File, n: int) -> op.Directory: + """Function mixing ``File``, ``int`` and ``Directory`` (ITL-946). DO NOT MOVE.""" + return op.Directory(str(f)) # pyright: ignore[reportArgumentType] (never called) + + +@case("semantic", "annotated_function") +def semantic_annotated_functions() -> dict[str, Any]: + """Hashes of annotated functions: the extracted signature info and the function value.""" + from orcapod.hashing.semantic_hashing.function_info_extractors import ( + FunctionSignatureExtractor, + ) + + functions = { + "none": golden_annotated_none, + "builtin": golden_annotated_builtin, + "file_param": golden_annotated_file_param, + "file_return": golden_annotated_file_return, + "file_list": golden_annotated_file_list, + "optional_file": golden_annotated_optional_file, + "mixed": golden_annotated_mixed, + } + hasher = _ctx().semantic_hasher + extractor = FunctionSignatureExtractor( + include_module=True, + include_defaults=True, + type_converter=_ctx().type_converter, + ) + out: dict[str, Any] = {} + for name, fn in functions.items(): + info = extractor.extract_function_info(fn) + out[f"{name}.info"] = hasher.hash_object(info).to_string() + out[f"{name}.object"] = hasher.hash_object(fn).to_string() + return out diff --git a/tests/test_hashing/golden/data/v1/schema-01.json b/tests/test_hashing/golden/data/v1/schema-01.json index b48a97a86..bf4ef4b81 100644 --- a/tests/test_hashing/golden/data/v1/schema-01.json +++ b/tests/test_hashing/golden/data/v1/schema-01.json @@ -59,6 +59,14 @@ "polars_single_field.dataframe.schema_hash": "semantic_v1:3bea81b6438f30365fefa091067fca4b0b76149083d1a4cc6298a82e7ddde74c", "polars_single_field.series.arrow_type": "extension[large_binary]", "polars_single_field.series.schema_hash": "semantic_v1:796fcf1abfe4c5224e86110f47d1adbc639a95a500bad11798844c4809cd904e", + "semantic_schema.d_directory": "semantic_v1:0fbb5bba5b858fc4677b791e2f8008cf01324ffb5eab6dd0eedcacb1f72389c0", + "semantic_schema.f_file": "semantic_v1:d4654f89159e297dbc7800dbeb753b1487abbbc3f3277dd3b0305e4685e2ad88", + "semantic_schema.f_file_d_directory": "semantic_v1:77baccb8f76522e8e18ca0c74c9226d4082551502bd1b17489e2ead7fff033c5", + "semantic_schema.f_optional_file": "semantic_v1:9131e3d1da9847ee3a3cb700b8c8ce18cb03dd2ecf505d244847d2add81399cb", + "semantic_schema.p_path": "semantic_v1:609252265570863f631a437c6d6d95b080a710fd18264eebd81446c49f3cfb35", + "semantic_schema.x_int": "semantic_v1:e2eed73f4a554064d6c3acd3d3b39b81d2ea6164e9326c1412ed820cde586f2c", + "semantic_schema.x_int_f_file": "semantic_v1:87e6966aaf164468d7714a17cccc83addc1be9b5292c4a48bd57d3c081b5b2b9", + "semantic_schema.x_int_y_str": "semantic_v1:1bb91fe0f62d9a633d9602fdc8d1893e60d5819f6693175f4fc5661f90bf8d09", "single_field.bool.arrow_type": "bool", "single_field.bool.schema_hash": "semantic_v1:d65d43b0876cd839e9c53022a3e523e8236ca0df552bd3fb7a6e199e259e6839", "single_field.bytes.arrow_type": "large_binary", diff --git a/tests/test_hashing/golden/data/v1/semantic-01.json b/tests/test_hashing/golden/data/v1/semantic-01.json index 66e2d16e6..abfc79ce8 100644 --- a/tests/test_hashing/golden/data/v1/semantic-01.json +++ b/tests/test_hashing/golden/data/v1/semantic-01.json @@ -1,4 +1,25 @@ { + "annotated_function.builtin.info": "semantic_v1:62647b5236b91b669a128d661865cb351133b9508ddab1aea8e563be50530977", + "annotated_function.builtin.object": "semantic_v1:62647b5236b91b669a128d661865cb351133b9508ddab1aea8e563be50530977", + "annotated_function.file_list.info": "semantic_v1:5c625823b4fe0c6616667d420be2f347c029cde68a0659c51e4d5f119eec0f0a", + "annotated_function.file_list.object": "semantic_v1:5c625823b4fe0c6616667d420be2f347c029cde68a0659c51e4d5f119eec0f0a", + "annotated_function.file_param.info": "semantic_v1:5bf627e036e5826093d17f5ca3d4c86dea474d14965f0f367df869cc5b48fc39", + "annotated_function.file_param.object": "semantic_v1:5bf627e036e5826093d17f5ca3d4c86dea474d14965f0f367df869cc5b48fc39", + "annotated_function.file_return.info": "semantic_v1:53e59f17f3adb6a0746419028194a1293280e94390a772d98826788b643a0ecb", + "annotated_function.file_return.object": "semantic_v1:53e59f17f3adb6a0746419028194a1293280e94390a772d98826788b643a0ecb", + "annotated_function.mixed.info": "semantic_v1:5b1df0d4adb16c89738315a9260ac391dd3c8120e5fa0f51c99c827a7e3713c1", + "annotated_function.mixed.object": "semantic_v1:5b1df0d4adb16c89738315a9260ac391dd3c8120e5fa0f51c99c827a7e3713c1", + "annotated_function.none.info": "semantic_v1:64999822b66304279c0ac19a4e76f7d001fb65830440b91f3723b48f6f09bf5d", + "annotated_function.none.object": "semantic_v1:64999822b66304279c0ac19a4e76f7d001fb65830440b91f3723b48f6f09bf5d", + "annotated_function.optional_file.info": "semantic_v1:b3550eec874ee9c786e13fa0a8e38706064d1f548f14cb8ea6ae470b3cd7073f", + "annotated_function.optional_file.object": "semantic_v1:b3550eec874ee9c786e13fa0a8e38706064d1f548f14cb8ea6ae470b3cd7073f", + "annotation.dict_str_file": "semantic_v1:d2ec6ad67d11d40506d4f9235c4e076c3b230e3f75c23dd9f117402a66b2074f", + "annotation.directory": "semantic_v1:8050d1b8d85bf525f182125fe61f9da234d5dad67e43b69416100f743f6166a1", + "annotation.file": "semantic_v1:a2fd9d1b211cc95352b7787760040ec7da93c232e5c8c4005a24c36df08926fb", + "annotation.file_or_none": "semantic_v1:553ef731133e3935c9efccf48a511bee41af5717f2d5652a5029022ed20ab22c", + "annotation.int_or_str": "semantic_v1:838cabab55983dce341eaaee078ea5eda7f8a2a31ca6f8a0ab53f9ea9528b73a", + "annotation.list_file": "semantic_v1:df514aaa5ce466d71578b354cdf70c051d12b3501fb98b02de129bd21683c1a8", + "annotation.optional_file": "semantic_v1:553ef731133e3935c9efccf48a511bee41af5717f2d5652a5029022ed20ab22c", "function.stored_signature.ratio": "semantic_v1:7804cfbf8935f990b4d7941be5dae40aa9b2ff0bc94a3eb81d97a6db4b0dce77", "function.stored_signature.scaled": "semantic_v1:143251902c84387a1acb97e0076d00a3d17c0b4671c3a3449271750c8c38866e", "function.value.defaults": "semantic_v1:a55248ece1fc5e1a3c19860c390217a2325b843a0469982749706f38175d4366", @@ -21,6 +42,32 @@ "polars_value.nulls_and_temporal": "polars_dataframe_v1:fccfc33b3a7f171257b87ff20d260d2221085e48e3740d567681414e5ace589a", "polars_value.series": "polars_series_v1:9ff68c5024750115e606f8e370ceaa55bcb3305227dc6ad45111adaf5a620d88", "polars_value.simple": "polars_dataframe_v1:92002a1ced22c899f5662ceb4ed2e435ea2d0dddcf28b489ac3f8c8d5226aadf", + "sample_value.dict_keys_values": "semantic_v1:af63a0d00f3d3b744714e12bc4b6003b2395e296fe23c7a02c3352794f156cdb", + "sample_value.dict_nested": "semantic_v1:635b42bee92bc9e78396200dd770fffce472fc9b768d9ea1b73118367efaf084", + "sample_value.dict_one": "semantic_v1:015abd7f5cc57a2dd94b7590f04ad8084273905ee33ec5cebeae62276a97f862", + "sample_value.dict_other_order": "semantic_v1:d3626ac30a87e6f7a6428233b3c68299976865fa5508e4267c5415c76af7a772", + "sample_value.dict_points_labels": "semantic_v1:a553155144ad09460431dbbaacf7fe9a2910dd03ef969af8a9a529aae1c62c4f", + "sample_value.dict_users": "semantic_v1:32c13fff89d73e616960914df441b14c74671ec89d9589c2bcdba76f436b8126", + "sample_value.float_negative_e": "semantic_v1:43dc3d0d4b9e9bb2a2dfcf9696b0d49fe1f4fa10ac48586ef40e0b037f9ebcfe", + "sample_value.float_pi": "semantic_v1:c0740dd25c9de39b9c8d5ab452e8b69bcc0bf86f2a60ed7e527e79d0a3035852", + "sample_value.frozenset_empty": "semantic_v1:469289aca5c121cc9cec7cbde2df99827e5f310b646d4b7367760ea5f3a72bc7", + "sample_value.int_42": "semantic_v1:73475cb40a568e8da8a045ced110137e159f890ac4da883b6b17dc651b3a8049", + "sample_value.list_mixed": "semantic_v1:8554b19db94b8f9b64793d82a73098ea8af738886c738a2183682ca0ce08489a", + "sample_value.list_of_dicts": "semantic_v1:d72ea3c2223310171bdcb8e1787ef9f67f0e0c9a96616ad1686ad98b9d1c5e9d", + "sample_value.list_of_dicts_with_lists": "semantic_v1:d88e8543683b6b82bd7ce2225762d5a908c9229d122f8a52ca5c64169aec9a9b", + "sample_value.list_strs": "semantic_v1:fa1844c2988ad15ab7b49e0ece09684500fad94df916859fb9a43ff85f5bb477", + "sample_value.lists_and_dict": "semantic_v1:aa4dc10726bca2bd65f5c953962b432c3aee33b37fe3fbc05b1514307d440a92", + "sample_value.nested_dict_list": "semantic_v1:2252fc2a778c83e9c9308bd3ff747afd8125a902ff23282f4324d48424987c5d", + "sample_value.nested_dicts": "semantic_v1:c6d14482c460f03004d38484d71298a40eb8ec52e264851222c8bda2f4982541", + "sample_value.nested_lists": "semantic_v1:fdc65463c02cb1c8788344fba97041b67fce86c4726d4e10fe135af24b9c6801", + "sample_value.ordered_dict": "semantic_v1:e6a3385fb77c287a712e7f406a451727f0625041823ecf23bea7ef39b2e39805", + "sample_value.set_empty": "semantic_v1:469289aca5c121cc9cec7cbde2df99827e5f310b646d4b7367760ea5f3a72bc7", + "sample_value.set_strs": "semantic_v1:dc64a5a5142e9849961bf58ac4900f662d8d1fcf81261063f46ac5c475c0f4e7", + "sample_value.str_hello": "semantic_v1:5aa762ae383fbb727af3c7a36d4940a5b8c40a989452d2304fc958ff3f354e7a", + "sample_value.str_hello_world": "semantic_v1:cc82ebbcf8b60a5821d1c51c72cd79380ecea47de343ccb3b158938a2b3bf764", + "sample_value.str_special_chars": "semantic_v1:bbfa5afce2bf76f5520f0acafbb986c6da70dc507717450af24f91e8d01832c9", + "sample_value.str_unicode": "semantic_v1:2dd7db78b058c582953d493cc25b725d3cd7671c246c30440cebb5effc7c9e91", + "sample_value.tuple_empty": "semantic_v1:095502da3bf5c96b476e96c1304bbf052aec9482f7c46ed649e74d9d9bc172a7", "type.bool": "semantic_v1:cd8f971312b872f0e9a3ef40aa58d6bb537258e72a9a142e0cf712f0a35020e3", "type.bytes": "semantic_v1:f65cea8d1f29ca5b96d67caf1ced2f630b509e32574a7b78b9071958818a1c59", "type.dataclass": "semantic_v1:86baa1ef8dbf2518c6dcf7d188f6e20474248e9d8bc08d22b7fa0a4ed98bf8a8", From aae1dfcb03e9bd9667762883abfce39b727a84ad Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 15:02:23 -0800 Subject: [PATCH 11/16] test(golden): pin the bytes sample values at today's encoding (ITL-946) The four bytes and bytearray values of the older hash sample file, as sample_value.* entries. Additions only; ITL-1152 changes them later while the file is still unreleased. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/contexts/data/v1.json | 4 ++-- tests/test_hashing/golden/cases.py | 4 ++++ tests/test_hashing/golden/data/v1/semantic-01.json | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index 1e88c54d2..28ddf132d 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -237,11 +237,11 @@ "datagram-01.json": "1151c8d391f6f7303657da83720e4fcfc8c5b5082e9cd670ef51ba8886575ca9", "pipeline-01.json": "e4e4e71b4ef9bdd9211b5c5dc15db03ee333f66af606a832d1b5fccfa21f44da", "schema-01.json": "f1bfda8bc4da3ecfc7973998c5579b5a3f151f7ec1f9c75c20ac5ad5244856ae", - "semantic-01.json": "f47a5ca0597163ea30bfb86e6ef79a3d657b524a4b67a477c6ff7a9160007fc1", + "semantic-01.json": "7fd0ccad63a7e3f83314b27e9d6df87a8cb15694758f92e517a0d9c988f98a53", "torch-01.json": "99af07350bc23a0200cecfd786ccd6c17985c2270ad5e3f44aca74273a1bcdc2" }, "previous": null, - "digest": "af319bff27032545759de7aba90d02366f826c14b7592e36f269de1a0dd3abc6" + "digest": "e20d0af197e1d2eca72f0524e626cb4a46b37d261cc7a06ea5768cce65d153a5" } ] }, diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 10f52e394..d5dfaaa76 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -2081,6 +2081,10 @@ def stream(ks: list[list[int]], xs: list[int]) -> ArrowTableStream: "str_hello_world": "Hello, World!", "str_special_chars": "Special chars: !@#$%^&*()", "str_unicode": "Unicode: 你好, Привет, こんにちは", + "bytes_hello": b"hello", + "bytes_binary": b"\x00\x01\x02\x03", + "bytearray_hello_world": bytearray(b"hello world"), + "bytearray_abc": bytearray([65, 66, 67]), "list_strs": ["a", "b", "c"], "list_mixed": [1, "a", True], "set_empty": set(), diff --git a/tests/test_hashing/golden/data/v1/semantic-01.json b/tests/test_hashing/golden/data/v1/semantic-01.json index abfc79ce8..6f30d1e20 100644 --- a/tests/test_hashing/golden/data/v1/semantic-01.json +++ b/tests/test_hashing/golden/data/v1/semantic-01.json @@ -42,6 +42,10 @@ "polars_value.nulls_and_temporal": "polars_dataframe_v1:fccfc33b3a7f171257b87ff20d260d2221085e48e3740d567681414e5ace589a", "polars_value.series": "polars_series_v1:9ff68c5024750115e606f8e370ceaa55bcb3305227dc6ad45111adaf5a620d88", "polars_value.simple": "polars_dataframe_v1:92002a1ced22c899f5662ceb4ed2e435ea2d0dddcf28b489ac3f8c8d5226aadf", + "sample_value.bytearray_abc": "semantic_v1:9117877bc5c6f08d682a7d9bb4e67b984d316921faf09bf224a5c9bedb3ebb20", + "sample_value.bytearray_hello_world": "semantic_v1:95b243c4c8b3e696c28c0ffc82ff70b87f505f487f57ed5e7595a70b21a6a6ff", + "sample_value.bytes_binary": "semantic_v1:cedf4873ac6b0aba6a4b8dc6574c5564ee4b971a0be7fd8ad76582d1377e9995", + "sample_value.bytes_hello": "semantic_v1:7375513f6d0b5271db377b03ad832f51966a263f2ceb70a747b17c076b273ede", "sample_value.dict_keys_values": "semantic_v1:af63a0d00f3d3b744714e12bc4b6003b2395e296fe23c7a02c3352794f156cdb", "sample_value.dict_nested": "semantic_v1:635b42bee92bc9e78396200dd770fffce472fc9b768d9ea1b73118367efaf084", "sample_value.dict_one": "semantic_v1:015abd7f5cc57a2dd94b7590f04ad8084273905ee33ec5cebeae62276a97f862", From 8f9605c99123429f4ea31f024a9f7deda653edfd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 16:05:53 -0800 Subject: [PATCH 12/16] feat(golden): the Release workflow refuses a release the golden history does not record (ITL-946) CodeRabbit CLI review of aae1dfcb: the Release workflow's tests run before the tag exists, under a development version, so a release that skipped generate_golden_hashes.py --release could be tagged and published. Developer decision 2026-11-15 (option A): scripts/check_golden_tags.py --release-version X.Y.Z (stdlib, reads HEAD) fails unless HEAD pins that release under the current context, the manifest pin is set and the pinned digest is that release's revision; release.yml's build job runs it after checkout and before the tag is created. RELEASING.md and the golden-hashes page describe it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 5 +++ RELEASING.md | 4 +- docs/reference/golden-hashes.md | 14 ++++++- scripts/check_golden_tags.py | 71 +++++++++++++++++++++++++++++++++ tests/test_check_golden_tags.py | 62 ++++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5501bfba8..1e1ac1218 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,6 +86,11 @@ jobs: fetch-depth: 0 # required: hatch-vcs reads tag to set version ref: ${{ inputs.branch }} + - name: Check the golden release step + # Nothing is tagged or published unless the release pull request ran + # generate_golden_hashes.py --release for this version (ITL-946). + run: python3 scripts/check_golden_tags.py --release-version "${{ steps.normalize.outputs.version }}" + - name: Configure git identity run: | git config user.name "github-actions[bot]" diff --git a/RELEASING.md b/RELEASING.md index d7668a122..685d61428 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -66,7 +66,9 @@ Do these on `metamorphic-labs` `main` before anything is pushed: `src/orcapod/contexts/released_manifests.json`, and at the data context's first release sets `released_in` and the `manifest` pin. It computes no hash and refuses a version already released. Commit both files with the release pull request; the release is tagged - on a commit that contains them. + on a commit that contains them. The Release workflow's build job checks this before it + creates the tag (`scripts/check_golden_tags.py --release-version X.Y.Z`) and stops if the + release is not recorded. 3. **Check that `main` is green** — the `Run Tests` workflow on the commit you are releasing. The Release workflow re-runs the unit tests on Python 3.11, 3.12 and 3.13 and the license check, but not the PostgreSQL or SpiralDB integration jobs. diff --git a/docs/reference/golden-hashes.md b/docs/reference/golden-hashes.md index 420940411..e21072c25 100644 --- a/docs/reference/golden-hashes.md +++ b/docs/reference/golden-hashes.md @@ -209,7 +209,7 @@ of the whole tree is caught only by records outside it, the tags and PyPI. | 8 | A new file repeats a released entry name with another value | either | Duplicate-name check | the name is reported with both files | | 9 | A golden entry is marked `skip` or `xfail` to silence it | either | No-skip rule | the golden test reports the mark | | 10 | A hasher definition or library in a released `v1.json` is edited | accidental | Manifest pin | the manifest without `golden` no longer matches `manifest` | -| 11 | A release is cut without the release step | accidental | Final-release check | a final version has no `releases` line | +| 11 | A release is cut without the release step | accidental | Release gate, final-release check | the Release workflow stops before tagging; a final version has no `releases` line | | 12 | A released file is edited and its hash in `added` updated to match | deliberate | Chain | that revision's digest no longer computes, or the next revision's `previous` no longer matches | | 13 | As 12, with every later digest computed again | deliberate | Release pins | the pinned release digest is no longer in the chain | | 14 | As 13, with the pins rewritten too | deliberate | Tag check, PyPI | the file and the history differ from the release tag; the published `v1.json` shows the original chain | @@ -289,6 +289,18 @@ PyPI protect everything against a rewrite of the tree. manifest. - **Protects against:** a release cut without the release step (11). +### Release gate + +`scripts/check_golden_tags.py --release-version X.Y.Z`, a step of the Release workflow's +build job before the tag is made; tested by `tests/test_check_golden_tags.py` + +- **Checks:** HEAD's `released_manifests.json` pins release `X.Y.Z` of the current data + context, its `manifest` pin is set, and the pinned digest is the release revision of + `X.Y.Z` in HEAD's golden history. It fails without git. +- **Protects against:** a release cut without the release step (11). The final-release + check cannot catch it in the Release workflow, whose tests run before the tag exists, + under a development version. + ### Tag check `scripts/check_golden_tags.py`, a step of the `Run Tests` workflow; tested by diff --git a/scripts/check_golden_tags.py b/scripts/check_golden_tags.py index 518f1e4fe..0205b65eb 100644 --- a/scripts/check_golden_tags.py +++ b/scripts/check_golden_tags.py @@ -183,6 +183,53 @@ def check_tag(repo: Path, tag: str, version: str) -> list[str] | None: return problems +def check_release(repo: Path, release: str) -> list[str]: + """Check that HEAD records a release of the current data context, before it is tagged. + + The Release workflow runs this before it creates the tag: HEAD's + ``released_manifests.json`` pins ``release`` under the current context's + ``releases``, its manifest pin is set, and the pinned digest is the digest of the + release revision of ``release`` in HEAD's golden history. + + Args: + repo: Repository root. + release: The library version being released, without a leading ``v``. + + Returns: + One sentence per problem; empty when HEAD records the release. + """ + version = _current_version(repo) + step = ( + f"run generate_golden_hashes.py --release {release} in the release pull request" + ) + pins = _show_json(repo, "HEAD", PINS) + pin = pins.get(version) if isinstance(pins, dict) else None + releases = pin.get("releases") if isinstance(pin, dict) else None + if not isinstance(pin, dict) or not isinstance(releases, dict): + return [f"HEAD has no pin of data context {version}: {step}"] + if release not in releases: + return [ + f"HEAD does not record release {release} of data context {version}: {step}" + ] + problems: list[str] = [] + if pin.get("manifest") is None: + problems.append(f"the manifest of data context {version} is not pinned: {step}") + manifest = _show_json(repo, "HEAD", CONTEXTS / "data" / f"{version}.json") + golden = manifest.get("golden") if isinstance(manifest, dict) else None + history = golden.get("history") if isinstance(golden, dict) else None + revisions = [ + revision + for revision in (history if isinstance(history, list) else []) + if isinstance(revision, dict) and revision.get("release") == release + ] + if [revision.get("digest") for revision in revisions] != [releases[release]]: + problems.append( + f"the pinned digest of {release} is not the release revision of {release} " + "in the golden history" + ) + return problems + + def main(argv: Sequence[str] | None = None) -> int: """Run the check. @@ -196,7 +243,31 @@ def main(argv: Sequence[str] | None = None) -> int: description="Check HEAD against every release tag of the current data context." ) _ = parser.add_argument("--repo", type=Path, default=REPO_ROOT) + _ = parser.add_argument( + "--release-version", + metavar="X.Y.Z", + help=( + "instead of comparing with the tags, check that HEAD records this release " + "(the Release workflow's gate before it tags); fails without git" + ), + ) args = parser.parse_args(argv) + if args.release_version is not None: + try: + problems = check_release(args.repo, args.release_version.removeprefix("v")) + except GitUnavailableError as exc: + print(f"error: cannot read HEAD: {exc}", file=sys.stderr) + return 1 + if problems: + print( + f"release {args.release_version} is not recorded in the golden history:", + file=sys.stderr, + ) + for problem in problems: + print(f" {problem}", file=sys.stderr) + return 1 + print(f"HEAD records release {args.release_version} in the golden history") + return 0 in_ci = os.environ.get("CI", "").lower() == "true" try: tags = _require_history(args.repo) diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py index 130df288b..62a962c12 100644 --- a/tests/test_check_golden_tags.py +++ b/tests/test_check_golden_tags.py @@ -218,3 +218,65 @@ def test_a_repository_without_tags_fails_in_ci(tmp_path: Path) -> None: result = _check(root, ci=True) assert result.returncode == 1 assert "no tags" in result.stderr + + +# --------------------------------------------------------------------------- +# --release-version: the Release workflow's gate before a tag is made +# --------------------------------------------------------------------------- + + +def _gate(repo: Path, version: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--repo", + str(repo), + "--release-version", + version, + ], + capture_output=True, + text=True, + check=False, + env=_env(ci=False, ceiling=repo.parent), + ) + + +def test_the_gate_passes_a_recorded_release(repo: Path) -> None: + result = _gate(repo, "0.3.0") + assert result.returncode == 0, result.stderr + assert "0.3.0" in result.stdout + + +def test_the_gate_refuses_a_release_without_its_golden_step(repo: Path) -> None: + result = _gate(repo, "0.3.1") + assert result.returncode == 1 + assert "generate_golden_hashes.py --release 0.3.1" in result.stderr + + +def test_the_gate_refuses_an_open_data_context(tmp_path: Path) -> None: + root = tmp_path / "repo" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") + _git(root.parent, "init", "-q", str(root)) + _commit(root, "open data context") + result = _gate(root, "0.3.0") + assert result.returncode == 1 + assert "0.3.0" in result.stderr + + +def test_the_gate_refuses_a_pin_that_is_not_the_release_revision(repo: Path) -> None: + pins = store.read_json(store.pins_path(repo)) + pins[VERSION]["releases"]["0.3.0"] = "0" * 64 + store.write_json(store.pins_path(repo), pins) + _commit(repo, "tamper with the pin") + result = _gate(repo, "0.3.0") + assert result.returncode == 1 + assert "release revision" in result.stderr + + +def test_the_gate_fails_outside_a_repository_even_locally(tmp_path: Path) -> None: + plain = tmp_path / "plain" + store.copy_tree(store.PROJECT_ROOT, plain, VERSION) + result = _gate(plain, "0.3.0") + assert result.returncode == 1 From f9e2bfa408410d735cb5e2f2ee29c35beeb176cf Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 16:14:36 -0800 Subject: [PATCH 13/16] fix(golden): an edited unreleased file's problem says to restore it (ITL-946) CodeRabbit CLI review of 8f9605c9: the message told the developer to record the edit with the generator, which refuses that state. It now says to restore the file, and that an approved value change goes through --remove and --add on the restored tree. Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/golden/store.py | 4 +++- tests/test_hashing/golden/test_golden_store.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_hashing/golden/store.py b/tests/test_hashing/golden/store.py index 855be911e..e4d06e9e2 100644 --- a/tests/test_hashing/golden/store.py +++ b/tests/test_hashing/golden/store.py @@ -479,7 +479,9 @@ def folder_problems(history: Sequence[Mapping[str, Any]], folder: Path) -> list[ + ( "a released golden file never changes" if end is not None and i <= end - else "run generate_golden_hashes.py to record it" + else "restore the file (git checkout); a value change no release " + "lists needs approval, then generate_golden_hashes.py --remove " + "and --add on the restored tree" ) ) if not folder.is_dir(): diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index 105b0be3f..673cafb88 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -138,12 +138,17 @@ def test_a_file_not_named_group_and_counter_is_an_error( def test_an_edited_listed_file_is_reported(tree: Path) -> None: + """The problem says to restore the file, never to record the edit with the generator.""" path = _folder(tree) / "arrow-01.json" _ = path.write_bytes(path.read_bytes().replace(b"\n", b"\r\n")) - assert any( - "arrow-01.json does not have the sha256" in p + [problem] = [ + p for p in store.tree_problems(tree, VERSION) - ) + if "arrow-01.json does not have the sha256" in p + ] + assert "restore" in problem + assert "--remove" in problem + assert "to record it" not in problem def test_an_unlisted_file_is_reported(tree: Path) -> None: From 2998644f9037031d80cbb3554fedcb3a08051a63 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 16:22:32 -0800 Subject: [PATCH 14/16] fix(golden): the release gate requires the release revision to be the last one (ITL-946) CodeRabbit CLI review of f9e2bfa4: a golden file added after the release step and before the tag would ship in the tagged commit while still open, editable afterwards. --release-version now refuses unless the release revision is the last revision of HEAD's history. Co-Authored-By: Claude Opus 5.5 --- RELEASING.md | 3 ++- docs/reference/golden-hashes.md | 4 +++- scripts/check_golden_tags.py | 13 +++++++++++-- tests/test_check_golden_tags.py | 19 +++++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 685d61428..524b87d1f 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -68,7 +68,8 @@ Do these on `metamorphic-labs` `main` before anything is pushed: already released. Commit both files with the release pull request; the release is tagged on a commit that contains them. The Release workflow's build job checks this before it creates the tag (`scripts/check_golden_tags.py --release-version X.Y.Z`) and stops if the - release is not recorded. + release is not recorded, or if a golden file was added after the release step (the + release revision must be the last revision when the commit is tagged). 3. **Check that `main` is green** — the `Run Tests` workflow on the commit you are releasing. The Release workflow re-runs the unit tests on Python 3.11, 3.12 and 3.13 and the license check, but not the PostgreSQL or SpiralDB integration jobs. diff --git a/docs/reference/golden-hashes.md b/docs/reference/golden-hashes.md index e21072c25..58e7a22b4 100644 --- a/docs/reference/golden-hashes.md +++ b/docs/reference/golden-hashes.md @@ -296,7 +296,9 @@ build job before the tag is made; tested by `tests/test_check_golden_tags.py` - **Checks:** HEAD's `released_manifests.json` pins release `X.Y.Z` of the current data context, its `manifest` pin is set, and the pinned digest is the release revision of - `X.Y.Z` in HEAD's golden history. It fails without git. + `X.Y.Z`, which is the last revision of HEAD's golden history: a golden file added after + the release step would ship unreleased, so such a commit is not tagged. It fails without + git. - **Protects against:** a release cut without the release step (11). The final-release check cannot catch it in the Release workflow, whose tests run before the tag exists, under a development version. diff --git a/scripts/check_golden_tags.py b/scripts/check_golden_tags.py index 0205b65eb..de435a5d8 100644 --- a/scripts/check_golden_tags.py +++ b/scripts/check_golden_tags.py @@ -189,7 +189,8 @@ def check_release(repo: Path, release: str) -> list[str]: The Release workflow runs this before it creates the tag: HEAD's ``released_manifests.json`` pins ``release`` under the current context's ``releases``, its manifest pin is set, and the pinned digest is the digest of the - release revision of ``release`` in HEAD's golden history. + release revision of ``release`` in HEAD's golden history, which is its last revision + (no golden file added after the release step ships unreleased). Args: repo: Repository root. @@ -217,9 +218,10 @@ def check_release(repo: Path, release: str) -> list[str]: manifest = _show_json(repo, "HEAD", CONTEXTS / "data" / f"{version}.json") golden = manifest.get("golden") if isinstance(manifest, dict) else None history = golden.get("history") if isinstance(golden, dict) else None + entries: list[Any] = history if isinstance(history, list) else [] revisions = [ revision - for revision in (history if isinstance(history, list) else []) + for revision in entries if isinstance(revision, dict) and revision.get("release") == release ] if [revision.get("digest") for revision in revisions] != [releases[release]]: @@ -227,6 +229,13 @@ def check_release(repo: Path, release: str) -> list[str]: f"the pinned digest of {release} is not the release revision of {release} " "in the golden history" ) + elif entries[-1] is not revisions[0]: + problems.append( + f"the release revision of {release} is not the last revision of the golden " + "history: a golden file added after the release step would ship unreleased; " + f"tag the commit of the release step, or run --release {release} again after " + "the addition (a new version)" + ) return problems diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py index 62a962c12..d548bb285 100644 --- a/tests/test_check_golden_tags.py +++ b/tests/test_check_golden_tags.py @@ -280,3 +280,22 @@ def test_the_gate_fails_outside_a_repository_even_locally(tmp_path: Path) -> Non store.copy_tree(store.PROJECT_ROOT, plain, VERSION) result = _gate(plain, "0.3.0") assert result.returncode == 1 + + +def test_the_gate_refuses_a_golden_addition_after_the_release_step(repo: Path) -> None: + """The tagged commit's last revision is the release: no unreleased file ships with it.""" + folder = _folder(repo) + name = store.next_file_name(store.group_files(folder)["schema"][-1].name) + _ = (folder / name).write_text(dump({"new.entry": "x"})) + manifest = store.read_json(store.manifest_path(repo, VERSION)) + manifest["golden"]["history"] = store.with_file( + manifest["golden"]["history"], + name, + store.file_digest(folder / name), + "2026-11-20", + ) + store.write_json(store.manifest_path(repo, VERSION), manifest) + _commit(repo, f"add {name} after the release step") + result = _gate(repo, "0.3.0") + assert result.returncode == 1 + assert "last revision" in result.stderr From 5b77b68f873ea5e02c7efe173cb13d4388e4adef Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 16:56:20 -0800 Subject: [PATCH 15/16] fix(golden): harden the store, the tag check and the no-skip rule (ITL-946) GitHub CodeRabbit review threads on #399: - check_golden_tags.py clears inherited git location variables (GIT_DIR, GIT_WORK_TREE, ...) and reads with --no-replace-objects, so --repo is the repository read. - store.read_json refuses a JSON object that repeats a key (json keeps the last), so a duplicate entry inside one golden file is an error. - folder_problems refuses a listed name that is not -NN.json of a known group (e.g. ../arrow-01.json) before building its path. - The no-skip rule moves to marks.py and is applied at collection by tests/test_hashing/golden/conftest.py (trylast), so a module-level skip on test_golden_hashes.py fails the collection instead of silencing the checks. Co-Authored-By: Claude Opus 5.5 --- scripts/check_golden_tags.py | 24 ++++- tests/test_check_golden_tags.py | 34 +++++++ tests/test_hashing/golden/conftest.py | 36 ++++++++ tests/test_hashing/golden/marks.py | 78 ++++++++++++++++ tests/test_hashing/golden/store.py | 30 ++++++- .../test_hashing/golden/test_golden_hashes.py | 90 ++++++++----------- .../test_hashing/golden/test_golden_store.py | 38 ++++++++ 7 files changed, 276 insertions(+), 54 deletions(-) create mode 100644 tests/test_hashing/golden/conftest.py create mode 100644 tests/test_hashing/golden/marks.py diff --git a/scripts/check_golden_tags.py b/scripts/check_golden_tags.py index de435a5d8..db007691c 100644 --- a/scripts/check_golden_tags.py +++ b/scripts/check_golden_tags.py @@ -45,10 +45,32 @@ class GitUnavailableError(RuntimeError): """git, the history or the tags cannot be read.""" +# Variables that point git at another repository than ``--repo`` (git(1), "Environment +# Variables"); they are removed so that the repository read is the one named, and +# ``--no-replace-objects`` keeps replace refs from substituting the objects read. +_GIT_LOCATION_VARIABLES = frozenset( + { + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_NAMESPACE", + "GIT_GRAFT_FILE", + "GIT_SHALLOW_FILE", + } +) + + def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[bytes]: + env = {k: v for k, v in os.environ.items() if k not in _GIT_LOCATION_VARIABLES} try: return subprocess.run( - ["git", "-C", str(repo), *args], capture_output=True, check=False + ["git", "--no-replace-objects", "-C", str(repo), *args], + capture_output=True, + check=False, + env=env, ) except FileNotFoundError as exc: raise GitUnavailableError("git is not installed") from exc diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py index d548bb285..1953e7d28 100644 --- a/tests/test_check_golden_tags.py +++ b/tests/test_check_golden_tags.py @@ -299,3 +299,37 @@ def test_the_gate_refuses_a_golden_addition_after_the_release_step(repo: Path) - result = _gate(repo, "0.3.0") assert result.returncode == 1 assert "last revision" in result.stderr + + +def test_inherited_git_location_variables_do_not_redirect_the_check( + repo: Path, tmp_path: Path +) -> None: + """``--repo`` is the repository read, whatever ``GIT_DIR`` and friends point at.""" + other = tmp_path / "other" + _ = subprocess.run( + ["git", "clone", "-q", str(repo), str(other)], + check=True, + capture_output=True, + env=_env(ci=False), + ) + path = _folder(other) / "semantic-01.json" + _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) + _commit(other, "edit a released file") + _git(other, "fetch", "-q", "--tags") + assert _check(other).returncode == 1 + env = _env(ci=True, ceiling=tmp_path) + env.update( + GIT_DIR=str(repo / ".git"), + GIT_WORK_TREE=str(other), + GIT_COMMON_DIR=str(repo / ".git"), + GIT_OBJECT_DIRECTORY=str(repo / ".git" / "objects"), + ) + result = subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(other)], + capture_output=True, + text=True, + check=False, + env=env, + ) + assert result.returncode == 1, result.stdout + assert "semantic-01.json differs" in result.stderr diff --git a/tests/test_hashing/golden/conftest.py b/tests/test_hashing/golden/conftest.py new file mode 100644 index 000000000..715ceefbb --- /dev/null +++ b/tests/test_hashing/golden/conftest.py @@ -0,0 +1,36 @@ +"""Collection-time refusal of a skipped or xfailed golden entry (ITL-946). + +The check lives here, not in ``test_golden_hashes.py``, so a mark on that module (a +``pytestmark`` skip) cannot switch it off together with the entries. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from tests.test_hashing.golden.marks import golden_entry_violations + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[Any]) -> None: + """Fail the collection when a golden entry carries a forbidden skip or xfail. + + Runs after the other collection hooks, so marks they add are seen. + + Args: + config: The pytest configuration (unused). + items: The collected items. + + Raises: + pytest.UsageError: A golden entry is marked skip or xfail other than by the + optional-package rule. + """ + found = golden_entry_violations(items) + if found: + lines = "\n".join(f" {node}: {marks}" for node, marks in sorted(found.items())) + raise pytest.UsageError( + "golden entries may not be skipped or expected to fail, except by the " + f"optional-package rule (ITL-871, ITL-946):\n{lines}" + ) diff --git a/tests/test_hashing/golden/marks.py b/tests/test_hashing/golden/marks.py new file mode 100644 index 000000000..9de049f32 --- /dev/null +++ b/tests/test_hashing/golden/marks.py @@ -0,0 +1,78 @@ +"""The one skip a golden entry may carry, and the check that refuses any other (ITL-946). + +A golden entry is never skipped or expected to fail, except by the optional-package rule +(ITL-871): a group whose optional package is not installed is skipped. ``conftest.py`` in +this directory applies the check when tests are collected, outside the golden test module, +so a mark on that module cannot switch it off. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pytest + +from tests.test_hashing.golden import cases + +SKIP_OR_XFAIL = frozenset({"skip", "skipif", "xfail"}) +"""The marks that skip a test or expect it to fail.""" + +GOLDEN_ENTRY_TEST = "test_golden_entry" +"""The test function, in ``test_golden_hashes.py``, that checks one golden entry.""" + + +def needs(group: str) -> Any: + """A skip mark for a group whose optional dependency is not installed (ITL-871).""" + module = cases.OPTIONAL_GROUPS.get(group) + return pytest.mark.skipif( + not cases.group_available(group), + reason=f"golden group {group!r} needs the optional package {module!r}", + ) + + +def entry_marks(group: str) -> list[Any]: + """The only marks a golden entry may carry: the optional-package skip (ITL-871).""" + return [needs(group)] if group in cases.OPTIONAL_GROUPS else [] + + +def disallowed_marks(group: str, marks: Iterable[Any]) -> list[str]: + """The skip and xfail marks on a golden entry of ``group`` that the rules forbid. + + The one allowed mark is the optional-package skip of an optional group, exactly as + ``needs`` makes it. + + Args: + group: The entry's group. + marks: The entry's marks, as ``pytest.Mark`` or ``pytest.MarkDecorator`` objects. + + Returns: + A description of each forbidden mark. + """ + allowed = [mark.mark for mark in entry_marks(group)] + found = [getattr(mark, "mark", mark) for mark in marks] + return [ + f"{mark.name}{mark.args}{mark.kwargs}" + for mark in found + if mark.name in SKIP_OR_XFAIL and mark not in allowed + ] + + +def golden_entry_violations(items: Iterable[Any]) -> dict[str, list[str]]: + """The collected golden entry tests that carry a forbidden skip or xfail. + + Args: + items: Collected test items (any others are ignored). + + Returns: + Node id to the forbidden marks it carries; empty when there are none. + """ + found: dict[str, list[str]] = {} + for item in items: + if getattr(item, "originalname", None) != GOLDEN_ENTRY_TEST: + continue + group = str(item.callspec.params["group"]) + bad = disallowed_marks(group, item.iter_markers()) + if bad: + found[item.nodeid] = bad + return found diff --git a/tests/test_hashing/golden/store.py b/tests/test_hashing/golden/store.py index e4d06e9e2..c75921c81 100644 --- a/tests/test_hashing/golden/store.py +++ b/tests/test_hashing/golden/store.py @@ -98,9 +98,28 @@ def golden_folder(root: Path, manifest: Mapping[str, Any]) -> Path: return root / GOLDEN_ROOT / manifest["golden"]["folder"] +def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build a JSON object, refusing a key that appears twice (JSON keeps the last).""" + out: dict[str, Any] = {} + for key, value in pairs: + if key in out: + raise GoldenLayoutError(f"the key {key!r} appears twice in one JSON object") + out[key] = value + return out + + def read_json(path: Path) -> Any: - """Parse a JSON file.""" - return json.loads(path.read_text(encoding="utf-8")) + """Parse a JSON file, refusing an object that repeats a key. + + Raises: + GoldenLayoutError: An object in the file repeats a key; the message names the file. + """ + try: + return json.loads( + path.read_text(encoding="utf-8"), object_pairs_hook=_unique_keys + ) + except GoldenLayoutError as exc: + raise GoldenLayoutError(f"{path}: {exc}") from exc def write_json(path: Path, value: Any) -> None: @@ -470,6 +489,13 @@ def folder_problems(history: Sequence[Mapping[str, Any]], folder: Path) -> list[ problems.append(f"{name} is listed by revisions {listed[name]} and {i}") continue listed[name] = i + match = FILE_NAME.match(name) + if match is None or match["group"] not in GROUPS: + problems.append( + f"{name}, listed by revision {i}, is not a golden file name " + "(-NN.json of a known group, in the golden folder)" + ) + continue path = folder / name if not path.is_file(): problems.append(f"{name}, listed by revision {i}, is missing") diff --git a/tests/test_hashing/golden/test_golden_hashes.py b/tests/test_hashing/golden/test_golden_hashes.py index 2f562e87f..157119aa6 100644 --- a/tests/test_hashing/golden/test_golden_hashes.py +++ b/tests/test_hashing/golden/test_golden_hashes.py @@ -22,7 +22,6 @@ import subprocess import sys import uuid -from collections.abc import Iterable from pathlib import Path from typing import Any @@ -33,6 +32,12 @@ from orcapod.contexts import get_default_context from orcapod.types import Schema from tests.test_hashing.golden import cases, store +from tests.test_hashing.golden.marks import ( + disallowed_marks, + entry_marks, + golden_entry_violations, + needs, +) GENERATOR = Path(__file__).with_name("generate_golden_hashes.py") @@ -56,26 +61,12 @@ def _load_golden() -> tuple[dict[str, dict[str, Any]], str | None]: GOLDEN, LOAD_ERROR = _load_golden() -def _needs(group: str) -> Any: - """A skip mark for a group whose optional dependency is not installed (ITL-871).""" - module = cases.OPTIONAL_GROUPS.get(group) - return pytest.mark.skipif( - not cases.group_available(group), - reason=f"golden group {group!r} needs the optional package {module!r}", - ) - - -def _entry_marks(group: str) -> list[Any]: - """The only marks a golden entry may carry: the optional-package skip (ITL-871).""" - return [_needs(group)] if group in cases.OPTIONAL_GROUPS else [] - - GOLDEN_ENTRIES = [ - pytest.param(group, entry, id=f"{group}:{entry}", marks=_entry_marks(group)) + pytest.param(group, entry, id=f"{group}:{entry}", marks=entry_marks(group)) for group in cases.GROUPS for entry in GOLDEN[group] ] -GROUP_PARAMS = [pytest.param(group, marks=_needs(group)) for group in cases.GROUPS] +GROUP_PARAMS = [pytest.param(group, marks=needs(group)) for group in cases.GROUPS] def _moved_message(group: str, entry: str, expected: Any, actual: Any) -> str: @@ -139,30 +130,6 @@ def test_no_unpinned_entries(live: dict[str, dict[str, Any]], group: str) -> Non # No golden entry is skipped or expected to fail, but for a missing optional package # --------------------------------------------------------------------------- -_SKIP_OR_XFAIL = frozenset({"skip", "skipif", "xfail"}) - - -def disallowed_marks(group: str, marks: Iterable[Any]) -> list[str]: - """The skip and xfail marks on a golden entry of ``group`` that the rules forbid. - - The one allowed mark is the optional-package skip of an optional group, exactly as - ``_needs`` makes it. - - Args: - group: The entry's group. - marks: The entry's marks, as ``pytest.Mark`` or ``pytest.MarkDecorator`` objects. - - Returns: - A description of each forbidden mark. - """ - allowed = [mark.mark for mark in _entry_marks(group)] - found = [getattr(mark, "mark", mark) for mark in marks] - return [ - f"{mark.name}{mark.args}{mark.kwargs}" - for mark in found - if mark.name in _SKIP_OR_XFAIL and mark not in allowed - ] - def test_golden_entry_params_carry_only_the_optional_package_skip() -> None: """No parametrized golden entry is marked skip or xfail but by the ITL-871 rule.""" @@ -177,17 +144,32 @@ def test_collected_golden_entries_carry_only_the_optional_package_skip( request: pytest.FixtureRequest, ) -> None: """The collected golden entry tests carry no other skip or xfail, whoever added it.""" - found: dict[str, list[str]] = {} - for item in request.session.items: - if getattr(item, "originalname", None) != "test_golden_entry": - continue - group = item.callspec.params["group"] # pyright: ignore[reportAttributeAccessIssue] - bad = disallowed_marks(group, item.iter_markers()) - if bad: - found[item.nodeid] = bad + found = golden_entry_violations(request.session.items) assert not found, f"golden entries marked skip or xfail: {found}" +def test_the_collection_hook_refuses_a_skipped_golden_entry( + pytestconfig: pytest.Config, +) -> None: + """The conftest hook, outside this module, fails the collection on a forbidden mark.""" + from tests.test_hashing.golden import conftest + + class _Spec: + params = {"group": "semantic", "entry": "value.none"} + + class _Item: + originalname = "test_golden_entry" + nodeid = "test_golden_hashes.py::test_golden_entry[semantic:value.none]" + callspec = _Spec() + + def iter_markers(self) -> list[Any]: + return [pytest.mark.skip(reason="module-level skip").mark] + + with pytest.raises(pytest.UsageError, match="semantic:value.none"): + conftest.pytest_collection_modifyitems(pytestconfig, [_Item()]) + conftest.pytest_collection_modifyitems(pytestconfig, []) + + def test_golden_test_module_marks_nothing() -> None: """Neither the module nor the entry test function carries a mark of its own.""" assert "pytestmark" not in globals() @@ -205,7 +187,7 @@ def test_golden_test_module_marks_nothing() -> None: ("semantic", pytest.mark.xfail(reason="moved")), ("schema", pytest.mark.skip(reason="later")), ("torch", pytest.mark.skipif(True, reason="another reason")), - ("arrow", _needs("torch")), + ("arrow", needs("torch")), ], ) def test_a_forbidden_mark_is_reported(group: str, mark: Any) -> None: @@ -215,7 +197,7 @@ def test_a_forbidden_mark_is_reported(group: str, mark: Any) -> None: def test_the_optional_package_skip_is_allowed() -> None: """The optional-package skip of an optional group is not reported.""" - assert disallowed_marks("torch", [_needs("torch").mark]) == [] + assert disallowed_marks("torch", [needs("torch").mark]) == [] GENERATOR_TIMEOUT_S = 600 @@ -516,6 +498,12 @@ def test_tensor_and_ndarray_pins_differ(name: str) -> None: from orcapod.contexts import get_default_context from tests.test_hashing.golden import cases, store +from tests.test_hashing.golden.marks import ( + disallowed_marks, + entry_marks, + golden_entry_violations, + needs, +) from tests.test_hashing.golden.generate_golden_hashes import main # Stand in for a supported PyTorch that predates float8_e8m0fnu (added in torch diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py index 673cafb88..c38bda799 100644 --- a/tests/test_hashing/golden/test_golden_store.py +++ b/tests/test_hashing/golden/test_golden_store.py @@ -175,6 +175,44 @@ def test_a_recorded_file_of_an_unknown_group_is_reported(tree: Path) -> None: ) +def test_a_key_repeated_inside_one_file_is_an_error(tmp_path: Path) -> None: + """JSON keeps the last of two equal keys; the store refuses the file instead.""" + _ = (tmp_path / "arrow-01.json").write_text( + '{"example.entry": "old", "example.entry": "current"}\n' + ) + with pytest.raises(store.GoldenLayoutError, match="example.entry"): + _ = store.load_folder(tmp_path) + + +def test_a_key_repeated_inside_a_recorded_file_is_reported(tree: Path) -> None: + name = NEXT_SCHEMA + folder = _folder(tree) + _ = (folder / name).write_bytes(b'{"dup.entry": "a", "dup.entry": "b"}\n') + manifest = _manifest(tree) + manifest["golden"]["history"] = store.with_file( + manifest["golden"]["history"], + name, + store.file_digest(folder / name), + "2026-11-15", + ) + store.write_json(store.manifest_path(tree, VERSION), manifest) + assert any("dup.entry" in p for p in store.tree_problems(tree, VERSION)) + + +@pytest.mark.parametrize( + "name", ["../arrow-01.json", "sub/arrow-01.json", "obsolete-01.json", "arrow.json"] +) +def test_a_listed_name_that_is_not_a_golden_file_name_is_reported( + tree: Path, name: str +) -> None: + """A history entry names a ``-NN.json`` file of a known group, in the folder.""" + history = _history(tree) + history[0]["added"][name] = "0" * 64 + history = store.rechain(history) + problems = store.folder_problems(history, _folder(tree)) + assert any(name in p and "not a golden file name" in p for p in problems) + + def test_a_missing_listed_file_is_reported(tree: Path) -> None: (_folder(tree) / "torch-01.json").unlink() assert any( From 322399991ccc4d6298b2f0f2289166f4a1732d34 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sun, 15 Nov 2026 17:21:49 -0800 Subject: [PATCH 16/16] feat(golden): the Release workflow also checks earlier release tags (ITL-946) GitHub CodeRabbit thread on release.yml, developer-approved extension of the release gate: the build job runs scripts/check_golden_tags.py before the tag is made, so a release dispatched from a branch that changed an earlier release's golden files, history or pins is refused. Tags of earlier data contexts are not checked: with only those (as on the public repository before v0.3.0) it passes with nothing to check, tested. The collection hook only looks at test_golden_entry items of test_golden_hashes.py and lets the optional-package skip through, tested. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 6 +++ RELEASING.md | 5 +- docs/reference/golden-hashes.md | 5 +- tests/test_check_golden_tags.py | 27 ++++++++++ tests/test_hashing/golden/marks.py | 6 +++ .../test_hashing/golden/test_golden_hashes.py | 52 +++++++++++++++---- 6 files changed, 87 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e1ac1218..f331e3609 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,6 +91,12 @@ jobs: # generate_golden_hashes.py --release for this version (ITL-946). run: python3 scripts/check_golden_tags.py --release-version "${{ steps.normalize.outputs.version }}" + - name: Check golden files against release tags + # Every golden file, revision and pin an earlier release tag of the current data + # context shipped is unchanged here (ITL-946). Tags of no such release (older + # data contexts) are not checked; it fails only without git, history or tags. + run: python3 scripts/check_golden_tags.py + - name: Configure git identity run: | git config user.name "github-actions[bot]" diff --git a/RELEASING.md b/RELEASING.md index 524b87d1f..f3e5c72a0 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -69,7 +69,10 @@ Do these on `metamorphic-labs` `main` before anything is pushed: on a commit that contains them. The Release workflow's build job checks this before it creates the tag (`scripts/check_golden_tags.py --release-version X.Y.Z`) and stops if the release is not recorded, or if a golden file was added after the release step (the - release revision must be the last revision when the commit is tagged). + release revision must be the last revision when the commit is tagged). It also runs the + tag check (`scripts/check_golden_tags.py`) there, so a branch that changed what an earlier + release of the data context shipped is not released. Tags of earlier data contexts are + not checked, so the first release of a data context passes with nothing to check. 3. **Check that `main` is green** — the `Run Tests` workflow on the commit you are releasing. The Release workflow re-runs the unit tests on Python 3.11, 3.12 and 3.13 and the license check, but not the PostgreSQL or SpiralDB integration jobs. diff --git a/docs/reference/golden-hashes.md b/docs/reference/golden-hashes.md index 58e7a22b4..d9dcb4a45 100644 --- a/docs/reference/golden-hashes.md +++ b/docs/reference/golden-hashes.md @@ -305,8 +305,9 @@ build job before the tag is made; tested by `tests/test_check_golden_tags.py` ### Tag check -`scripts/check_golden_tags.py`, a step of the `Run Tests` workflow; tested by -`tests/test_check_golden_tags.py` +`scripts/check_golden_tags.py`, a step of the `Run Tests` workflow and of the Release +workflow's build job before the tag is made (so a release dispatched from a branch that +rewrote an earlier release is refused); tested by `tests/test_check_golden_tags.py` - **Checks:** for each tag `vX.Y.Z` whose `released_manifests.json` pins release `X.Y.Z` of the current data context (the one manifest at HEAD): every golden file listed by the diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py index 1953e7d28..8e329711d 100644 --- a/tests/test_check_golden_tags.py +++ b/tests/test_check_golden_tags.py @@ -333,3 +333,30 @@ def test_inherited_git_location_variables_do_not_redirect_the_check( ) assert result.returncode == 1, result.stdout assert "semantic-01.json differs" in result.stderr + + +def test_tags_of_earlier_data_contexts_pass_with_nothing_to_check( + tmp_path: Path, +) -> None: + """The Release workflow's first release of a context: older tags only, none of it. + + As on the public repository before v0.3.0: tags whose commits have no + ``released_manifests.json`` (v0.1.x) or the old ``{"v1": null}`` form (v0.2.0). In CI + the check passes with nothing to check; it never fails for want of a release tag. + """ + root = tmp_path / "repo" + pins = root / store.PINS_FILE + pins.parent.mkdir(parents=True) + _ = (root / "README.md").write_text("old release\n") + _git(root.parent, "init", "-q", str(root)) + _commit(root, "v0.1.x: no released_manifests.json") + _git(root, "tag", "v0.1.4") + _ = pins.write_text('{\n "v1": null\n}\n') + _commit(root, "v0.2.0: the old pin form") + _git(root, "tag", "v0.2.0") + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") + _commit(root, "the open v1 data context") + result = _check(root, ci=True) + assert result.returncode == 0, result.stderr + assert "nothing to check" in result.stdout diff --git a/tests/test_hashing/golden/marks.py b/tests/test_hashing/golden/marks.py index 9de049f32..253a86b47 100644 --- a/tests/test_hashing/golden/marks.py +++ b/tests/test_hashing/golden/marks.py @@ -9,6 +9,7 @@ from __future__ import annotations from collections.abc import Iterable +from pathlib import Path from typing import Any import pytest @@ -21,6 +22,9 @@ GOLDEN_ENTRY_TEST = "test_golden_entry" """The test function, in ``test_golden_hashes.py``, that checks one golden entry.""" +GOLDEN_TEST_MODULE = Path(__file__).resolve().with_name("test_golden_hashes.py") +"""The module holding ``GOLDEN_ENTRY_TEST``; items of any other module are ignored.""" + def needs(group: str) -> Any: """A skip mark for a group whose optional dependency is not installed (ITL-871).""" @@ -71,6 +75,8 @@ def golden_entry_violations(items: Iterable[Any]) -> dict[str, list[str]]: for item in items: if getattr(item, "originalname", None) != GOLDEN_ENTRY_TEST: continue + if Path(str(getattr(item, "path", ""))).resolve() != GOLDEN_TEST_MODULE: + continue group = str(item.callspec.params["group"]) bad = disallowed_marks(group, item.iter_markers()) if bad: diff --git a/tests/test_hashing/golden/test_golden_hashes.py b/tests/test_hashing/golden/test_golden_hashes.py index 157119aa6..54df51d2b 100644 --- a/tests/test_hashing/golden/test_golden_hashes.py +++ b/tests/test_hashing/golden/test_golden_hashes.py @@ -148,26 +148,56 @@ def test_collected_golden_entries_carry_only_the_optional_package_skip( assert not found, f"golden entries marked skip or xfail: {found}" +class _FakeItem: + """A collected golden entry test with the given group and marks.""" + + originalname = "test_golden_entry" + + def __init__( + self, group: str, marks: list[Any], path: Path = Path(__file__) + ) -> None: + self.nodeid = f"test_golden_hashes.py::test_golden_entry[{group}:x]" + self.path = path + self.callspec = type( + "CallSpec", (), {"params": {"group": group, "entry": "x"}} + )() + self._marks = [getattr(mark, "mark", mark) for mark in marks] + + def iter_markers(self) -> list[Any]: + return self._marks + + def test_the_collection_hook_refuses_a_skipped_golden_entry( pytestconfig: pytest.Config, ) -> None: """The conftest hook, outside this module, fails the collection on a forbidden mark.""" from tests.test_hashing.golden import conftest - class _Spec: - params = {"group": "semantic", "entry": "value.none"} + skipped = _FakeItem("semantic", [pytest.mark.skip(reason="module-level skip")]) + with pytest.raises(pytest.UsageError, match="semantic:x"): + conftest.pytest_collection_modifyitems(pytestconfig, [skipped]) - class _Item: - originalname = "test_golden_entry" - nodeid = "test_golden_hashes.py::test_golden_entry[semantic:value.none]" - callspec = _Spec() - def iter_markers(self) -> list[Any]: - return [pytest.mark.skip(reason="module-level skip").mark] +def test_the_collection_hook_allows_the_optional_package_skip( + pytestconfig: pytest.Config, +) -> None: + """The torch group's optional-package skip passes the collection hook.""" + from tests.test_hashing.golden import conftest + + items = [_FakeItem("torch", [needs("torch")]), _FakeItem("semantic", [])] + conftest.pytest_collection_modifyitems(pytestconfig, items) + - with pytest.raises(pytest.UsageError, match="semantic:value.none"): - conftest.pytest_collection_modifyitems(pytestconfig, [_Item()]) - conftest.pytest_collection_modifyitems(pytestconfig, []) +def test_the_collection_hook_ignores_other_modules(pytestconfig: pytest.Config) -> None: + """A test of the same name in another module is not a golden entry.""" + from tests.test_hashing.golden import conftest + + other = _FakeItem( + "semantic", + [pytest.mark.skip(reason="x")], + path=Path(__file__).with_name("x.py"), + ) + conftest.pytest_collection_modifyitems(pytestconfig, [other]) def test_golden_test_module_marks_nothing() -> None: