diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..4c240a5d6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Golden hash files are hashed byte for byte: never convert their line endings (ITL-946). +tests/test_hashing/golden/data/** -text diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5501bfba8..f331e3609 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,6 +86,17 @@ jobs: fetch-depth: 0 # required: hatch-vcs reads tag to set version ref: ${{ inputs.branch }} + - name: Check the golden release step + # Nothing is tagged or published unless the release pull request ran + # generate_golden_hashes.py --release for this version (ITL-946). + run: python3 scripts/check_golden_tags.py --release-version "${{ steps.normalize.outputs.version }}" + + - name: Check golden files against release tags + # Every golden file, revision and pin an earlier release tag of the current data + # context shipped is unchanged here (ITL-946). Tags of no such release (older + # data contexts) are not checked; it fails only without git, history or tags. + run: python3 scripts/check_golden_tags.py + - name: Configure git identity run: | git config user.name "github-actions[bot]" diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index dc2c7f83a..379ef7e3e 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -33,6 +33,11 @@ jobs: - name: Install dependencies run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }} + - name: Check golden files against release tags + # Every golden file and pin a release tag of the current data context shipped must + # be unchanged in HEAD (ITL-946). Fails when git, the history or the tags are missing. + run: uv run --python ${{ matrix.python-version }} python scripts/check_golden_tags.py + - name: Run tests run: uv run --python ${{ matrix.python-version }} pytest -v -rsxX --durations=20 -m "not postgres" --cov=src --cov-report=term-missing --cov-report=xml diff --git a/.zed/rules b/.zed/rules index f8d0bb8ea..a646e3419 100644 --- a/.zed/rules +++ b/.zed/rules @@ -287,7 +287,9 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension differently. It is not a package version. Built-in types use `version_metadata(N)` (`{"version":N}`); a new built-in type must write one, never empty metadata. - A changed identifier is a schema change: it moves stored paths and golden values of that - type. Treat it like any other hash change (developer approval, migration assessment). + type. Treat it like any other hash change (developer approval, migration assessment). There + is no exemption for a logical type's own change (developer, November 15, 2026, ITL-946): if + a golden value it moves has shipped in a release, it needs a new data context version. - The native containers `list[T]` / `set[T]` of a logical type (`ListLogicalType`) have no version of their own, by design: never add a `"version"` key to their metadata. Their identifier is the container kind plus the element's identifier; the type hash treats them @@ -386,14 +388,34 @@ A class a user defines is one of two kinds (developer, November 12, 2026, ITL-97 ## Golden hash values Hash values must stay identical across releases (developer, November 10, 2026). Every hash -Orcapod stores or embeds in a name is pinned in `tests/test_hashing/golden/data/*.json` and -checked by `tests/test_hashing/golden/test_golden_hashes.py`. +Orcapod stores or embeds in a name is pinned in the golden folder of the current data context, +`tests/test_hashing/golden/data//-NN.json`, and checked by +`tests/test_hashing/golden/test_golden_hashes.py`. The mechanism is +`docs/reference/golden-hashes.md` (developer, November 15, 2026, ITL-946). - A failing golden entry means a hash moved. Treat it as a bug in the change, not a stale - file. Never regenerate the golden files to make a test pass. -- Regenerate (`uv run python tests/test_hashing/golden/generate_golden_hashes.py`) only to add - entries (check the diff adds lines and changes none), or for a hash change the developer has - approved together with a migration assessment. + file. Never edit or regenerate a golden file to make a test pass. +- The unit of immutability is the golden file. The manifest's `golden.history` + (`src/orcapod/contexts/data/.json`) lists every file with the sha256 of its bytes; + a release appends a revision with `added: {}`, pinned in + `src/orcapod/contexts/released_manifests.json`. A file listed by a released revision never + changes and is never removed or renamed. +- Adding golden values is allowed in any release: + `uv run python tests/test_hashing/golden/generate_golden_hashes.py` writes only entries no + file pins yet (into a new `-NN.json` once the group's newest file is released) and + records them in the open revision. Check the diff adds and changes no value. `--check` + reports drift without writing. +- Any change to a released golden value needs a new data context version, with no exemption + (also not a logical type's own identifier change). A new data context version ships only in + a minor release (before 1.0) or a major one, never in a patch. When it starts, the old + context's golden files and code move to `orcapod-sanctuary`; this repository carries only + the current context's golden folder. +- Before a release, a value no release lists changes only with the developer's approval and + a migration assessment: `generate_golden_hashes.py --remove ` (refuses an entry a + release lists), then the generator writes the new value. +- The release pull request runs `generate_golden_hashes.py --release X.Y.Z`. CI checks HEAD + against every release tag (`scripts/check_golden_tags.py`). Never mark a golden entry + `skip` or `xfail`; the only skip is the optional-package rule (`torch`). - Never move or rename the fixtures in `tests/test_hashing/golden/cases.py`: function pods, dataclasses and pydantic models hash with their module path. @@ -685,10 +707,12 @@ generation of hashing; rows of the current data context are the current entries. proceed (developer, November 11, 2026). Operator caches and cached sources have no stable name; PipelineJob.run() reports one total for them. Every database class implements list_table_paths. -- A released data context manifest (contexts/data/.json) is frozen: its sha256 is - pinned in contexts/released_manifests.json. A change that moves an existing golden hash - value for a reason other than a logical type's own change needs a new data context version - with its own manifest file, never an edit of a released one. +- A released data context manifest (contexts/data/.json) is frozen: the digest of + its canonical JSON without golden is pinned in contexts/released_manifests.json + (manifest), with one releases line per library release (ITL-946). Only its golden + history grows, by appending. A change that moves a released golden hash value, for any + reason (a logical type's own change included), needs a new data context version with its + own manifest file, never an edit of a released one. - In memory the data context column is still _context_key (ColumnConfig context). - A writer refuses an input of another data context than its own (contexts/stored.require_same_data_context, InputValidationError naming both keys) before diff --git a/CLAUDE.md b/CLAUDE.md index d3006acd6..a2b085b31 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -298,7 +298,9 @@ A logical type's identifier is its Arrow extension name plus its Arrow extension differently. It is not a package version. Built-in types use `version_metadata(N)` (`{"version":N}`); a new built-in type must write one, never empty metadata. - A changed identifier is a schema change: it moves stored paths and golden values of that - type. Treat it like any other hash change (developer approval, migration assessment). + type. Treat it like any other hash change (developer approval, migration assessment). There + is no exemption for a logical type's own change (developer, November 15, 2026, ITL-946): if + a golden value it moves has shipped in a release, it needs a new data context version. - The native containers `list[T]` / `set[T]` of a logical type (`ListLogicalType`) have no version of their own, by design: never add a `"version"` key to their metadata. Their identifier is the container kind plus the element's identifier; the type hash treats them @@ -397,14 +399,34 @@ A class a user defines is one of two kinds (developer, November 12, 2026, ITL-97 ## Golden hash values Hash values must stay identical across releases (developer, November 10, 2026). Every hash -Orcapod stores or embeds in a name is pinned in `tests/test_hashing/golden/data/*.json` and -checked by `tests/test_hashing/golden/test_golden_hashes.py`. +Orcapod stores or embeds in a name is pinned in the golden folder of the current data context, +`tests/test_hashing/golden/data//-NN.json`, and checked by +`tests/test_hashing/golden/test_golden_hashes.py`. The mechanism is +`docs/reference/golden-hashes.md` (developer, November 15, 2026, ITL-946). - A failing golden entry means a hash moved. Treat it as a bug in the change, not a stale - file. Never regenerate the golden files to make a test pass. -- Regenerate (`uv run python tests/test_hashing/golden/generate_golden_hashes.py`) only to add - entries (check the diff adds lines and changes none), or for a hash change the developer has - approved together with a migration assessment. + file. Never edit or regenerate a golden file to make a test pass. +- The unit of immutability is the golden file. The manifest's `golden.history` + (`src/orcapod/contexts/data/.json`) lists every file with the sha256 of its bytes; + a release appends a revision with `added: {}`, pinned in + `src/orcapod/contexts/released_manifests.json`. A file listed by a released revision never + changes and is never removed or renamed. +- Adding golden values is allowed in any release: + `uv run python tests/test_hashing/golden/generate_golden_hashes.py` writes only entries no + file pins yet (into a new `-NN.json` once the group's newest file is released) and + records them in the open revision. Check the diff adds and changes no value. `--check` + reports drift without writing. +- Any change to a released golden value needs a new data context version, with no exemption + (also not a logical type's own identifier change). A new data context version ships only in + a minor release (before 1.0) or a major one, never in a patch. When it starts, the old + context's golden files and code move to `orcapod-sanctuary`; this repository carries only + the current context's golden folder. +- Before a release, a value no release lists changes only with the developer's approval and + a migration assessment: `generate_golden_hashes.py --remove ` (refuses an entry a + release lists), then the generator writes the new value. +- The release pull request runs `generate_golden_hashes.py --release X.Y.Z`. CI checks HEAD + against every release tag (`scripts/check_golden_tags.py`). Never mark a golden entry + `skip` or `xfail`; the only skip is the optional-package rule (`torch`). - Never move or rename the fixtures in `tests/test_hashing/golden/cases.py`: function pods, dataclasses and pydantic models hash with their module path. @@ -713,10 +735,12 @@ current data context are the current entries. proceed (developer, November 11, 2026). Operator caches and cached sources have no stable name; `PipelineJob.run()` reports one total for them. Every database class implements `list_table_paths`. -- A released data context manifest (`contexts/data/.json`) is frozen: its sha256 is - pinned in `contexts/released_manifests.json`. A change that moves an existing golden hash - value for a reason other than a logical type's own change needs a new data context version - with its own manifest file, never an edit of a released one. +- A released data context manifest (`contexts/data/.json`) is frozen: the digest of + its canonical JSON without `golden` is pinned in `contexts/released_manifests.json` + (`manifest`), with one `releases` line per library release (ITL-946). Only its `golden` + history grows, by appending. A change that moves a released golden hash value, for any + reason (a logical type's own change included), needs a new data context version with its + own manifest file, never an edit of a released one. - In memory the data context column is still `_context_key` (`ColumnConfig(context=True)`). - A writer refuses an input of another data context than its own (`contexts/stored.require_same_data_context`, `InputValidationError` naming both keys) diff --git a/RELEASING.md b/RELEASING.md index 3cad656d7..f3e5c72a0 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -54,10 +54,29 @@ Do these on `metamorphic-labs` `main` before anything is pushed: The GitHub Release body is generated from PR titles; `CHANGELOG.md` is the curated record. -2. **Check that `main` is green** — the `Run Tests` workflow on the commit you are +2. **Record the release in the golden history**, in the same release pull request, as its + last change (see *Golden hash values* below): + + ``` + uv run python tests/test_hashing/golden/generate_golden_hashes.py --release X.Y.Z + ``` + + It appends the release revision to the `golden` history of the current data context's + manifest, writes its digest under `releases` in + `src/orcapod/contexts/released_manifests.json`, and at the data context's first release + sets `released_in` and the `manifest` pin. It computes no hash and refuses a version + already released. Commit both files with the release pull request; the release is tagged + on a commit that contains them. The Release workflow's build job checks this before it + creates the tag (`scripts/check_golden_tags.py --release-version X.Y.Z`) and stops if the + release is not recorded, or if a golden file was added after the release step (the + release revision must be the last revision when the commit is tagged). It also runs the + tag check (`scripts/check_golden_tags.py`) there, so a branch that changed what an earlier + release of the data context shipped is not released. Tags of earlier data contexts are + not checked, so the first release of a data context passes with nothing to check. +3. **Check that `main` is green** — the `Run Tests` workflow on the commit you are releasing. The Release workflow re-runs the unit tests on Python 3.11, 3.12 and 3.13 and the license check, but not the PostgreSQL or SpiralDB integration jobs. -3. **Check the version is free** — the tag must not exist on origin, and the version must +4. **Check the version is free** — the tag must not exist on origin, and the version must not exist on PyPI or TestPyPI; neither index allows a version to be uploaded twice, even after it is deleted. @@ -71,12 +90,33 @@ All of these hold before the developer is asked for the go: 1. **The full suite is green**: `uv run pytest tests/` passes on the commit to be pushed. 2. **The release pull request is merged** on `metamorphic-labs` (`CHANGELOG.md` folded, section dated, `changelog.d/` fragments deleted). -3. **The data context freeze is done**: every released manifest is pinned in - `contexts/released_manifests.json` and the golden hash tests pass unchanged. +3. **The golden release step is done**: `contexts/released_manifests.json` has the current + data context's `releases` line for this version and a pinned `manifest`, and the golden + hash tests and the data context tests pass unchanged. 4. **`main` is green** on `metamorphic-labs`, and `git status` is clean. 5. **The version is free**: the tag does not exist on `nauticalab`, and the version is on neither PyPI nor TestPyPI. +## Golden hash values + +Developer decisions, November 15, 2026 (ITL-946); the mechanism and what each check protects +against are in `docs/reference/golden-hashes.md`. + +- A golden file that a release listed never changes, and is never removed or renamed. +- New golden values can be added in any release, a patch included. +- Any change to a released golden value, a logical type's own identifier change included, + needs a new data context version. A new data context version ships only in a minor release + (before 1.0) or a major one, never in a patch. +- When a new data context starts, the old context's golden files and code move to + `orcapod-sanctuary`; this repository carries only the current context's golden folder. +- Release tags are protected on both `metamorphic-labs/orcapod-python` and + `nauticalab/orcapod-python`: the repository ruleset `protect-release-tags` (target + `refs/tags/v*`; rules deletion, update and non-fast-forward; no bypass actors) is in place + since 2026-11-15. A `v*` tag can be created, as the Release workflow does, but never moved + or deleted. This is a precondition of the CI tag check (`scripts/check_golden_tags.py`), + which compares HEAD with every release tag of the current data context. Agents never change + repository settings. + ## Cutting a Release Releases are triggered from the GitHub Actions UI — no manual `git tag` step required. @@ -142,7 +182,11 @@ environments: that name `metamorphic-labs`, if any. 3. Protect `main` on `nauticalab` (no force push, no deletion). It is written only by the fast-forward push above. -4. Optional: set `CODECOV_TOKEN` there. The Codecov, SpiralDB and Linear jobs run only on +4. Protect release tags on both `metamorphic-labs/orcapod-python` and + `nauticalab/orcapod-python` with the ruleset `protect-release-tags` (target + `refs/tags/v*`; deletion, update, non-fast-forward; no bypass actors), so the golden tag + check has a fixed anchor (ITL-946). Done on 2026-11-15. +5. Optional: set `CODECOV_TOKEN` there. The Codecov, SpiralDB and Linear jobs run only on `metamorphic-labs` and are skipped on `nauticalab`. ## Pre-releases diff --git a/changelog.d/ITL-944.md b/changelog.d/ITL-944.md index 8b801eec1..58ed4bbb6 100644 --- a/changelog.d/ITL-944.md +++ b/changelog.d/ITL-944.md @@ -8,18 +8,18 @@ read its tables as results, and computes every input again into its own tables. - **One data context key for the whole of Orcapod's hashing.** The default data context is now `std:v1:default` (manifest `orcapod/contexts/data/v1.json`). The version increments - when, and only when, an existing hash value changes for a reason other than a logical - type's own change. Everything stored before v0.3 (rows that carry `std:v0.1:default`, and - tables without the new column) is the earlier generation. A manifest is frozen at the + when, and only when, a released hash value changes, for any reason (ITL-946). Everything + stored before v0.3 (rows that carry `std:v0.1:default`, and tables without the new column) + is the earlier generation. A manifest is frozen at the release of the library version that contains it, and may be edited in place only while - that release is prepared: `orcapod/contexts/released_manifests.json` pins the sha256 of - each released one, and a test fails when a pinned file is edited. `v1.json` is pinned - when v0.3.0 is tagged. + that release is prepared: `orcapod/contexts/released_manifests.json` pins the digest of + each released one without its `golden` history, and a test fails when a pinned manifest is + edited (ITL-946). `v1.json` is pinned when v0.3.0 is tagged. - **Manifest contents.** Besides the wiring of the hashers, a manifest now has `released_in` (the library version that froze it; `null` while open), `previous` (the data context it follows), `definitions` (a name for each definition that lives in code), - `libraries` (the libraries the core hashes depend on, with versions), `golden` (a digest - of the golden hash files, set when the manifest is frozen) and `changes` (what differs + `libraries` (the libraries the core hashes depend on, with versions), `golden` (the folder + and the append-only history of the golden hash files, ITL-946) and `changes` (what differs from `previous`). - **Logical types moved out of the manifest.** The registered logical types and the value handlers that hash their Python values are now in diff --git a/changelog.d/ITL-946.md b/changelog.d/ITL-946.md new file mode 100644 index 000000000..b09500096 --- /dev/null +++ b/changelog.d/ITL-946.md @@ -0,0 +1,14 @@ +### Changed + +- **Released golden hash values are protected by CI (ITL-946).** The golden hash files of a + data context live in one folder per data context version + (`tests/test_hashing/golden/data/v1/-NN.json`), and the manifest's `golden` section + is an append-only history that lists every file with the sha256 of its bytes. A release + appends a revision whose digest is pinned under `releases` in + `orcapod/contexts/released_manifests.json`, which also pins the manifest without its + `golden` section. A golden file a release listed never changes; new golden values go into + new files and can be added in any release. A change to a released golden value, a logical + type's own identifier change included, needs a new data context version, shipped only in a + minor (before 1.0) or major release. A CI step (`scripts/check_golden_tags.py`) checks HEAD + against every release tag of the current data context. See + `docs/reference/golden-hashes.md`. diff --git a/design_issues/ITL-1043-LJ1.md b/design_issues/ITL-1043-LJ1.md index 7734770f5..2141bddcb 100644 --- a/design_issues/ITL-1043-LJ1.md +++ b/design_issues/ITL-1043-LJ1.md @@ -22,8 +22,8 @@ Found November 12, 2026. Evidence and scripts: `superpowers/specs/itl-1043/FINDI - The operator's `cache_mode` is not saved (ITL-977). The running node's hashes are the intended ones: nullability is part of the schema hash -(`docs/reference/hashing.md`; golden `datagram.json` `arrow_source.int64` against -`arrow_source.int64_nullable`, `schema.json` `multi_field.nullable_type`). No live hash, +(`docs/reference/hashing.md`; golden `v1/datagram-01.json` `arrow_source.int64` against +`arrow_source.int64_nullable`, `v1/schema-01.json` `multi_field.nullable_type`). No live hash, stored path or golden value is wrong; only what a loaded job reconstructs. **Fix:** (ITL-1043; developer decision, November 12, 2026: record the information in the diff --git a/design_issues/ITL-626-HS1.md b/design_issues/ITL-626-HS1.md index 63787590b..7d438419b 100644 --- a/design_issues/ITL-626-HS1.md +++ b/design_issues/ITL-626-HS1.md @@ -14,7 +14,7 @@ and v0.2, for example, the semantic hashes of the `Path`, `UUID`, `DataFrame`, ` carrying those types. That change was noticed only by comparing releases. **Fix:** `tests/test_hashing/golden/` pins about 600 values in committed JSON files, one -generator regenerates them, and every entry is its own test, so a failure names the value that +generator writes them, and every entry is its own test, so a failure names the value that moved. Coverage: semantic hashes of scalars, collections, type objects and logical-type values; `Schema` hashes and stored Arrow types for scalar, nested, nullable and logical types (`Path`, `UUID`, `ndarray`, `DataFrame`, dataclass, pydantic, `list[Path]`, `set[UUID]`); @@ -24,6 +24,9 @@ nullability, nesting, column and row order, extension columns); and end-to-end i (content and pipeline hashes, system-tag columns, record ids, per-row data hashes, database table paths) of representative pipelines and a `PipelineJob`. Values are checked in subprocesses with different `PYTHONHASHSEED` values and were verified identical on Python -3.11, 3.12 and 3.13. The generator's header says when regeneration is legitimate: a changed -value needs a migration assessment and the developer's approval. Not pinned yet: the -`StarfixArrowHasher` gaps tracked in ITL-848. +3.11, 3.12 and 3.13. Not pinned yet: the `StarfixArrowHasher` gaps tracked in ITL-848. +Since ITL-946 the files live in one folder per data context version +(`data/v1/-NN.json`), the manifest's `golden` history pins each file's bytes, a file a +release listed never changes (new values go into new files), a change to a released value +needs a new data context version, and CI checks HEAD against every release tag; see +`docs/reference/golden-hashes.md`. diff --git a/design_issues/ITL-944-DC1.md b/design_issues/ITL-944-DC1.md index df13201b9..76aa6ab81 100644 --- a/design_issues/ITL-944-DC1.md +++ b/design_issues/ITL-944-DC1.md @@ -14,10 +14,11 @@ not a generation, and is missing from operator-cache and cached-source record id **Fix:** (developer's decisions of November 11, 2026, ITL-938 design document Parts 5.5 to 8) - The default data context is `std:v1:default`, manifest `contexts/data/v1.json`. - `contexts/released_manifests.json` pins the sha256 of every released manifest (`null` while - a manifest is open); `tests/test_contexts/test_data_context_v1.py` fails when a pinned file - changes, when more than the default manifest is open, or when a final release ships an open - manifest. `v1.json` is pinned once, in the v0.3.0 release-preparation commit. + `contexts/released_manifests.json` pins the digest of every released manifest without its + `golden` history (`null` while a manifest is open) and, since ITL-946, one revision digest + per release; `tests/test_contexts/test_data_context_v1.py` fails when a pinned manifest + changes, when more than the default manifest is open, or when a final release has no + releases line. `v1.json` is pinned once, by the v0.3.0 release step. - Every stored row carries `__data_context` (`large_string`): function pipeline and result tables, operator-cache tables, cached-source tables, side-effect invocation logs. It replaces `__input_data_context_key` and the result table's `_context_key`. The value is the @@ -52,8 +53,9 @@ Known limits, follow-ups: their value handlers moved to the unfrozen `contexts/logical_types.json`. The names under `definitions` are labels only: nothing in the code reads them or checks that a definition changed, and `changes` lists what is true on this branch (starfix format 2); the entries - for ITL-935 and ITL-942 are added when those land. `golden.digest` stays `null` until the - manifest is frozen, because the golden files still move. + for ITL-935 and ITL-942 are added when those land. `golden` holds the golden folder and the + append-only history of the golden files, and `released_manifests.json` pins the manifest + without `golden` and one revision digest per release (ITL-946). - `v0.1.json` is removed (to live in `orcapod-sanctuary`); `std:v0.1:default` raises `ContextResolutionError`, and a node saved under it loads under the current data context with a warning (`contexts.resolve_saved_context`). diff --git a/docs/concepts/logical-type-identifiers.md b/docs/concepts/logical-type-identifiers.md index 6cf9989f0..e96c39302 100644 --- a/docs/concepts/logical-type-identifiers.md +++ b/docs/concepts/logical-type-identifiers.md @@ -65,6 +65,10 @@ differently**. The identifier is therefore not a package version. A type can go through many releases under one identifier. +A built-in type's values are pinned as golden hashes. Once a release has shipped them, a new +identifier for that type moves released golden values, so it needs a new data context +version, like any other change to a released hash ([Golden Hashes](../reference/golden-hashes.md)). + ## Built-in types Orcapod's own types follow the same contract as yours. Their metadata is the JSON text diff --git a/docs/reference/golden-hashes.md b/docs/reference/golden-hashes.md new file mode 100644 index 000000000..d9dcb4a45 --- /dev/null +++ b/docs/reference/golden-hashes.md @@ -0,0 +1,377 @@ +# Golden Hashes + +Within one data context version, a golden hash value that has shipped in a release never +changes and is never removed. New golden values can be added at any time. Everything on this +page exists to keep that promise and to make CI fail when something breaks it (ITL-626, +ITL-946). + +The promise matters because Orcapod hashes are addresses, not just checksums. They name +stored table paths, record ids, memoization keys, default source ids and system-tag column +names (see the [Hashing Reference](hashing.md)). A hash that moves without notice strands +data: after an upgrade, a pipeline no longer finds its stored results and computes them +again, or writes a second copy next to the first. + +A **golden entry** is one named case and the exact value Orcapod computes for it, for +example the `semantic` entry `value.uuid.nil`, whose value is `semantic_v1:...`. The golden +test computes every entry again from the code on every CI run. A **data context version** +(today `v1`, key `std:v1:default`, first shipped in Orcapod 0.3) names one generation of +hashing. A hash change that is wanted on purpose starts a new data context version. It never +edits a released value. + +## The pieces at a glance + +The mechanism has five layers. Each one is checked against the layer below it, and the last +one is a set of records outside the repository that no pull request can change. Read an arrow +as "is checked against". The numbers refer to the scenarios in the +[threat model](#threat-model-who-catches-what). + +```text +Orcapod code computes every hash: stored paths, record ids, cache keys, source ids + | golden recompute: stops behaviour drift (1); no-skip rule (9) + v +Golden files data/v1/-NN.json: entry name -> pinned value + | file hashes: stops edited, deleted and unlisted files (2-7); duplicate names (8) + v +Chain golden.history in v1.json: revisions with the files they added (12) + | release pins: stops a recomputed chain (13) + v +Release pins released_manifests.json: one revision digest per release, manifest pin (10, 11) + | tag check in CI: stops a rewrite of files, chain and pins (14, 15) + v +Outside the tree a protected git tag per release; the PyPI wheel and sdist (16) +``` + +The first three checks stop accidental drift. A deliberate rewrite of the whole tree stops +only at the tags and on PyPI. + +## Layout and formats + +Three places hold the mechanism: the golden folder of the current data context, the +`golden` section of its manifest, and the release pins. The examples below show the state +after a hypothetical patch release 0.3.1 that added one file; digests are shortened. + +### Golden files: `tests/test_hashing/golden/data/v1/` + +```text +tests/test_hashing/golden/data/ + v1/ + arrow-01.json # the files of 0.3.0 + datagram-01.json + pipeline-01.json + schema-01.json + semantic-01.json + torch-01.json + semantic-02.json # added after 0.3.0: new semantic entries only +``` + +- A file is named after the group of cases it pins plus a two-digit counter: + `-NN.json`, where the group is one of `cases.GROUPS` (`semantic`, `schema`, + `arrow`, `datagram`, `pipeline`, `torch`). Names carry no date, issue or release; that + information lives in the manifest. The folder holds nothing else. +- A file maps entry names to values. An entry name appears in exactly one file of the + folder, whatever its group. +- A file listed by a released revision is never edited. New values go into the group's next + file. +- The folder holds only the current data context. The golden folders of older data + contexts live in `orcapod-sanctuary`. + +### The chain: `golden` in `src/orcapod/contexts/data/v1.json` + +```json +"golden": { + "folder": "v1", + "history": [ + {"date": "2026-11-15", "release": null, + "added": {"arrow-01.json": "d62d...", "datagram-01.json": "1151...", + "pipeline-01.json": "e4e4...", "schema-01.json": "315a...", + "semantic-01.json": "187c...", "torch-01.json": "99af..."}, + "previous": null, "digest": "2bc7..."}, + {"date": "2026-11-16", "release": "0.3.0", "added": {}, + "previous": "2bc7...", "digest": "d002..."}, + {"date": "2026-11-24", "release": null, + "added": {"semantic-02.json": "e810..."}, + "previous": "d002...", "digest": "d003..."}, + {"date": "2026-12-01", "release": "0.3.1", "added": {}, + "previous": "d003...", "digest": "d004..."} + ] +} +``` + +- `folder` names the golden folder, under `tests/test_hashing/golden/data/`. +- A revision either adds files or marks a release, never both. A revision that adds files + lists each with the sha256 of its bytes. A release revision has `added` empty and names the + library version in `release`. +- Revisions after the last release revision are open: they may still change, and their + digests are then computed again. The revisions up to the last release revision, and every + file they list, are fixed. +- `date` is the date the revision was last changed. It is informational: nothing can verify + it. + +### The pins: `src/orcapod/contexts/released_manifests.json` + +```json +{"v1": {"manifest": "b249...", + "releases": {"0.3.0": "d002...", "0.3.1": "d004..."}}} +``` + +- `manifest` pins the rest of `v1.json` (the hashers, `definitions`, `libraries`, + `previous`, `released_in`). It is `null` while the data context is open and is set once, + at the data context's first release. +- `releases` gains one line per library release: the digest of that release's revision. + +### Digest definitions + +| Value | Computed as | +| --- | --- | +| File hash in `added` | sha256 of the file's bytes, hex | +| Revision `digest` | sha256 of the canonical JSON (sorted keys, no spaces, `json.dumps(..., sort_keys=True, separators=(",", ":"))`) of `{date, release, added, previous}` | +| `manifest` pin | sha256 of the canonical JSON of the manifest without its `golden` key | +| `releases` pin | the `digest` of that release's revision | + +Each revision's digest covers the previous revision's digest, so pinning a release digest +also fixes every revision before it. + +## Lifecycle + +Three events touch the golden files: a pull request adds values, a release freezes them, and +a new data context version starts a new folder and chain. Nothing else writes to them. All +writes go through `tests/test_hashing/golden/generate_golden_hashes.py`. + +### Adding golden values (any pull request) + +1. Add the case to `tests/test_hashing/golden/cases.py`. Never move or rename an existing + fixture: function pods, dataclasses and pydantic models hash with their module path. +2. Run `uv run python tests/test_hashing/golden/generate_golden_hashes.py` (`--add` is the + default). It computes every case and writes only the entries that no golden file pins + yet. A group's new entries go into its newest file when no release lists that file yet, + and otherwise into a new file with the next counter (`semantic-02.json`). The file is + recorded in the open revision, dated today, and the open revisions' digests are computed + again; after a release revision, a new open revision is appended. It never changes an + existing value, and it writes nothing when the tree already breaks a rule. +3. The diff shows only added entries or a new file, and the edit to the open revision. +4. `--check` computes every case and reports moved, unpinned and no longer produced entries + without writing anything. + +Before a release, a value that no release lists can change only with the developer's +approval and a written migration assessment. `--remove ` (repeatable) takes the entry +out of its file and records the file's new sha256 in the open revision; it refuses an entry +in a file a release lists and writes nothing unless every entry can be removed. `--add` then +writes the new value. Editing the file by hand does not work: the file hashes report the +edit and the generator refuses to write over it. + +### Releasing (the release pull request) + +1. Precondition, in place since 2026-11-15 (developer decision): the repository ruleset + `protect-release-tags` on both `metamorphic-labs/orcapod-python` and + `nauticalab/orcapod-python` targets `refs/tags/v*` with the rules deletion, update and + non-fast-forward and no bypass actors. A `v*` tag can be created (the Release workflow + pushes it) but never moved or deleted. Agents never change repository settings. +2. Run `uv run python tests/test_hashing/golden/generate_golden_hashes.py --release X.Y.Z`. + It computes no hash. It appends a revision with `added: {}` and `release: "X.Y.Z"`, writes + its digest into `released_manifests.json` under `releases`, and at the data context's first + release sets `released_in` and the `manifest` pin. It refuses a version already released, + a development or local version, and a tree that breaks a rule. Try it first on a copy with + `--root ` (`store.copy_tree` writes one). +3. The Release workflow tags the commit it builds, which contains the release revision. From + then on every file listed up to that revision is frozen. +4. The tag and the PyPI wheel and sdist, which carry `v1.json` and `released_manifests.json`, + are records of what shipped, outside the repository. + +### Starting a new data context version + +Within `v1` a change to a released golden value is refused, with no exemption: also not for +a logical type's own identifier change. When such a change is wanted on purpose, it ships as +a new data context version, only in a minor release (before 1.0) or a major one, never in a +patch: + +1. The new manifest `v2.json` starts with `"golden": {"folder": "v2", "history": []}`, an + empty folder `data/v2/`, and the pin `"v2": {"manifest": null, "releases": {}}`. The + generator then writes `v2/-01.json` from the new code. +2. The `v1` code, manifest, golden folder and pins move to `orcapod-sanctuary`, which keeps + them computable. +3. This repository then carries only the `v2` folder and chain. + +## Threat model: who catches what + +Every drift scenario known to us fails at least one check, as long as git history and the +release tags are protected. Accidental drift is caught inside the tree; a deliberate rewrite +of the whole tree is caught only by records outside it, the tags and PyPI. + +| # | Scenario | Kind | Caught by | What fails | +| --- | --- | --- | --- | --- | +| 1 | A code change or a dependency bump in `uv.lock` moves a hash | accidental | Golden recompute | the entry's test names the case and both values | +| 2 | Someone regenerates or edits a released file to make the test pass | accidental | File hashes | the file's sha256 no longer matches its revision | +| 3 | An editor or formatter reformats a golden file | accidental | File hashes | as in 2, though no value moved | +| 4 | A Windows checkout converts line endings | accidental | `.gitattributes` (prevents it) | nothing: the bytes stay as committed | +| 5 | An entry is deleted from a released file | accidental | File hashes | the file's sha256 no longer matches | +| 6 | A released file is deleted | accidental | File hashes | a listed file is missing | +| 7 | A golden file is added but not listed | accidental | File hashes | an unlisted file is in the folder | +| 8 | A new file repeats a released entry name with another value | either | Duplicate-name check | the name is reported with both files | +| 9 | A golden entry is marked `skip` or `xfail` to silence it | either | No-skip rule | the golden test reports the mark | +| 10 | A hasher definition or library in a released `v1.json` is edited | accidental | Manifest pin | the manifest without `golden` no longer matches `manifest` | +| 11 | A release is cut without the release step | accidental | Release gate, final-release check | the Release workflow stops before tagging; a final version has no `releases` line | +| 12 | A released file is edited and its hash in `added` updated to match | deliberate | Chain | that revision's digest no longer computes, or the next revision's `previous` no longer matches | +| 13 | As 12, with every later digest computed again | deliberate | Release pins | the pinned release digest is no longer in the chain | +| 14 | As 13, with the pins rewritten too | deliberate | Tag check, PyPI | the file and the history differ from the release tag; the published `v1.json` shows the original chain | +| 15 | A release line and its revision are removed | deliberate | Tag check | the pins at the tag are missing in HEAD | +| 16 | A release tag is moved or deleted | deliberate | Tag protection, PyPI | GitHub refuses it; PyPI never accepts a second upload of a version | +| 17 | The checks themselves are edited to pass | deliberate | Review only | nothing automatic: see the residual risks | +| 18 | `main` is force-pushed with rewritten history | deliberate | Branch protection | outside this mechanism | + +## Each component in detail + +The recompute protects behaviour against the files, the file hashes protect the files against +the manifest, the chain and the pins protect the manifest against itself, and the tags and +PyPI protect everything against a rewrite of the tree. + +### Golden recompute + +`tests/test_hashing/golden/test_golden_hashes.py` + +- **Checks:** every entry of every golden file of the current folder is computed again from + the code and compared with its value, one test per entry; computed entries that no file + pins fail too. The values are also computed in subprocesses with different + `PYTHONHASHSEED` values. +- **Protects against:** behaviour drift: a refactor, an interpreter difference (3.11 to + 3.13), or a library bump in `uv.lock` that moves a pinned hash (1). +- **Cannot catch:** a golden file edited to match the new behaviour. The file hashes catch + that. + +### File hashes (the `added` map) + +`tests/test_contexts/test_data_context_v1.py`, rules in `tests/test_hashing/golden/store.py` + +- **Checks:** every listed file exists with exactly the recorded sha256, and every file in + the folder is listed by exactly one revision. +- **Protects against:** a change to a golden file's bytes, a deleted released file, an + unlisted file (2, 3, 5, 6, 7). "The test passes again after regenerating" becomes a + failure. +- **Cannot catch:** an edit that also updates the hash in `added` (12). + +### Chain (`previous` and `digest`) + +`tests/test_contexts/test_data_context_v1.py` + +- **Checks:** every revision has exactly its five fields; a release revision adds no file + and any other revision adds at least one; each digest computes from the revision's fields; + each `previous` is the digest of the revision before; no version is released twice. +- **Protects against:** an edit to one revision without touching the rest (12). It also gives + every change a date and a place in order. +- **Cannot catch:** a chain computed again from the edit onwards (13). + +### Release pins (`releases`) + +`tests/test_contexts/test_data_context_v1.py` + +- **Checks:** each pinned digest is the digest of the release revision of that version, and + each release revision is pinned. +- **Protects against:** a chain computed again (13). A rewrite has to touch a second file, + visible in review. +- **Cannot catch:** a rewrite that edits the pins as well (14). + +### Manifest pin (`manifest`) + +`tests/test_contexts/test_data_context_v1.py` + +- **Checks:** the manifest without its `golden` section has the pinned digest; + `released_in` is set exactly when the manifest is pinned, which is exactly when the chain + has a release, and names the first release. +- **Protects against:** an edit to the hashers, `definitions`, `libraries` or `previous` of a + released data context (10). It is computed over parsed JSON, so a whitespace edit does not + trip it. + +### Final-release check + +`tests/test_contexts/test_data_context_v1.py` + +- **Checks:** a final version of the library (not a development, pre-release or local + version; the version comes from the git tag) has its own `releases` line and a pinned + manifest. +- **Protects against:** a release cut without the release step (11). + +### Release gate + +`scripts/check_golden_tags.py --release-version X.Y.Z`, a step of the Release workflow's +build job before the tag is made; tested by `tests/test_check_golden_tags.py` + +- **Checks:** HEAD's `released_manifests.json` pins release `X.Y.Z` of the current data + context, its `manifest` pin is set, and the pinned digest is the release revision of + `X.Y.Z`, which is the last revision of HEAD's golden history: a golden file added after + the release step would ship unreleased, so such a commit is not tagged. It fails without + git. +- **Protects against:** a release cut without the release step (11). The final-release + check cannot catch it in the Release workflow, whose tests run before the tag exists, + under a development version. + +### Tag check + +`scripts/check_golden_tags.py`, a step of the `Run Tests` workflow and of the Release +workflow's build job before the tag is made (so a release dispatched from a branch that +rewrote an earlier release is refused); tested by `tests/test_check_golden_tags.py` + +- **Checks:** for each tag `vX.Y.Z` whose `released_manifests.json` pins release `X.Y.Z` of + the current data context (the one manifest at HEAD): every golden file listed by the + revisions up to that release exists in HEAD with the bytes it has at the tag; those + revisions are unchanged in HEAD's history; and the pins of the data context at the tag (the + manifest pin and every `releases` line) are unchanged in HEAD. +- **Protects against:** a full rewrite of files, chain and pins inside the tree (14, 15). + This is the anchor outside the tree. +- **Relies on:** protected tags and history. In CI (`CI=true`) it fails, never skips, when + git, the history (a shallow clone) or the tags are missing; outside CI it skips with a + message. With no release tag of the current data context it passes with nothing to check. + +### PyPI artifacts + +A record, not a check: the wheel and sdist of each release carry `v1.json` and +`released_manifests.json`, and PyPI never accepts a second upload of a version (16). + +### `.gitattributes` + +`tests/test_hashing/golden/data/** -text` keeps git from converting line endings on checkout, +so the bytes that are hashed are the bytes that were committed (4). + +### No-skip rule + +`tests/test_hashing/golden/test_golden_hashes.py` + +- **Checks:** no golden entry carries a `skip`, `skipif` or `xfail` mark, from its + parameters or added by any plugin, except the optional-package rule (ITL-871): the `torch` + group is skipped where torch is not installed, and an entry the installed torch cannot + compute (a newer dtype) is skipped. CI installs every extra. +- **Protects against:** a golden entry silenced instead of fixed (9). + +### Duplicate-name check + +`tests/test_hashing/golden/test_golden_hashes.py` and the generator + +- **Checks:** the folder holds only `-NN.json` files of known groups, and an entry name + appears in only one of them (8). + +## Residual risks and assumptions + +| Risk or assumption | Why it matters | What covers it | +| --- | --- | --- | +| Git history and `v*` tags are protected on both repositories | The tag check trusts the tags | The `protect-release-tags` ruleset on both repositories (in place since 2026-11-15) and branch protection | +| The checks themselves can be edited (17) | A weakened check passes | Review: changes to these tests and scripts get explicit review | +| Golden values are computed only with the `uv.lock` versions | A user's newer pyarrow, polars or numpy could hash differently unnoticed | ITL-1151: CI against the lowest and highest allowed dependency versions | +| A hash with no golden entry is not protected | Coverage is only as wide as `cases.py` | Add cases whenever a new stored or embedded hash appears | +| Values on `main` after the last release are open | An unreleased commit has no guarantee | Use a released version for anything that must stay stable | +| Optional groups skip without their package | `torch` values are unchecked where torch is missing | CI installs every extra | +| The `date` field cannot be verified | Informational | Order and integrity come from the chain and the tags | + +A deliberate attacker with merge rights who also rewrites history and tags is beyond what any +check inside the repository can stop; the PyPI artifacts remain the record of what shipped. + +## Rules + +- A released golden file never changes, and is never removed or renamed. +- New golden values can be added in any release, a patch included, with + `generate_golden_hashes.py`. +- No exemption: any change to a released golden value, including one caused by a logical + type's own identifier change, needs a new data context version. +- A new data context version ships only in a minor release (before 1.0) or a major one, never + in a patch. +- The release pull request runs `generate_golden_hashes.py --release X.Y.Z`, and the release + is tagged on a commit that contains it. +- When a new data context starts, the old context's code and golden files move to + `orcapod-sanctuary`. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 7545fee9e..29fb94d3f 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -3,6 +3,8 @@ This document is a developer-facing reference for every hash computation site in the orcapod-python framework. For conceptual background on the two identity chains (`content_hash` and `pipeline_hash`), see [Identity & Hashing](../concepts/identity.md). +Every hash listed here that Orcapod stores or embeds in a name is pinned as a golden value; +how released values are protected is described in [Golden Hashes](golden-hashes.md). --- @@ -424,7 +426,8 @@ definition. Consequences: change to the encoding of a supported construct needs a new label (`polars_expr_v2`). The canonical trees of a corpus of expressions are pinned in -`tests/test_hashing/test_polars_expr_hashing.py`, and their hashes in the golden suite. +`tests/test_hashing/test_polars_expr_hashing.py`, and their hashes in the golden suite +([Golden Hashes](golden-hashes.md)). ### 1d. Function values and parameter defaults @@ -726,7 +729,7 @@ topology suffix, encoding the full join lineage into each column name. | **Known exclusions** | When `record_id_column` is not specified, the row index is used as the provenance token; `record_id` changes if rows are **reordered** within the source table | The recipe recomputes a record id from the source id and the token alone. The value below is -the one pinned for this row in the golden hash file (`pipeline.json`): +the one pinned for this row in the golden hash file (`v1/pipeline-01.json`): ```python import uuid @@ -1297,7 +1300,9 @@ whenever any existing hash value can change (the hashers, the identity definitio record-id recipes, the libraries behind them). Orcapod v0.3 writes `std:v1:default`; everything stored before v0.3 is the earlier generation, `std:v0.1:default`. The manifest of the current data context is `orcapod/contexts/data/v1.json`. It is frozen at the release of -the library version that contains it and never edited afterwards. It names the hashers, the +the library version that contains it and never edited afterwards, except that its `golden` +history gains revisions as golden values are added and released +([Golden Hashes](golden-hashes.md)). It names the hashers, the definitions that live in code, the libraries the hashes depend on and what changed from the previous data context. Logical types are not part of it: they are registered in `orcapod/contexts/logical_types.json` and versioned by their own identifier. diff --git a/mkdocs.yml b/mkdocs.yml index 9ae18972c..412aa3a5a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,4 +70,5 @@ nav: - Types: api/types.md - Reference: - Hashing: reference/hashing.md + - Golden hashes: reference/golden-hashes.md - Troubleshooting: troubleshooting.md diff --git a/scripts/check_golden_tags.py b/scripts/check_golden_tags.py new file mode 100644 index 000000000..db007691c --- /dev/null +++ b/scripts/check_golden_tags.py @@ -0,0 +1,337 @@ +"""Check HEAD against every release tag of the current data context (ITL-946). + +A golden file that a release listed never changes, and the pins of a release never change. +The tests in the tree check the golden files against the manifest's history and the history +against ``released_manifests.json``; this check anchors all three to the release tags, +which no pull request can move. For each tag ``v`` whose +``src/orcapod/contexts/released_manifests.json`` pins release ``X.Y.Z`` of the current data +context, it checks in HEAD: + +- every golden file listed by the revisions up to that release exists with the bytes it has + at the tag; +- the revisions up to that release are present unchanged in the golden history; +- the pins of the current data context at the tag (the manifest pin and every ``releases`` + line) are present unchanged. + +The current data context is the one manifest in ``src/orcapod/contexts/data/`` at HEAD. + +Usage: + + uv run python scripts/check_golden_tags.py [--repo DIR] + +It exits 1 on a difference. Without git, history or tags (not a repository, a shallow +clone, no tags) it exits 1 in CI (``CI=true``) and 0 with a message elsewhere. The script +uses the standard library only. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from collections.abc import Sequence +from pathlib import Path, PurePosixPath +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] +CONTEXTS = PurePosixPath("src/orcapod/contexts") +PINS = CONTEXTS / "released_manifests.json" +GOLDEN_ROOT = PurePosixPath("tests/test_hashing/golden/data") + + +class GitUnavailableError(RuntimeError): + """git, the history or the tags cannot be read.""" + + +# Variables that point git at another repository than ``--repo`` (git(1), "Environment +# Variables"); they are removed so that the repository read is the one named, and +# ``--no-replace-objects`` keeps replace refs from substituting the objects read. +_GIT_LOCATION_VARIABLES = frozenset( + { + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_NAMESPACE", + "GIT_GRAFT_FILE", + "GIT_SHALLOW_FILE", + } +) + + +def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[bytes]: + env = {k: v for k, v in os.environ.items() if k not in _GIT_LOCATION_VARIABLES} + try: + return subprocess.run( + ["git", "--no-replace-objects", "-C", str(repo), *args], + capture_output=True, + check=False, + env=env, + ) + except FileNotFoundError as exc: + raise GitUnavailableError("git is not installed") from exc + + +def _show(repo: Path, rev: str, path: PurePosixPath) -> bytes | None: + """The bytes of ``path`` at ``rev``, or ``None`` when it has no such file.""" + result = _git(repo, "cat-file", "blob", f"{rev}:{path}") + return result.stdout if result.returncode == 0 else None + + +def _show_json(repo: Path, rev: str, path: PurePosixPath) -> Any: + """The parsed JSON of ``path`` at ``rev``, or ``None`` when absent or not JSON.""" + data = _show(repo, rev, path) + if data is None: + return None + try: + return json.loads(data) + except ValueError: + return None + + +def _require_history(repo: Path) -> list[str]: + """The repository's tags; raises when git, its history or its tags are missing.""" + inside = _git(repo, "rev-parse", "--is-inside-work-tree") + if inside.returncode != 0 or inside.stdout.strip() != b"true": + raise GitUnavailableError(f"{repo} is not a git work tree") + shallow = _git(repo, "rev-parse", "--is-shallow-repository") + if shallow.stdout.strip() != b"false": + raise GitUnavailableError( + "the clone is shallow; check out with the full history and tags " + "(actions/checkout fetch-depth: 0)" + ) + tags = _git(repo, "tag", "--list") + names = tags.stdout.decode().split() + if tags.returncode != 0 or not names: + raise GitUnavailableError("the repository has no tags; fetch them") + return names + + +def _current_version(repo: Path) -> str: + """The version of the one data context manifest at HEAD.""" + listing = _git(repo, "ls-tree", "--name-only", "HEAD", f"{CONTEXTS / 'data'}/") + names = [ + PurePosixPath(line).name + for line in listing.stdout.decode().splitlines() + if line.endswith(".json") + ] + if listing.returncode != 0 or len(names) != 1: + raise SystemExit( + f"cannot tell the current data context: manifests at HEAD are {names}" + ) + return names[0].removesuffix(".json") + + +def check_tag(repo: Path, tag: str, version: str) -> list[str] | None: + """Check HEAD against one tag. + + Args: + repo: Repository root. + tag: Tag name. + version: Current data context version. + + Returns: + ``None`` when the tag is not a release of ``version``; otherwise one sentence per + difference (empty when HEAD keeps everything the tag released). + """ + release = tag.removeprefix("v") + tag_pins = _show_json(repo, tag, PINS) + pin = tag_pins.get(version) if isinstance(tag_pins, dict) else None + if not isinstance(pin, dict) or not isinstance(pin.get("releases"), dict): + return None + if release not in pin["releases"]: + return None + problems: list[str] = [] + head_pins = _show_json(repo, "HEAD", PINS) + head_pin = head_pins.get(version) if isinstance(head_pins, dict) else None + if not isinstance(head_pin, dict): + return [f"{tag}: HEAD has no pin of data context {version}"] + if head_pin.get("manifest") != pin.get("manifest"): + problems.append(f"{tag}: the manifest pin of {version} changed") + head_releases = head_pin.get("releases") + head_releases = head_releases if isinstance(head_releases, dict) else {} + for name, digest in pin["releases"].items(): + if head_releases.get(name) != digest: + problems.append(f"{tag}: the releases line {name} of {version} changed") + + manifest_path = CONTEXTS / "data" / f"{version}.json" + manifest = _show_json(repo, tag, manifest_path) + golden = manifest.get("golden") if isinstance(manifest, dict) else None + if not isinstance(golden, dict): + return [*problems, f"{tag}: {manifest_path} has no golden section"] + history = golden.get("history") + if not isinstance(history, list): + return [*problems, f"{tag}: {manifest_path} has no golden history"] + end = next( + ( + i + for i, revision in enumerate(history) + if isinstance(revision, dict) + and revision.get("release") == release + and revision.get("digest") == pin["releases"][release] + ), + None, + ) + if end is None: + return [*problems, f"{tag}: the history has no release revision {release}"] + released = history[: end + 1] + head_manifest = _show_json(repo, "HEAD", manifest_path) + head_golden = ( + head_manifest.get("golden") if isinstance(head_manifest, dict) else None + ) + head_history = head_golden.get("history") if isinstance(head_golden, dict) else None + if not isinstance(head_history, list) or head_history[: end + 1] != released: + problems.append( + f"{tag}: the golden history up to release {release} changed in HEAD" + ) + folder = GOLDEN_ROOT / str(golden.get("folder")) + for revision in released: + added = revision.get("added") if isinstance(revision, dict) else None + if not isinstance(added, dict): + problems.append(f"{tag}: a released revision lists no files") + continue + for name in added: + path = folder / name + at_tag = _show(repo, tag, path) + at_head = _show(repo, "HEAD", path) + if at_head is None: + problems.append(f"{tag}: {path} is missing in HEAD") + elif at_head != at_tag: + problems.append(f"{tag}: {path} differs from the released file") + return problems + + +def check_release(repo: Path, release: str) -> list[str]: + """Check that HEAD records a release of the current data context, before it is tagged. + + The Release workflow runs this before it creates the tag: HEAD's + ``released_manifests.json`` pins ``release`` under the current context's + ``releases``, its manifest pin is set, and the pinned digest is the digest of the + release revision of ``release`` in HEAD's golden history, which is its last revision + (no golden file added after the release step ships unreleased). + + Args: + repo: Repository root. + release: The library version being released, without a leading ``v``. + + Returns: + One sentence per problem; empty when HEAD records the release. + """ + version = _current_version(repo) + step = ( + f"run generate_golden_hashes.py --release {release} in the release pull request" + ) + pins = _show_json(repo, "HEAD", PINS) + pin = pins.get(version) if isinstance(pins, dict) else None + releases = pin.get("releases") if isinstance(pin, dict) else None + if not isinstance(pin, dict) or not isinstance(releases, dict): + return [f"HEAD has no pin of data context {version}: {step}"] + if release not in releases: + return [ + f"HEAD does not record release {release} of data context {version}: {step}" + ] + problems: list[str] = [] + if pin.get("manifest") is None: + problems.append(f"the manifest of data context {version} is not pinned: {step}") + manifest = _show_json(repo, "HEAD", CONTEXTS / "data" / f"{version}.json") + golden = manifest.get("golden") if isinstance(manifest, dict) else None + history = golden.get("history") if isinstance(golden, dict) else None + entries: list[Any] = history if isinstance(history, list) else [] + revisions = [ + revision + for revision in entries + if isinstance(revision, dict) and revision.get("release") == release + ] + if [revision.get("digest") for revision in revisions] != [releases[release]]: + problems.append( + f"the pinned digest of {release} is not the release revision of {release} " + "in the golden history" + ) + elif entries[-1] is not revisions[0]: + problems.append( + f"the release revision of {release} is not the last revision of the golden " + "history: a golden file added after the release step would ship unreleased; " + f"tag the commit of the release step, or run --release {release} again after " + "the addition (a new version)" + ) + return problems + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the check. + + Args: + argv: Command-line arguments; ``None`` reads ``sys.argv``. + + Returns: + The exit status. + """ + parser = argparse.ArgumentParser( + description="Check HEAD against every release tag of the current data context." + ) + _ = parser.add_argument("--repo", type=Path, default=REPO_ROOT) + _ = parser.add_argument( + "--release-version", + metavar="X.Y.Z", + help=( + "instead of comparing with the tags, check that HEAD records this release " + "(the Release workflow's gate before it tags); fails without git" + ), + ) + args = parser.parse_args(argv) + if args.release_version is not None: + try: + problems = check_release(args.repo, args.release_version.removeprefix("v")) + except GitUnavailableError as exc: + print(f"error: cannot read HEAD: {exc}", file=sys.stderr) + return 1 + if problems: + print( + f"release {args.release_version} is not recorded in the golden history:", + file=sys.stderr, + ) + for problem in problems: + print(f" {problem}", file=sys.stderr) + return 1 + print(f"HEAD records release {args.release_version} in the golden history") + return 0 + in_ci = os.environ.get("CI", "").lower() == "true" + try: + tags = _require_history(args.repo) + except GitUnavailableError as exc: + if in_ci: + print(f"error: cannot check the release tags: {exc}", file=sys.stderr) + return 1 + print(f"skipped: cannot check the release tags: {exc}") + return 0 + version = _current_version(args.repo) + checked: list[str] = [] + problems: list[str] = [] + for tag in sorted(tags): + found = check_tag(args.repo, tag, version) + if found is None: + continue + checked.append(tag) + problems.extend(found) + if problems: + print( + "HEAD changes what a release of data context " + f"{version} shipped; a released golden file, revision or pin never changes:", + file=sys.stderr, + ) + for problem in problems: + print(f" {problem}", file=sys.stderr) + return 1 + if checked: + print(f"HEAD keeps what {', '.join(checked)} released (data context {version})") + else: + print(f"no release tag of data context {version}: nothing to check") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/orcapod/contexts/__init__.py b/src/orcapod/contexts/__init__.py index ffdbdc2ad..f86d17dcb 100644 --- a/src/orcapod/contexts/__init__.py +++ b/src/orcapod/contexts/__init__.py @@ -148,7 +148,7 @@ def get_context_info(version: str) -> dict[str, Any]: Example: >>> info = get_context_info("v1") >>> print(info['description']) - 'Data context of Orcapod v0.3: the hashing generation H1. Frozen when v0.3.0 is released; editable in place while that release is prepared.' + 'Data context of Orcapod v0.3: the hashing generation H1. Frozen when v0.3.0 is released, except its golden history, which only grows; editable in place while that release is prepared.' """ registry = _get_registry() return registry.get_context_info(version) diff --git a/src/orcapod/contexts/data/schemas/context_schema.json b/src/orcapod/contexts/data/schemas/context_schema.json index 739ba59cc..dded61b31 100644 --- a/src/orcapod/contexts/data/schemas/context_schema.json +++ b/src/orcapod/contexts/data/schemas/context_schema.json @@ -102,23 +102,59 @@ "golden": { "type": "object", "properties": { - "files": { - "type": "string" - }, - "algorithm": { - "type": "string" + "folder": { + "type": "string", + "description": "The golden folder of this data context, under tests/test_hashing/golden/data/" }, - "digest": { - "type": [ - "string", - "null" - ] + "history": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + }, + "release": { + "type": [ + "string", + "null" + ] + }, + "added": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "previous": { + "type": [ + "string", + "null" + ] + }, + "digest": { + "type": "string" + } + }, + "required": [ + "date", + "release", + "added", + "previous", + "digest" + ], + "additionalProperties": false + }, + "description": "Append-only chain of revisions. A revision lists the golden files it added, each with the sha256 of its bytes, or marks a release (added empty, release set); digest is the sha256 of the canonical JSON (sorted keys, no spaces) of date, release, added and previous" } }, "required": [ - "digest" + "folder", + "history" ], - "description": "Digest of the golden hash files this data context stands for; null while the manifest is open" + "additionalProperties": false, + "description": "The golden hash files of this data context (ITL-946). The revisions up to the last release revision, and the files they list, never change" }, "changes": { "type": "array", diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index 538faac3f..28ddf132d 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -1,7 +1,7 @@ { "context_key": "std:v1:default", "version": "v1", - "description": "Data context of Orcapod v0.3: the hashing generation H1. Frozen when v0.3.0 is released; editable in place while that release is prepared.", + "description": "Data context of Orcapod v0.3: the hashing generation H1. Frozen when v0.3.0 is released, except its golden history, which only grows; editable in place while that release is prepared.", "released_in": null, "previous": "std:v0.1:default", "file_hasher": { @@ -227,9 +227,23 @@ "polars": "1.41.2: serialisation of filter expressions, which is hashed" }, "golden": { - "files": "tests/test_hashing/golden/data/*.json", - "algorithm": "sha256 over the files in name order; each contributes its name, a NUL byte, its bytes and a NUL byte", - "digest": null + "folder": "v1", + "history": [ + { + "date": "2026-11-15", + "release": null, + "added": { + "arrow-01.json": "d62d1fd8ef374e4fd55a82d7eeb09f3ed5906b9de6fc96b6b1596edcb1579da8", + "datagram-01.json": "1151c8d391f6f7303657da83720e4fcfc8c5b5082e9cd670ef51ba8886575ca9", + "pipeline-01.json": "e4e4e71b4ef9bdd9211b5c5dc15db03ee333f66af606a832d1b5fccfa21f44da", + "schema-01.json": "f1bfda8bc4da3ecfc7973998c5579b5a3f151f7ec1f9c75c20ac5ad5244856ae", + "semantic-01.json": "7fd0ccad63a7e3f83314b27e9d6df87a8cb15694758f92e517a0d9c988f98a53", + "torch-01.json": "99af07350bc23a0200cecfd786ccd6c17985c2270ad5e3f44aca74273a1bcdc2" + }, + "previous": null, + "digest": "e20d0af197e1d2eca72f0524e626cb4a46b37d261cc7a06ea5768cce65d153a5" + } + ] }, "changes": [ { diff --git a/src/orcapod/contexts/released_manifests.json b/src/orcapod/contexts/released_manifests.json index 82c352d29..49ff26e31 100644 --- a/src/orcapod/contexts/released_manifests.json +++ b/src/orcapod/contexts/released_manifests.json @@ -1,3 +1,6 @@ { - "v1": null + "v1": { + "manifest": null, + "releases": {} + } } diff --git a/tests/test_check_golden_tags.py b/tests/test_check_golden_tags.py new file mode 100644 index 000000000..8e329711d --- /dev/null +++ b/tests/test_check_golden_tags.py @@ -0,0 +1,362 @@ +"""The release-tag check of the golden files (ITL-946): ``scripts/check_golden_tags.py``. + +Every test runs the script as a command against a temporary git repository holding a copy of +the golden tree, released with ``generate_golden_hashes.py --release`` and tagged. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from orcapod.contexts import get_default_context +from tests.test_hashing.golden import store +from tests.test_hashing.golden.generate_golden_hashes import dump, main + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "check_golden_tags.py" +VERSION = get_default_context().version + + +def _env(*, ci: bool, ceiling: Path | None = None) -> dict[str, str]: + env = { + k: v for k, v in os.environ.items() if k != "CI" and not k.startswith("GIT_") + } + env.update( + GIT_CONFIG_GLOBAL=os.devnull, + GIT_CONFIG_NOSYSTEM="1", + GIT_AUTHOR_NAME="test", + GIT_AUTHOR_EMAIL="test@example.invalid", + GIT_COMMITTER_NAME="test", + GIT_COMMITTER_EMAIL="test@example.invalid", + ) + if ci: + env["CI"] = "true" + if ceiling is not None: + env["GIT_CEILING_DIRECTORIES"] = str(ceiling) + return env + + +def _git(repo: Path, *args: str) -> None: + _ = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + env=_env(ci=False), + ) + + +def _commit(repo: Path, message: str) -> None: + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "--no-gpg-sign", "-m", message) + + +def _check(repo: Path, *, ci: bool = False) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(repo)], + capture_output=True, + text=True, + check=False, + env=_env(ci=ci, ceiling=repo.parent), + ) + + +def _folder(repo: Path) -> Path: + return store.golden_folder( + repo, store.read_json(store.manifest_path(repo, VERSION)) + ) + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + """A repository with an open commit tagged ``v0.2.0`` and a release tagged ``v0.3.0``.""" + root = tmp_path / "repo" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") + _git(root.parent, "init", "-q", str(root)) + _commit(root, "open data context") + _git(root, "tag", "v0.2.0") + main(["--root", str(root), "--release", "0.3.0"]) + _commit(root, "release 0.3.0") + _git(root, "tag", "v0.3.0") + return root + + +def test_head_at_the_release_passes(repo: Path) -> None: + result = _check(repo, ci=True) + assert result.returncode == 0, result.stderr + assert "v0.3.0" in result.stdout + + +def test_a_tag_that_released_nothing_is_not_checked(tmp_path: Path) -> None: + root = tmp_path / "repo" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") + _git(root.parent, "init", "-q", str(root)) + _commit(root, "open data context") + _git(root, "tag", "v0.2.0") + result = _check(root, ci=True) + assert result.returncode == 0, result.stderr + assert "nothing to check" in result.stdout + + +def test_a_golden_file_added_after_the_release_passes(repo: Path) -> None: + folder = _folder(repo) + name = store.next_file_name(store.group_files(folder)["schema"][-1].name) + _ = (folder / name).write_text(dump({"new.entry": "x"})) + manifest = store.read_json(store.manifest_path(repo, VERSION)) + manifest["golden"]["history"] = store.with_file( + manifest["golden"]["history"], + name, + store.file_digest(folder / name), + "2026-11-20", + ) + store.write_json(store.manifest_path(repo, VERSION), manifest) + _commit(repo, f"add {name}") + assert store.tree_problems(repo, VERSION) == [] + result = _check(repo, ci=True) + assert result.returncode == 0, result.stderr + + +def test_an_edited_released_file_fails(repo: Path) -> None: + path = _folder(repo) / "semantic-01.json" + _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) + _commit(repo, "edit a released file") + result = _check(repo) + assert result.returncode == 1 + assert "semantic-01.json differs from the released file" in result.stderr + + +def test_a_released_file_rewritten_with_its_history_and_pins_fails(repo: Path) -> None: + """A rewrite of file, revision and pins inside the tree still differs from the tag.""" + folder = _folder(repo) + path = folder / "semantic-01.json" + _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) + manifest = store.read_json(store.manifest_path(repo, VERSION)) + history = manifest["golden"]["history"] + history[0]["added"]["semantic-01.json"] = store.file_digest(path) + for i, revision in enumerate(history): + revision["previous"] = history[i - 1]["digest"] if i else None + revision["digest"] = store.revision_digest(revision) + store.write_json(store.manifest_path(repo, VERSION), manifest) + pins = store.read_json(store.pins_path(repo)) + pins[VERSION]["releases"]["0.3.0"] = history[-1]["digest"] + store.write_json(store.pins_path(repo), pins) + assert store.tree_problems(repo, VERSION) == [] + _commit(repo, "rewrite the release") + result = _check(repo) + assert result.returncode == 1 + assert "semantic-01.json differs" in result.stderr + assert "the releases line 0.3.0" in result.stderr + assert "history up to release 0.3.0 changed" in result.stderr + + +def test_a_deleted_released_file_fails(repo: Path) -> None: + (_folder(repo) / "torch-01.json").unlink() + _commit(repo, "delete a released file") + result = _check(repo) + assert result.returncode == 1 + assert "torch-01.json is missing in HEAD" in result.stderr + + +def test_a_removed_releases_line_fails(repo: Path) -> None: + pins = store.read_json(store.pins_path(repo)) + pins[VERSION]["releases"].clear() + store.write_json(store.pins_path(repo), pins) + _commit(repo, "drop the releases line") + result = _check(repo) + assert result.returncode == 1 + assert "the releases line 0.3.0" in result.stderr + + +def test_a_changed_manifest_pin_fails(repo: Path) -> None: + pins = store.read_json(store.pins_path(repo)) + pins[VERSION]["manifest"] = "0" * 64 + store.write_json(store.pins_path(repo), pins) + _commit(repo, "change the manifest pin") + result = _check(repo) + assert result.returncode == 1 + assert "manifest pin" in result.stderr + + +def test_outside_a_repository_it_skips_locally_and_fails_in_ci(tmp_path: Path) -> None: + plain = tmp_path / "plain" + store.copy_tree(store.PROJECT_ROOT, plain, VERSION) + store.reopen_tree(plain, VERSION, "2026-11-15") + local = _check(plain) + assert local.returncode == 0 + assert "skipped" in local.stdout + ci = _check(plain, ci=True) + assert ci.returncode == 1 + assert "not a git work tree" in ci.stderr + + +def test_a_shallow_clone_fails_in_ci(repo: Path, tmp_path: Path) -> None: + clone = tmp_path / "clone" + _ = subprocess.run( + ["git", "clone", "-q", "--depth", "1", f"file://{repo}", str(clone)], + check=True, + capture_output=True, + env=_env(ci=False), + ) + result = _check(clone, ci=True) + assert result.returncode == 1 + assert "shallow" in result.stderr + assert _check(clone).returncode == 0 + + +def test_a_repository_without_tags_fails_in_ci(tmp_path: Path) -> None: + root = tmp_path / "repo" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") + _git(root.parent, "init", "-q", str(root)) + _commit(root, "open data context") + result = _check(root, ci=True) + assert result.returncode == 1 + assert "no tags" in result.stderr + + +# --------------------------------------------------------------------------- +# --release-version: the Release workflow's gate before a tag is made +# --------------------------------------------------------------------------- + + +def _gate(repo: Path, version: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--repo", + str(repo), + "--release-version", + version, + ], + capture_output=True, + text=True, + check=False, + env=_env(ci=False, ceiling=repo.parent), + ) + + +def test_the_gate_passes_a_recorded_release(repo: Path) -> None: + result = _gate(repo, "0.3.0") + assert result.returncode == 0, result.stderr + assert "0.3.0" in result.stdout + + +def test_the_gate_refuses_a_release_without_its_golden_step(repo: Path) -> None: + result = _gate(repo, "0.3.1") + assert result.returncode == 1 + assert "generate_golden_hashes.py --release 0.3.1" in result.stderr + + +def test_the_gate_refuses_an_open_data_context(tmp_path: Path) -> None: + root = tmp_path / "repo" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") + _git(root.parent, "init", "-q", str(root)) + _commit(root, "open data context") + result = _gate(root, "0.3.0") + assert result.returncode == 1 + assert "0.3.0" in result.stderr + + +def test_the_gate_refuses_a_pin_that_is_not_the_release_revision(repo: Path) -> None: + pins = store.read_json(store.pins_path(repo)) + pins[VERSION]["releases"]["0.3.0"] = "0" * 64 + store.write_json(store.pins_path(repo), pins) + _commit(repo, "tamper with the pin") + result = _gate(repo, "0.3.0") + assert result.returncode == 1 + assert "release revision" in result.stderr + + +def test_the_gate_fails_outside_a_repository_even_locally(tmp_path: Path) -> None: + plain = tmp_path / "plain" + store.copy_tree(store.PROJECT_ROOT, plain, VERSION) + result = _gate(plain, "0.3.0") + assert result.returncode == 1 + + +def test_the_gate_refuses_a_golden_addition_after_the_release_step(repo: Path) -> None: + """The tagged commit's last revision is the release: no unreleased file ships with it.""" + folder = _folder(repo) + name = store.next_file_name(store.group_files(folder)["schema"][-1].name) + _ = (folder / name).write_text(dump({"new.entry": "x"})) + manifest = store.read_json(store.manifest_path(repo, VERSION)) + manifest["golden"]["history"] = store.with_file( + manifest["golden"]["history"], + name, + store.file_digest(folder / name), + "2026-11-20", + ) + store.write_json(store.manifest_path(repo, VERSION), manifest) + _commit(repo, f"add {name} after the release step") + result = _gate(repo, "0.3.0") + assert result.returncode == 1 + assert "last revision" in result.stderr + + +def test_inherited_git_location_variables_do_not_redirect_the_check( + repo: Path, tmp_path: Path +) -> None: + """``--repo`` is the repository read, whatever ``GIT_DIR`` and friends point at.""" + other = tmp_path / "other" + _ = subprocess.run( + ["git", "clone", "-q", str(repo), str(other)], + check=True, + capture_output=True, + env=_env(ci=False), + ) + path = _folder(other) / "semantic-01.json" + _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) + _commit(other, "edit a released file") + _git(other, "fetch", "-q", "--tags") + assert _check(other).returncode == 1 + env = _env(ci=True, ceiling=tmp_path) + env.update( + GIT_DIR=str(repo / ".git"), + GIT_WORK_TREE=str(other), + GIT_COMMON_DIR=str(repo / ".git"), + GIT_OBJECT_DIRECTORY=str(repo / ".git" / "objects"), + ) + result = subprocess.run( + [sys.executable, str(SCRIPT), "--repo", str(other)], + capture_output=True, + text=True, + check=False, + env=env, + ) + assert result.returncode == 1, result.stdout + assert "semantic-01.json differs" in result.stderr + + +def test_tags_of_earlier_data_contexts_pass_with_nothing_to_check( + tmp_path: Path, +) -> None: + """The Release workflow's first release of a context: older tags only, none of it. + + As on the public repository before v0.3.0: tags whose commits have no + ``released_manifests.json`` (v0.1.x) or the old ``{"v1": null}`` form (v0.2.0). In CI + the check passes with nothing to check; it never fails for want of a release tag. + """ + root = tmp_path / "repo" + pins = root / store.PINS_FILE + pins.parent.mkdir(parents=True) + _ = (root / "README.md").write_text("old release\n") + _git(root.parent, "init", "-q", str(root)) + _commit(root, "v0.1.x: no released_manifests.json") + _git(root, "tag", "v0.1.4") + _ = pins.write_text('{\n "v1": null\n}\n') + _commit(root, "v0.2.0: the old pin form") + _git(root, "tag", "v0.2.0") + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") + _commit(root, "the open v1 data context") + result = _check(root, ci=True) + assert result.returncode == 0, result.stderr + assert "nothing to check" in result.stdout diff --git a/tests/test_contexts/test_data_context_v1.py b/tests/test_contexts/test_data_context_v1.py index ad939c231..2ff80005f 100644 --- a/tests/test_contexts/test_data_context_v1.py +++ b/tests/test_contexts/test_data_context_v1.py @@ -1,21 +1,20 @@ """The data context of v0.3 is ``std:v1:default``, and released manifests are frozen (ITL-944). -A data context version names one generation of hashing. Its manifest file -(``src/orcapod/contexts/data/.json``) must not change once released, so -``released_manifests.json`` pins the sha256 of every released file. The manifest of the -version under development is listed with ``null`` until the release-preparation commit -sets its digest, once. +A data context version names one generation of hashing. ``released_manifests.json`` pins, +for each version, the sha256 of its manifest without the ``golden`` section (``null`` while +the manifest is open) and, under ``releases``, the digest of each library release's +revision in the manifest's golden history (ITL-946). The rules of the golden history and +its folder are stated in ``tests/test_hashing/golden/store.py``. """ from __future__ import annotations -import hashlib import json from importlib import resources from pathlib import Path +from typing import Any import pytest -from packaging.version import Version import orcapod from orcapod import contexts @@ -23,13 +22,14 @@ PythonTypeHandlerRegistry, ) from orcapod.semantic_types.universal_converter import UniversalTypeConverter +from tests.test_hashing.golden import store CONTEXTS_DIR = Path(str(resources.files("orcapod.contexts"))) MANIFEST_DIR = CONTEXTS_DIR / "data" -def _pins() -> dict[str, str | None]: - """The pinned sha256 of each manifest, by version; ``None`` for an open manifest.""" +def _pins() -> dict[str, dict[str, Any]]: + """The pin of each manifest, by version: ``{"manifest": ..., "releases": {...}}``.""" return json.loads((CONTEXTS_DIR / "released_manifests.json").read_text()) @@ -38,6 +38,16 @@ def _manifest_files() -> dict[str, Path]: return {path.stem: path for path in sorted(MANIFEST_DIR.glob("*.json"))} +def _manifest(version: str) -> dict[str, Any]: + """A shipped manifest as parsed JSON.""" + return json.loads(_manifest_files()[version].read_text()) + + +def _history(version: str) -> list[dict[str, Any]]: + """The golden history of a shipped manifest.""" + return _manifest(version)["golden"]["history"] + + def test_default_data_context_is_v1() -> None: """The default data context has key ``std:v1:default`` and version ``v1``.""" assert contexts.get_default_context_key() == "std:v1:default" @@ -56,60 +66,70 @@ def test_every_manifest_is_listed() -> None: assert set(_pins()) == set(_manifest_files()) -def test_released_manifests_are_unchanged() -> None: - """Each pinned manifest file still has the sha256 recorded for it.""" - files = _manifest_files() - for version, pinned in _pins().items(): - if pinned is None: - continue - found = hashlib.sha256(files[version].read_bytes()).hexdigest() - assert found == pinned, ( - f"The released data context manifest {files[version].name} was edited. " - "A released manifest is frozen: a change to hashing needs a new data " - "context version with its own manifest file." - ) +@pytest.mark.parametrize("version", sorted(_pins())) +def test_a_pinned_manifest_is_unchanged(version: str) -> None: + """A pinned manifest without its golden section still has the pinned sha256.""" + pinned = _pins()[version]["manifest"] + if pinned is None: + return + assert store.fixed_manifest_digest(_manifest(version)) == pinned, ( + f"The released data context manifest {version}.json was edited. A released " + "manifest is frozen: a change to hashing needs a new data context version with " + "its own manifest file." + ) def test_only_the_default_manifest_is_open() -> None: """At most one manifest has no pinned sha256, and it is the default version's.""" - open_versions = [version for version, pinned in _pins().items() if pinned is None] + open_versions = [v for v, pin in _pins().items() if pin["manifest"] is None] assert open_versions in ([], [contexts.get_default_context().version]) -def test_a_final_release_ships_no_open_manifest() -> None: - """A final release (not a prerelease or local build) pins every manifest.""" - version = Version(orcapod.__version__) - if version.is_prerelease or version.local is not None: - # A development build or a release candidate: the manifest may still be open. - return - assert None not in _pins().values(), ( - "A release must freeze its data context manifest: set its sha256 in " - "src/orcapod/contexts/released_manifests.json." - ) +def test_a_final_release_has_its_releases_line_and_a_pinned_manifest() -> None: + """A final release (not dev, rc or local) is recorded in the pins of its context.""" + pin = _pins()[contexts.get_default_context().version] + assert store.final_release_problems(pin, orcapod.__version__) == [] # --------------------------------------------------------------------------- -# The contents of the manifest (ITL-938 design document, section 7.4) +# The golden history (ITL-946) # --------------------------------------------------------------------------- -GOLDEN_DIR = Path(__file__).parents[1] / "test_hashing" / "golden" / "data" +@pytest.mark.parametrize("version", sorted(_pins())) +def test_the_golden_history_chain_recomputes(version: str) -> None: + """Each revision's digest recomputes and names the revision before it.""" + assert store.history_problems(_history(version)) == [] -def _v1() -> dict[str, object]: - """The v1 manifest as parsed JSON.""" - return json.loads((MANIFEST_DIR / "v1.json").read_text()) +@pytest.mark.parametrize("version", sorted(_pins())) +def test_every_golden_file_is_listed_once_with_its_hash(version: str) -> None: + """Every listed file exists with its sha256; every file is listed by one revision.""" + manifest = _manifest(version) + folder = store.golden_folder(store.PROJECT_ROOT, manifest) + assert store.folder_problems(manifest["golden"]["history"], folder) == [] -def golden_digest() -> str: - """The digest a manifest's ``golden`` field holds once the manifest is frozen. - sha256 over the golden hash files in name order; each file contributes its name, a - NUL byte, its bytes and a NUL byte. +@pytest.mark.parametrize("version", sorted(_pins())) +def test_the_pins_agree_with_the_golden_history(version: str) -> None: + """Each pinned release is a release revision with that version and digest, and the + reverse; ``released_in`` is set exactly when the manifest is pinned. """ - digest = hashlib.sha256() - for path in sorted(GOLDEN_DIR.glob("*.json")): - digest.update(path.name.encode() + b"\0" + path.read_bytes() + b"\0") - return digest.hexdigest() + manifest = _manifest(version) + problems = store.pin_problems( + manifest, _pins()[version], manifest["golden"]["history"] + ) + assert problems == [] + + +# --------------------------------------------------------------------------- +# The contents of the manifest (ITL-938 design document, section 7.4) +# --------------------------------------------------------------------------- + + +def _v1() -> dict[str, object]: + """The v1 manifest as parsed JSON.""" + return json.loads((MANIFEST_DIR / "v1.json").read_text()) def test_v1_follows_the_pre_v1_data_context() -> None: @@ -137,23 +157,6 @@ def test_v1_names_its_definitions_libraries_and_changes() -> None: ) -def test_released_in_is_set_exactly_when_the_manifest_is_pinned() -> None: - """An open manifest has no ``released_in`` and no golden digest; a frozen one has both.""" - pinned = _pins()["v1"] - manifest = _v1() - golden = manifest["golden"] - assert isinstance(golden, dict) - if pinned is None: - assert manifest["released_in"] is None - assert golden["digest"] is None - else: - assert isinstance(manifest["released_in"], str) and manifest["released_in"] - assert golden["digest"] == golden_digest(), ( - "The golden hash files differ from the ones this released data context " - "stands for. A released data context's values must not move." - ) - - def test_v1_lists_no_logical_types() -> None: """Logical types are outside the data context version: registering one edits no manifest.""" text = (MANIFEST_DIR / "v1.json").read_text() diff --git a/tests/test_hashing/golden/README.md b/tests/test_hashing/golden/README.md new file mode 100644 index 000000000..a1749c496 --- /dev/null +++ b/tests/test_hashing/golden/README.md @@ -0,0 +1,26 @@ +# Golden hash values + +Every hash Orcapod stores or embeds in a name is pinned here and checked on every CI run. The +full description, with the threat model and what each check protects against, is +[`docs/reference/golden-hashes.md`](../../../docs/reference/golden-hashes.md). + +In short: + +- `data//-NN.json` holds the values of the current data context; the folder is + named by `golden.folder` in `src/orcapod/contexts/data/.json`. +- `golden.history` in that manifest lists every file with the sha256 of its bytes, in a chain + of revisions; a release appends a revision with `added: {}` and pins its digest in + `src/orcapod/contexts/released_manifests.json`. +- A file listed by a released revision never changes. New values go into new files, in any + release. A change to a released value needs a new data context version. + +| File | Role | +| --- | --- | +| `cases.py` | The cases. Never move or rename a fixture: they hash with their module path. | +| `generate_golden_hashes.py` | Adds new entries (default), reports drift (`--check`), removes an unreleased entry (`--remove`), records a release (`--release X.Y.Z`). Read its header first. | +| `store.py` | The rules: file names, digests, history, pins. | +| `test_golden_hashes.py` | Computes every entry again; refuses skips but the optional-package rule. | +| `test_golden_store.py` | The rules and the generator, on copies of the tree. | + +`tests/test_contexts/test_data_context_v1.py` checks the repository's history and pins, and +`scripts/check_golden_tags.py` (a CI step) checks HEAD against every release tag. diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 132923b0b..d5dfaaa76 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -2,8 +2,9 @@ Each case computes a flat ``{entry: value}`` map. ``compute_group`` prefixes each entry with its case name, so a golden file entry reads ``.``. The values are pinned in -``data/.json`` by ``generate_golden_hashes.py`` and checked by -``test_golden_hashes.py``. +the golden folder of the current data context, ``data//-NN.json``, by +``generate_golden_hashes.py`` and checked by ``test_golden_hashes.py`` (ITL-946; see +``README.md``). DO NOT MOVE OR RENAME anything defined in this module. Function pods, dataclasses and pydantic models hash together with their defining module and qualified name, so moving them @@ -36,6 +37,7 @@ import pyarrow as pa import pydantic +import orcapod as op from orcapod import dataclass as structural_dataclass from orcapod import function_pod @@ -46,8 +48,8 @@ OPTIONAL_GROUPS: dict[str, str] = {"torch": "torch"} """Groups that need an optional dependency, mapped to the module they need (ITL-871). -Such a group is computed, generated and checked only where the module is installed; its -golden file is never rewritten without it. +Such a group is computed, added to and checked only where the module is installed; a +golden file is never rewritten, with the module or without it (ITL-946). """ @@ -78,7 +80,8 @@ def case(group: str, name: str) -> Callable[[CaseFn], CaseFn]: """Register a golden case under ``group``. Args: - group: One of ``GROUPS``; selects the golden file the entries are written to. + group: One of ``GROUPS``; selects the golden files (``-NN.json``) that + pin the entries. name: Case name, used as the prefix of every entry the case returns. Returns: @@ -2060,3 +2063,162 @@ def stream(ks: list[list[int]], xs: list[int]) -> ArrowTableStream: "as_given": stream([[2], [1]], [20, 10]).content_hash().to_string(), "reversed": stream([[1], [2]], [10, 20]).content_hash().to_string(), } + + +# --------------------------------------------------------------------------- +# Cases brought in from tests/test_hashing/hash_samples (ITL-946) +# +# The values, annotations, schemas and annotated functions that the older sample files +# pinned and no case above did. The functions are defined here, so their pins carry this +# module's path. DO NOT MOVE OR RENAME them. +# --------------------------------------------------------------------------- + +SAMPLE_VALUES: dict[str, Any] = { + "int_42": 42, + "float_pi": 3.14159, + "float_negative_e": -2.71828, + "str_hello": "hello", + "str_hello_world": "Hello, World!", + "str_special_chars": "Special chars: !@#$%^&*()", + "str_unicode": "Unicode: 你好, Привет, こんにちは", + "bytes_hello": b"hello", + "bytes_binary": b"\x00\x01\x02\x03", + "bytearray_hello_world": bytearray(b"hello world"), + "bytearray_abc": bytearray([65, 66, 67]), + "list_strs": ["a", "b", "c"], + "list_mixed": [1, "a", True], + "set_empty": set(), + "set_strs": {"a", "b", "c"}, + "frozenset_empty": frozenset(), + "tuple_empty": (), + "dict_one": {"a": 1}, + "dict_other_order": {"b": 1, "a": 2}, + "dict_nested": {"nested": {"a": 1, "b": 2}}, + "nested_lists": [1, [2, [3, [4, [5]]]]], + "nested_dicts": {"a": {"b": {"c": {"d": {"e": 42}}}}}, + "nested_dict_list": {"a": [1, 2, {"b": [3, 4, {"c": 5}]}]}, + "list_of_dicts": [{"a": 1}, {"b": 2}, {"c": [3, 4, 5]}], + "dict_keys_values": {"keys": ["a", "b", "c"], "values": [1, 2, 3]}, + "list_of_dicts_with_lists": [{"a": 1, "b": [2, 3]}, {"c": 4, "d": [5, 6]}], + "dict_users": {"users": [{"name": "Alice", "age": 30}, {"name": "Bob", "age": 25}]}, + "dict_points_labels": { + "data": {"points": [[1, 2], [3, 4], [5, 6]], "labels": ["A", "B", "C"]} + }, + "ordered_dict": collections.OrderedDict([("a", 1), ("b", 2), ("c", 3)]), + "lists_and_dict": [[1, 2], [3, 4], {"a": [5, 6], "b": [7, 8]}], +} + + +@case("semantic", "sample_value") +def semantic_sample_values() -> dict[str, Any]: + """Semantic hashes of the plain Python values of the older hash sample file.""" + hasher = _ctx().semantic_hasher + return {k: hasher.hash_object(v).to_string() for k, v in SAMPLE_VALUES.items()} + + +def _annotations() -> dict[str, Any]: + import typing + + import orcapod as op + + return { + "file": op.File, + "directory": op.Directory, + "list_file": list[op.File], + "dict_str_file": dict[str, op.File], + "int_or_str": int | str, + "file_or_none": op.File | None, + "optional_file": typing.Optional[op.File], # noqa: UP045 (the typing form is the case) + } + + +@case("semantic", "annotation") +def semantic_annotations() -> dict[str, Any]: + """Semantic hashes of type annotations over Orcapod's logical types (ITL-638).""" + hasher = _ctx().semantic_hasher + return {k: hasher.hash_object(t).to_string() for k, t in _annotations().items()} + + +@case("schema", "semantic_schema") +def schema_semantic_hashes() -> dict[str, Any]: + """Semantic hashes of ``Schema`` objects with primitive and logical types.""" + import orcapod as op + from orcapod.types import Schema + + schemas = { + "x_int": Schema({"x": int}), + "x_int_y_str": Schema({"x": int, "y": str}), + "f_file": Schema({"f": op.File}), + "d_directory": Schema({"d": op.Directory}), + "p_path": Schema({"p": op.Path}), + "x_int_f_file": Schema({"x": int, "f": op.File}), + "f_file_d_directory": Schema({"f": op.File, "d": op.Directory}), + "f_optional_file": Schema({"f": op.File | None}), + } + hasher = _ctx().semantic_hasher + return {k: hasher.hash_object(s).to_string() for k, s in schemas.items()} + + +def golden_annotated_none(): + """Function without annotations (ITL-946). DO NOT MOVE.""" + return None + + +def golden_annotated_builtin(x: int, y: str) -> float: + """Function annotated with builtins (ITL-946). DO NOT MOVE.""" + return float(x) + + +def golden_annotated_file_param(f: op.File) -> str: + """Function taking a ``File`` (ITL-946). DO NOT MOVE.""" + return str(f) + + +def golden_annotated_file_return(s: str) -> op.File: + """Function returning a ``File`` (ITL-946). DO NOT MOVE.""" + return op.File(s) # pyright: ignore[reportArgumentType] (never called; only hashed) + + +def golden_annotated_file_list(files: list[op.File]) -> list[str]: + """Function taking a list of ``File`` (ITL-946). DO NOT MOVE.""" + return [] + + +def golden_annotated_optional_file(f: op.File | None) -> op.File | None: + """Function taking and returning an optional ``File`` (ITL-946). DO NOT MOVE.""" + return f + + +def golden_annotated_mixed(f: op.File, n: int) -> op.Directory: + """Function mixing ``File``, ``int`` and ``Directory`` (ITL-946). DO NOT MOVE.""" + return op.Directory(str(f)) # pyright: ignore[reportArgumentType] (never called) + + +@case("semantic", "annotated_function") +def semantic_annotated_functions() -> dict[str, Any]: + """Hashes of annotated functions: the extracted signature info and the function value.""" + from orcapod.hashing.semantic_hashing.function_info_extractors import ( + FunctionSignatureExtractor, + ) + + functions = { + "none": golden_annotated_none, + "builtin": golden_annotated_builtin, + "file_param": golden_annotated_file_param, + "file_return": golden_annotated_file_return, + "file_list": golden_annotated_file_list, + "optional_file": golden_annotated_optional_file, + "mixed": golden_annotated_mixed, + } + hasher = _ctx().semantic_hasher + extractor = FunctionSignatureExtractor( + include_module=True, + include_defaults=True, + type_converter=_ctx().type_converter, + ) + out: dict[str, Any] = {} + for name, fn in functions.items(): + info = extractor.extract_function_info(fn) + out[f"{name}.info"] = hasher.hash_object(info).to_string() + out[f"{name}.object"] = hasher.hash_object(fn).to_string() + return out diff --git a/tests/test_hashing/golden/conftest.py b/tests/test_hashing/golden/conftest.py new file mode 100644 index 000000000..715ceefbb --- /dev/null +++ b/tests/test_hashing/golden/conftest.py @@ -0,0 +1,36 @@ +"""Collection-time refusal of a skipped or xfailed golden entry (ITL-946). + +The check lives here, not in ``test_golden_hashes.py``, so a mark on that module (a +``pytestmark`` skip) cannot switch it off together with the entries. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from tests.test_hashing.golden.marks import golden_entry_violations + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[Any]) -> None: + """Fail the collection when a golden entry carries a forbidden skip or xfail. + + Runs after the other collection hooks, so marks they add are seen. + + Args: + config: The pytest configuration (unused). + items: The collected items. + + Raises: + pytest.UsageError: A golden entry is marked skip or xfail other than by the + optional-package rule. + """ + found = golden_entry_violations(items) + if found: + lines = "\n".join(f" {node}: {marks}" for node, marks in sorted(found.items())) + raise pytest.UsageError( + "golden entries may not be skipped or expected to fail, except by the " + f"optional-package rule (ITL-871, ITL-946):\n{lines}" + ) diff --git a/tests/test_hashing/golden/data/arrow.json b/tests/test_hashing/golden/data/v1/arrow-01.json similarity index 100% rename from tests/test_hashing/golden/data/arrow.json rename to tests/test_hashing/golden/data/v1/arrow-01.json diff --git a/tests/test_hashing/golden/data/datagram.json b/tests/test_hashing/golden/data/v1/datagram-01.json similarity index 100% rename from tests/test_hashing/golden/data/datagram.json rename to tests/test_hashing/golden/data/v1/datagram-01.json diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/v1/pipeline-01.json similarity index 100% rename from tests/test_hashing/golden/data/pipeline.json rename to tests/test_hashing/golden/data/v1/pipeline-01.json diff --git a/tests/test_hashing/golden/data/schema.json b/tests/test_hashing/golden/data/v1/schema-01.json similarity index 96% rename from tests/test_hashing/golden/data/schema.json rename to tests/test_hashing/golden/data/v1/schema-01.json index b48a97a86..bf4ef4b81 100644 --- a/tests/test_hashing/golden/data/schema.json +++ b/tests/test_hashing/golden/data/v1/schema-01.json @@ -59,6 +59,14 @@ "polars_single_field.dataframe.schema_hash": "semantic_v1:3bea81b6438f30365fefa091067fca4b0b76149083d1a4cc6298a82e7ddde74c", "polars_single_field.series.arrow_type": "extension[large_binary]", "polars_single_field.series.schema_hash": "semantic_v1:796fcf1abfe4c5224e86110f47d1adbc639a95a500bad11798844c4809cd904e", + "semantic_schema.d_directory": "semantic_v1:0fbb5bba5b858fc4677b791e2f8008cf01324ffb5eab6dd0eedcacb1f72389c0", + "semantic_schema.f_file": "semantic_v1:d4654f89159e297dbc7800dbeb753b1487abbbc3f3277dd3b0305e4685e2ad88", + "semantic_schema.f_file_d_directory": "semantic_v1:77baccb8f76522e8e18ca0c74c9226d4082551502bd1b17489e2ead7fff033c5", + "semantic_schema.f_optional_file": "semantic_v1:9131e3d1da9847ee3a3cb700b8c8ce18cb03dd2ecf505d244847d2add81399cb", + "semantic_schema.p_path": "semantic_v1:609252265570863f631a437c6d6d95b080a710fd18264eebd81446c49f3cfb35", + "semantic_schema.x_int": "semantic_v1:e2eed73f4a554064d6c3acd3d3b39b81d2ea6164e9326c1412ed820cde586f2c", + "semantic_schema.x_int_f_file": "semantic_v1:87e6966aaf164468d7714a17cccc83addc1be9b5292c4a48bd57d3c081b5b2b9", + "semantic_schema.x_int_y_str": "semantic_v1:1bb91fe0f62d9a633d9602fdc8d1893e60d5819f6693175f4fc5661f90bf8d09", "single_field.bool.arrow_type": "bool", "single_field.bool.schema_hash": "semantic_v1:d65d43b0876cd839e9c53022a3e523e8236ca0df552bd3fb7a6e199e259e6839", "single_field.bytes.arrow_type": "large_binary", diff --git a/tests/test_hashing/golden/data/semantic.json b/tests/test_hashing/golden/data/v1/semantic-01.json similarity index 66% rename from tests/test_hashing/golden/data/semantic.json rename to tests/test_hashing/golden/data/v1/semantic-01.json index 66e2d16e6..6f30d1e20 100644 --- a/tests/test_hashing/golden/data/semantic.json +++ b/tests/test_hashing/golden/data/v1/semantic-01.json @@ -1,4 +1,25 @@ { + "annotated_function.builtin.info": "semantic_v1:62647b5236b91b669a128d661865cb351133b9508ddab1aea8e563be50530977", + "annotated_function.builtin.object": "semantic_v1:62647b5236b91b669a128d661865cb351133b9508ddab1aea8e563be50530977", + "annotated_function.file_list.info": "semantic_v1:5c625823b4fe0c6616667d420be2f347c029cde68a0659c51e4d5f119eec0f0a", + "annotated_function.file_list.object": "semantic_v1:5c625823b4fe0c6616667d420be2f347c029cde68a0659c51e4d5f119eec0f0a", + "annotated_function.file_param.info": "semantic_v1:5bf627e036e5826093d17f5ca3d4c86dea474d14965f0f367df869cc5b48fc39", + "annotated_function.file_param.object": "semantic_v1:5bf627e036e5826093d17f5ca3d4c86dea474d14965f0f367df869cc5b48fc39", + "annotated_function.file_return.info": "semantic_v1:53e59f17f3adb6a0746419028194a1293280e94390a772d98826788b643a0ecb", + "annotated_function.file_return.object": "semantic_v1:53e59f17f3adb6a0746419028194a1293280e94390a772d98826788b643a0ecb", + "annotated_function.mixed.info": "semantic_v1:5b1df0d4adb16c89738315a9260ac391dd3c8120e5fa0f51c99c827a7e3713c1", + "annotated_function.mixed.object": "semantic_v1:5b1df0d4adb16c89738315a9260ac391dd3c8120e5fa0f51c99c827a7e3713c1", + "annotated_function.none.info": "semantic_v1:64999822b66304279c0ac19a4e76f7d001fb65830440b91f3723b48f6f09bf5d", + "annotated_function.none.object": "semantic_v1:64999822b66304279c0ac19a4e76f7d001fb65830440b91f3723b48f6f09bf5d", + "annotated_function.optional_file.info": "semantic_v1:b3550eec874ee9c786e13fa0a8e38706064d1f548f14cb8ea6ae470b3cd7073f", + "annotated_function.optional_file.object": "semantic_v1:b3550eec874ee9c786e13fa0a8e38706064d1f548f14cb8ea6ae470b3cd7073f", + "annotation.dict_str_file": "semantic_v1:d2ec6ad67d11d40506d4f9235c4e076c3b230e3f75c23dd9f117402a66b2074f", + "annotation.directory": "semantic_v1:8050d1b8d85bf525f182125fe61f9da234d5dad67e43b69416100f743f6166a1", + "annotation.file": "semantic_v1:a2fd9d1b211cc95352b7787760040ec7da93c232e5c8c4005a24c36df08926fb", + "annotation.file_or_none": "semantic_v1:553ef731133e3935c9efccf48a511bee41af5717f2d5652a5029022ed20ab22c", + "annotation.int_or_str": "semantic_v1:838cabab55983dce341eaaee078ea5eda7f8a2a31ca6f8a0ab53f9ea9528b73a", + "annotation.list_file": "semantic_v1:df514aaa5ce466d71578b354cdf70c051d12b3501fb98b02de129bd21683c1a8", + "annotation.optional_file": "semantic_v1:553ef731133e3935c9efccf48a511bee41af5717f2d5652a5029022ed20ab22c", "function.stored_signature.ratio": "semantic_v1:7804cfbf8935f990b4d7941be5dae40aa9b2ff0bc94a3eb81d97a6db4b0dce77", "function.stored_signature.scaled": "semantic_v1:143251902c84387a1acb97e0076d00a3d17c0b4671c3a3449271750c8c38866e", "function.value.defaults": "semantic_v1:a55248ece1fc5e1a3c19860c390217a2325b843a0469982749706f38175d4366", @@ -21,6 +42,36 @@ "polars_value.nulls_and_temporal": "polars_dataframe_v1:fccfc33b3a7f171257b87ff20d260d2221085e48e3740d567681414e5ace589a", "polars_value.series": "polars_series_v1:9ff68c5024750115e606f8e370ceaa55bcb3305227dc6ad45111adaf5a620d88", "polars_value.simple": "polars_dataframe_v1:92002a1ced22c899f5662ceb4ed2e435ea2d0dddcf28b489ac3f8c8d5226aadf", + "sample_value.bytearray_abc": "semantic_v1:9117877bc5c6f08d682a7d9bb4e67b984d316921faf09bf224a5c9bedb3ebb20", + "sample_value.bytearray_hello_world": "semantic_v1:95b243c4c8b3e696c28c0ffc82ff70b87f505f487f57ed5e7595a70b21a6a6ff", + "sample_value.bytes_binary": "semantic_v1:cedf4873ac6b0aba6a4b8dc6574c5564ee4b971a0be7fd8ad76582d1377e9995", + "sample_value.bytes_hello": "semantic_v1:7375513f6d0b5271db377b03ad832f51966a263f2ceb70a747b17c076b273ede", + "sample_value.dict_keys_values": "semantic_v1:af63a0d00f3d3b744714e12bc4b6003b2395e296fe23c7a02c3352794f156cdb", + "sample_value.dict_nested": "semantic_v1:635b42bee92bc9e78396200dd770fffce472fc9b768d9ea1b73118367efaf084", + "sample_value.dict_one": "semantic_v1:015abd7f5cc57a2dd94b7590f04ad8084273905ee33ec5cebeae62276a97f862", + "sample_value.dict_other_order": "semantic_v1:d3626ac30a87e6f7a6428233b3c68299976865fa5508e4267c5415c76af7a772", + "sample_value.dict_points_labels": "semantic_v1:a553155144ad09460431dbbaacf7fe9a2910dd03ef969af8a9a529aae1c62c4f", + "sample_value.dict_users": "semantic_v1:32c13fff89d73e616960914df441b14c74671ec89d9589c2bcdba76f436b8126", + "sample_value.float_negative_e": "semantic_v1:43dc3d0d4b9e9bb2a2dfcf9696b0d49fe1f4fa10ac48586ef40e0b037f9ebcfe", + "sample_value.float_pi": "semantic_v1:c0740dd25c9de39b9c8d5ab452e8b69bcc0bf86f2a60ed7e527e79d0a3035852", + "sample_value.frozenset_empty": "semantic_v1:469289aca5c121cc9cec7cbde2df99827e5f310b646d4b7367760ea5f3a72bc7", + "sample_value.int_42": "semantic_v1:73475cb40a568e8da8a045ced110137e159f890ac4da883b6b17dc651b3a8049", + "sample_value.list_mixed": "semantic_v1:8554b19db94b8f9b64793d82a73098ea8af738886c738a2183682ca0ce08489a", + "sample_value.list_of_dicts": "semantic_v1:d72ea3c2223310171bdcb8e1787ef9f67f0e0c9a96616ad1686ad98b9d1c5e9d", + "sample_value.list_of_dicts_with_lists": "semantic_v1:d88e8543683b6b82bd7ce2225762d5a908c9229d122f8a52ca5c64169aec9a9b", + "sample_value.list_strs": "semantic_v1:fa1844c2988ad15ab7b49e0ece09684500fad94df916859fb9a43ff85f5bb477", + "sample_value.lists_and_dict": "semantic_v1:aa4dc10726bca2bd65f5c953962b432c3aee33b37fe3fbc05b1514307d440a92", + "sample_value.nested_dict_list": "semantic_v1:2252fc2a778c83e9c9308bd3ff747afd8125a902ff23282f4324d48424987c5d", + "sample_value.nested_dicts": "semantic_v1:c6d14482c460f03004d38484d71298a40eb8ec52e264851222c8bda2f4982541", + "sample_value.nested_lists": "semantic_v1:fdc65463c02cb1c8788344fba97041b67fce86c4726d4e10fe135af24b9c6801", + "sample_value.ordered_dict": "semantic_v1:e6a3385fb77c287a712e7f406a451727f0625041823ecf23bea7ef39b2e39805", + "sample_value.set_empty": "semantic_v1:469289aca5c121cc9cec7cbde2df99827e5f310b646d4b7367760ea5f3a72bc7", + "sample_value.set_strs": "semantic_v1:dc64a5a5142e9849961bf58ac4900f662d8d1fcf81261063f46ac5c475c0f4e7", + "sample_value.str_hello": "semantic_v1:5aa762ae383fbb727af3c7a36d4940a5b8c40a989452d2304fc958ff3f354e7a", + "sample_value.str_hello_world": "semantic_v1:cc82ebbcf8b60a5821d1c51c72cd79380ecea47de343ccb3b158938a2b3bf764", + "sample_value.str_special_chars": "semantic_v1:bbfa5afce2bf76f5520f0acafbb986c6da70dc507717450af24f91e8d01832c9", + "sample_value.str_unicode": "semantic_v1:2dd7db78b058c582953d493cc25b725d3cd7671c246c30440cebb5effc7c9e91", + "sample_value.tuple_empty": "semantic_v1:095502da3bf5c96b476e96c1304bbf052aec9482f7c46ed649e74d9d9bc172a7", "type.bool": "semantic_v1:cd8f971312b872f0e9a3ef40aa58d6bb537258e72a9a142e0cf712f0a35020e3", "type.bytes": "semantic_v1:f65cea8d1f29ca5b96d67caf1ced2f630b509e32574a7b78b9071958818a1c59", "type.dataclass": "semantic_v1:86baa1ef8dbf2518c6dcf7d188f6e20474248e9d8bc08d22b7fa0a4ed98bf8a8", diff --git a/tests/test_hashing/golden/data/torch.json b/tests/test_hashing/golden/data/v1/torch-01.json similarity index 100% rename from tests/test_hashing/golden/data/torch.json rename to tests/test_hashing/golden/data/v1/torch-01.json diff --git a/tests/test_hashing/golden/generate_golden_hashes.py b/tests/test_hashing/golden/generate_golden_hashes.py index fedbfb4f8..7d233e58c 100644 --- a/tests/test_hashing/golden/generate_golden_hashes.py +++ b/tests/test_hashing/golden/generate_golden_hashes.py @@ -1,11 +1,16 @@ -"""Regenerate the golden hash files checked by ``test_golden_hashes.py`` (ITL-626). +"""Add, check and release the golden hash files checked by ``test_golden_hashes.py``. Usage:: - uv run python tests/test_hashing/golden/generate_golden_hashes.py + uv run python tests/test_hashing/golden/generate_golden_hashes.py # add + uv run python tests/test_hashing/golden/generate_golden_hashes.py --check # report + uv run python tests/test_hashing/golden/generate_golden_hashes.py --release 0.3.0 + uv run python tests/test_hashing/golden/generate_golden_hashes.py --remove -writes ``tests/test_hashing/golden/data/.json`` for every group in -``cases.GROUPS``. +The golden values of the current data context live in +``tests/test_hashing/golden/data//-NN.json``, and the manifest of the data +context (``src/orcapod/contexts/data/.json``) lists every file, with the sha256 of +its bytes, in its ``golden.history`` (see ``store.py`` and ``README.md`` in this directory). READ THIS BEFORE REGENERATING ============================= @@ -15,15 +20,26 @@ by an earlier release is no longer found, is keyed differently, or is read under another column name. **A changed value is a breaking change.** -Regenerating is legitimate only when: - -1. You ADD a case or entry (existing values must come out unchanged; check the diff), or -2. A hash change is intentional, has a written migration assessment (which stored data it - strands and how users recover it), and the developer has approved it. Name the approving - decision and the migration plan in the PR description. - -Never regenerate to make a failing test pass without that approval. If a test fails and you -did not mean to change a hash, the code change is the bug. +The rules (developer, November 15, 2026, ITL-946): + +1. A golden file listed by a released revision never changes and is never removed. This + script never rewrites one. +2. Adding values is allowed in any release. ``--add`` (the default) writes the entries the + cases compute and no file pins yet: into the newest file of their group when no release + lists that file yet, otherwise into a new ``-NN.json`` with the next counter. It + records the file in the open revision of the history (appending one after a release). + Existing values are never rewritten, so check that the diff only adds. +3. A change to a released value has no exemption, not even a logical type's own identifier + change: it needs a new data context version, with a new manifest and a new golden folder, + shipped only in a minor (before 1.0) or major release. The old context's golden files and + code move to ``orcapod-sanctuary``. +4. Before a release, a value no release lists may change only with the developer's approval + and a written migration assessment: ``--remove `` takes the entry out of its + unreleased file and records that file again in the open revision (it refuses an entry a + release lists), then ``--add`` writes the new value. + +Never edit a golden file to make a failing test pass. If a test fails and you did not mean to +change a hash, the code change is the bug. Values must be identical on every supported Python version (3.11, 3.12, 3.13) and in every process, whatever ``PYTHONHASHSEED`` is. A value that differs by interpreter is a defect to @@ -34,6 +50,15 @@ columns, the canonical ``arrow.uuid`` and ``arrow.fixed_shape_tensor`` types, and fixed-size-list values are hashable and pinned. +Releasing +--------- + +``--release X.Y.Z`` is the golden step of a release pull request: it appends the release +revision (``added`` empty), pins its digest under ``releases`` in +``src/orcapod/contexts/released_manifests.json``, and, at the data context's first release, +sets ``released_in`` and pins the digest of the manifest without ``golden``. It computes no +hash. Try it on a copy first with ``--root`` (``store.copy_tree`` makes one). + Comparing releases ------------------ @@ -41,15 +66,17 @@ that checkout's environment and this checkout's test package on the path:: uv run --directory python /tests/test_hashing/golden/generate_golden_hashes.py \\ - --allow-errors --output-dir + --stdout --allow-errors > -then diff ```` against ``tests/test_hashing/golden/data``. ``--allow-errors`` -records ``ERROR: ...`` for a case the old API cannot express instead of stopping. +and compare ```` with the golden folder. ``--allow-errors`` records +``ERROR: ...`` for a case the old API cannot express instead of stopping. """ from __future__ import annotations import argparse +import datetime +import hashlib import json import pathlib import sys @@ -61,54 +88,293 @@ if str(_PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(_PROJECT_ROOT)) -from tests.test_hashing.golden import cases # noqa: E402 +from orcapod.contexts import get_default_context # noqa: E402 +from tests.test_hashing.golden import cases, store # noqa: E402 + + +def dump(values: dict[str, Any]) -> str: + """Serialize one golden file's entries the way the golden files are stored. + + Args: + values: Entries of one file. + + Returns: + Indented JSON with sorted keys and a trailing newline. + """ + return json.dumps(values, indent=2, sort_keys=True, ensure_ascii=False) + "\n" -DATA_DIR = pathlib.Path(__file__).resolve().parent / "data" +def _today() -> str: + return datetime.date.today().isoformat() -def golden_path(group: str, data_dir: pathlib.Path = DATA_DIR) -> pathlib.Path: - """Return the golden file path for ``group``. + +def _require_sound(root: pathlib.Path, version: str) -> None: + """Stop when the tree already breaks the golden rules; nothing is written then.""" + problems = store.tree_problems(root, version) + if problems: + raise SystemExit( + "the golden history does not match the files; nothing written:\n " + + "\n ".join(problems) + ) + + +def add(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: + """Write the computed entries that no golden file pins yet. Args: - group: One of ``cases.GROUPS``. - data_dir: Directory holding the golden files. + root: Project root. + version: Data context version. + groups: Groups to compute. Returns: - The path ``/.json``. + Names of the files written. """ - return data_dir / f"{group}.json" + _require_sound(root, version) + path = store.manifest_path(root, version) + manifest = store.read_json(path) + folder = store.golden_folder(root, manifest) + pinned = store.load_folder(folder) + owner = {entry: group for group, values in pinned.items() for entry in values} + files = store.group_files(folder) + history = manifest["golden"]["history"] + released = store.released_files(history) + # Compute and plan every group before writing anything, so a failure leaves the files + # and their history as they were. + planned: list[tuple[pathlib.Path, bytes, int]] = [] + for group in groups: + live = cases.compute_group(group) + clash = sorted(e for e in live if owner.get(e, group) != group) + if clash: + raise SystemExit(f"{group} entries pinned under another group: {clash}") + new = {e: v for e, v in live.items() if e not in owner} + if not new: + continue + # An entry name appears once in a folder: claim it before the next group plans. + owner.update(dict.fromkeys(new, group)) + existing = files.get(group, []) + latest = existing[-1] if existing else None + if latest is not None and latest.name not in released: + target = latest + values = {**store.read_json(latest), **new} + else: + number = 1 + if latest is not None: + match = store.FILE_NAME.match(latest.name) + assert match is not None + number = int(match["number"]) + 1 + if number > 99: + raise SystemExit(f"{group} has no free file counter") + target = folder / f"{group}-{number:02d}.json" + values = new + data = dump(values).encode("utf-8") + history = store.with_file( + history, target.name, hashlib.sha256(data).hexdigest(), _today() + ) + planned.append((target, data, len(new))) + if not planned: + print("nothing to add: every computed entry is pinned") + return [] + for target, data, count in planned: + _ = target.write_bytes(data) + print(f"added {count} entries to {target}") + manifest["golden"]["history"] = history + store.write_json(path, manifest) + return [target.name for target, _, _ in planned] -def dump(values: dict[str, Any]) -> str: - """Serialize one group's entries the way the golden files are stored. +def check(root: pathlib.Path, version: str, groups: list[str]) -> list[str]: + """Compare the computed values with the golden folder; write nothing. Args: - values: Entries of one group. + root: Project root. + version: Data context version. + groups: Groups to compute. Returns: - Indented JSON with sorted keys and a trailing newline. + One sentence per difference or rule problem. """ - return json.dumps(values, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + problems = store.tree_problems(root, version) + manifest = store.read_json(store.manifest_path(root, version)) + try: + pinned = store.load_folder(store.golden_folder(root, manifest)) + except store.GoldenLayoutError as exc: + return [*problems, str(exc)] + for group in groups: + live = cases.compute_group(group) + pins = pinned.get(group, {}) + skipped = cases.unavailable_entries(group) + for entry in sorted(set(live) | set(pins)): + if entry in skipped: + continue + if entry not in pins: + problems.append(f"{group}:{entry} is computed but not pinned") + elif entry not in live: + problems.append(f"{group}:{entry} is pinned but no longer produced") + elif live[entry] != pins[entry]: + problems.append( + f"{group}:{entry} moved: pinned {pins[entry]!r}, live {live[entry]!r}" + ) + return problems + + +def remove(root: pathlib.Path, version: str, entries: list[str]) -> list[str]: + """Remove entries from golden files no release lists, recording the files again. + + The first step of an approved change to an unreleased value: remove the entry, then + ``--add`` writes its new value. An entry in a file a release lists is refused, and + nothing is written unless every entry can be removed. + + Args: + root: Project root. + version: Data context version. + entries: Entry names to remove. + + Returns: + Names of the files rewritten. + """ + _require_sound(root, version) + path = store.manifest_path(root, version) + manifest = store.read_json(path) + folder = store.golden_folder(root, manifest) + history = manifest["golden"]["history"] + released = store.released_files(history) + where = { + entry: file + for files in store.group_files(folder).values() + for file in files + for entry in store.read_json(file) + } + unknown = sorted(e for e in entries if e not in where) + if unknown: + raise SystemExit(f"no golden file pins {unknown}; nothing written") + frozen = sorted(e for e in entries if where[e].name in released) + if frozen: + raise SystemExit( + f"{frozen} are in files a release lists; a released value never changes " + "(a change needs a new data context version); nothing written" + ) + planned: dict[pathlib.Path, dict[str, Any]] = {} + for entry in dict.fromkeys(entries): + file = where[entry] + values = planned.setdefault(file, store.read_json(file)) + _ = values.pop(entry) + writes: list[tuple[pathlib.Path, bytes]] = [] + for file, values in planned.items(): + data = dump(values).encode("utf-8") + history = store.with_file( + history, file.name, hashlib.sha256(data).hexdigest(), _today() + ) + writes.append((file, data)) + for file, data in writes: + _ = file.write_bytes(data) + print(f"removed entries from {file}") + manifest["golden"]["history"] = history + store.write_json(path, manifest) + return [file.name for file, _ in writes] + + +def release(root: pathlib.Path, version: str, release_text: str) -> str: + """Record a library release in the golden history and the pins. + + Args: + root: Project root. + version: Data context version. + release_text: The library version, e.g. ``0.3.0``. + + Returns: + The digest of the release revision. + """ + try: + name = store.normalize_release(release_text) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + _require_sound(root, version) + path = store.manifest_path(root, version) + manifest = store.read_json(path) + pins = store.read_json(store.pins_path(root)) + pin = pins[version] + if name in pin["releases"]: + raise SystemExit(f"{name} is already released") + history = manifest["golden"]["history"] + if not history: + raise SystemExit("the golden history lists no file") + history = store.with_release(history, name, _today()) + manifest["golden"]["history"] = history + if manifest["released_in"] is None: + manifest["released_in"] = name + fixed = store.fixed_manifest_digest(manifest) + pins[version] = { + "manifest": fixed, + "releases": {**pin["releases"], name: history[-1]["digest"]}, + } + store.write_json(path, manifest) + store.write_json(store.pins_path(root), pins) + print(f"released {name}: revision {history[-1]['digest']}, manifest {fixed}") + return history[-1]["digest"] def main(argv: list[str] | None = None) -> None: - """Compute the golden groups and write (or print) them. + """Add, check or release the golden files, or print the computed values. Args: argv: Command-line arguments; ``None`` reads ``sys.argv``. """ parser = argparse.ArgumentParser( - description="Regenerate the golden hash files (ITL-626)." + description="Add, check or release the golden hash files (ITL-626, ITL-946)." + ) + mode = parser.add_mutually_exclusive_group() + _ = mode.add_argument( + "--add", + action="store_true", + help="write computed entries no golden file pins yet (the default)", + ) + _ = mode.add_argument( + "--check", + action="store_true", + help="report moved, unpinned and missing entries; write nothing", + ) + _ = mode.add_argument( + "--release", + metavar="X.Y.Z", + help="append the release revision and pin it; computes no hash", + ) + _ = mode.add_argument( + "--remove", + metavar="ENTRY", + action="append", + help=( + "remove an entry from a file no release lists (repeatable); with the " + "developer's approval, the first step of changing an unreleased value" + ), + ) + _ = mode.add_argument( + "--stdout", + action="store_true", + help="print all computed groups as one JSON object; write nothing", ) - _ = parser.add_argument("--output-dir", type=pathlib.Path, default=DATA_DIR) _ = parser.add_argument("--group", choices=cases.GROUPS, action="append") - _ = parser.add_argument("--allow-errors", action="store_true") _ = parser.add_argument( - "--stdout", + "--allow-errors", action="store_true", - help="print all groups as one JSON object instead of writing files", + help="with --stdout: record a failing case as ERROR instead of stopping", + ) + _ = parser.add_argument( + "--root", + type=pathlib.Path, + default=_PROJECT_ROOT, + help="project root holding the manifest, the pins and the golden folder", ) args = parser.parse_args(argv) + if args.allow_errors and not args.stdout: + parser.error("--allow-errors is only for --stdout: errors are never pinned") + + version = get_default_context().version + if args.remove: + _ = remove(args.root, version, args.remove) + return + if args.release is not None: + _ = release(args.root, version, args.release) + return unavailable = [g for g in args.group or [] if not cases.group_available(g)] if unavailable: @@ -121,40 +387,22 @@ def main(argv: list[str] | None = None) -> None: g for g in cases.GROUPS if g not in groups and not cases.group_available(g) ] if skipped and not args.stdout: - # Never rewrite an optional group's file without its dependency (ITL-871). - print( - f"skipped {skipped}: optional dependency not installed; files left as they are" - ) - computed = { - g: cases.compute_group(g, allow_errors=args.allow_errors) for g in groups - } + print(f"skipped {skipped}: optional dependency not installed") if args.stdout: - sys.stdout.write(dump(computed)) + computed = { + g: cases.compute_group(g, allow_errors=args.allow_errors) for g in groups + } + _ = sys.stdout.write(dump(computed)) return - args.output_dir.mkdir(parents=True, exist_ok=True) - for group, values in computed.items(): - path = golden_path(group, args.output_dir) - missing = cases.unavailable_entries(group) - if missing: - # Keep the pins this environment cannot compute (an older optional - # dependency), from the target file or else the repository's, so a - # golden file is never written without them. - source = path if path.exists() else golden_path(group) - pinned = ( - json.loads(source.read_text(encoding="utf-8")) - if source.exists() - else {} + if args.check: + problems = check(args.root, version, groups) + if problems: + raise SystemExit( + f"{len(problems)} golden problem(s):\n " + "\n ".join(problems) ) - lost = sorted(e for e in missing if e not in pinned) - if lost: - raise SystemExit( - f"cannot write {path}: this environment cannot compute {lost} " - "and no existing golden file pins them; install the newer " - "optional dependency" - ) - values = {**values, **{e: pinned[e] for e in missing}} - path.write_text(dump(values), encoding="utf-8") - print(f"wrote {len(values)} entries to {path}") + print(f"golden values of {', '.join(groups)} match") + return + _ = add(args.root, version, groups) if __name__ == "__main__": diff --git a/tests/test_hashing/golden/marks.py b/tests/test_hashing/golden/marks.py new file mode 100644 index 000000000..253a86b47 --- /dev/null +++ b/tests/test_hashing/golden/marks.py @@ -0,0 +1,84 @@ +"""The one skip a golden entry may carry, and the check that refuses any other (ITL-946). + +A golden entry is never skipped or expected to fail, except by the optional-package rule +(ITL-871): a group whose optional package is not installed is skipped. ``conftest.py`` in +this directory applies the check when tests are collected, outside the golden test module, +so a mark on that module cannot switch it off. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +import pytest + +from tests.test_hashing.golden import cases + +SKIP_OR_XFAIL = frozenset({"skip", "skipif", "xfail"}) +"""The marks that skip a test or expect it to fail.""" + +GOLDEN_ENTRY_TEST = "test_golden_entry" +"""The test function, in ``test_golden_hashes.py``, that checks one golden entry.""" + +GOLDEN_TEST_MODULE = Path(__file__).resolve().with_name("test_golden_hashes.py") +"""The module holding ``GOLDEN_ENTRY_TEST``; items of any other module are ignored.""" + + +def needs(group: str) -> Any: + """A skip mark for a group whose optional dependency is not installed (ITL-871).""" + module = cases.OPTIONAL_GROUPS.get(group) + return pytest.mark.skipif( + not cases.group_available(group), + reason=f"golden group {group!r} needs the optional package {module!r}", + ) + + +def entry_marks(group: str) -> list[Any]: + """The only marks a golden entry may carry: the optional-package skip (ITL-871).""" + return [needs(group)] if group in cases.OPTIONAL_GROUPS else [] + + +def disallowed_marks(group: str, marks: Iterable[Any]) -> list[str]: + """The skip and xfail marks on a golden entry of ``group`` that the rules forbid. + + The one allowed mark is the optional-package skip of an optional group, exactly as + ``needs`` makes it. + + Args: + group: The entry's group. + marks: The entry's marks, as ``pytest.Mark`` or ``pytest.MarkDecorator`` objects. + + Returns: + A description of each forbidden mark. + """ + allowed = [mark.mark for mark in entry_marks(group)] + found = [getattr(mark, "mark", mark) for mark in marks] + return [ + f"{mark.name}{mark.args}{mark.kwargs}" + for mark in found + if mark.name in SKIP_OR_XFAIL and mark not in allowed + ] + + +def golden_entry_violations(items: Iterable[Any]) -> dict[str, list[str]]: + """The collected golden entry tests that carry a forbidden skip or xfail. + + Args: + items: Collected test items (any others are ignored). + + Returns: + Node id to the forbidden marks it carries; empty when there are none. + """ + found: dict[str, list[str]] = {} + for item in items: + if getattr(item, "originalname", None) != GOLDEN_ENTRY_TEST: + continue + if Path(str(getattr(item, "path", ""))).resolve() != GOLDEN_TEST_MODULE: + continue + group = str(item.callspec.params["group"]) + bad = disallowed_marks(group, item.iter_markers()) + if bad: + found[item.nodeid] = bad + return found diff --git a/tests/test_hashing/golden/store.py b/tests/test_hashing/golden/store.py new file mode 100644 index 000000000..c75921c81 --- /dev/null +++ b/tests/test_hashing/golden/store.py @@ -0,0 +1,654 @@ +"""The golden hash files of a data context and the history that pins them (ITL-946). + +The golden values of a data context version live in one folder, +``tests/test_hashing/golden/data//``, named by the ``golden.folder`` field of the +version's manifest (``src/orcapod/contexts/data/.json``). Each file is +``-NN.json``: the entries of one group of ``cases.GROUPS`` and a two-digit counter. +The unit of immutability is the file: a file listed by a released revision never changes, +and new values go into a new file. + +The manifest's ``golden.history`` is an append-only chain of revisions:: + + {"date": "YYYY-MM-DD", "release": "X.Y.Z" or null, + "added": {"": ""}, + "previous": , + "digest": } + +A release appends a revision with ``added`` empty and ``release`` set; its digest is pinned +in ``src/orcapod/contexts/released_manifests.json``. Revisions after the last release +revision may still be edited (their digests are recomputed); the ones up to it may not. + +This module holds the rules: reading the folder, computing digests, and listing what in a +tree breaks the rules. ``generate_golden_hashes.py`` writes the files and the history, +``test_golden_hashes.py`` and ``tests/test_contexts/test_data_context_v1.py`` check them, +and ``scripts/check_golden_tags.py`` compares them with release tags. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import shutil +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +from packaging.version import InvalidVersion, Version + +from tests.test_hashing.golden.cases import GROUPS + +PROJECT_ROOT = Path(__file__).resolve().parents[3] + +CONTEXTS_DIR = Path("src/orcapod/contexts") +"""The data context directory, relative to a project root.""" + +PINS_FILE = CONTEXTS_DIR / "released_manifests.json" +"""The pins of released manifests, relative to a project root.""" + +GOLDEN_ROOT = Path("tests/test_hashing/golden/data") +"""The directory holding one golden folder per data context version, relative to a root.""" + +FILE_NAME = re.compile(r"^(?P[a-z][a-z0-9_]*)-(?P[0-9]{2})\.json$") +"""A golden file name: ``-NN.json``.""" + +REVISION_FIELDS = ("date", "release", "added", "previous") +"""The fields of a revision its digest covers, besides the digest itself.""" + +_DATE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$") + + +class GoldenLayoutError(ValueError): + """The golden folder breaks the file rules (an unknown file name, a duplicate entry).""" + + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- + + +def manifest_path(root: Path, version: str) -> Path: + """The manifest file of a data context version. + + Args: + root: Project root (the repository, or a copy of its golden tree). + version: Data context version, e.g. ``v1``. + + Returns: + ``/src/orcapod/contexts/data/.json``. + """ + return root / CONTEXTS_DIR / "data" / f"{version}.json" + + +def pins_path(root: Path) -> Path: + """The ``released_manifests.json`` file under ``root``.""" + return root / PINS_FILE + + +def golden_folder(root: Path, manifest: Mapping[str, Any]) -> Path: + """The golden folder a manifest names. + + Args: + root: Project root. + manifest: Parsed manifest. + + Returns: + ``/tests/test_hashing/golden/data/``. + """ + return root / GOLDEN_ROOT / manifest["golden"]["folder"] + + +def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build a JSON object, refusing a key that appears twice (JSON keeps the last).""" + out: dict[str, Any] = {} + for key, value in pairs: + if key in out: + raise GoldenLayoutError(f"the key {key!r} appears twice in one JSON object") + out[key] = value + return out + + +def read_json(path: Path) -> Any: + """Parse a JSON file, refusing an object that repeats a key. + + Raises: + GoldenLayoutError: An object in the file repeats a key; the message names the file. + """ + try: + return json.loads( + path.read_text(encoding="utf-8"), object_pairs_hook=_unique_keys + ) + except GoldenLayoutError as exc: + raise GoldenLayoutError(f"{path}: {exc}") from exc + + +def write_json(path: Path, value: Any) -> None: + """Write a manifest or pin file the way the repository stores it. + + Four-space indentation, keys in their given order, a trailing newline. + """ + _ = path.write_bytes( + (json.dumps(value, indent=4, ensure_ascii=False) + "\n").encode("utf-8") + ) + + +def copy_tree(source_root: Path, target_root: Path, version: str) -> None: + """Copy what the golden rules cover to another root, for a trial run. + + Copies the version's manifest, ``released_manifests.json`` and the version's golden + folder, keeping their paths relative to the root. ``generate_golden_hashes.py --root`` + can then add or release on the copy without touching the repository. + + Args: + source_root: Project root to copy from. + target_root: Root to copy to; created if missing. + version: Data context version whose files are copied. + """ + manifest = read_json(manifest_path(source_root, version)) + for path in (manifest_path(source_root, version), pins_path(source_root)): + target = target_root / path.relative_to(source_root) + target.parent.mkdir(parents=True, exist_ok=True) + _ = shutil.copyfile(path, target) + folder = golden_folder(source_root, manifest) + _ = shutil.copytree(folder, target_root / folder.relative_to(source_root)) + + +def reopen_tree(root: Path, version: str, date: str) -> None: + """Give a copied tree a fixed open starting point, whatever the repository released. + + The history becomes one open revision listing every file of the golden folder with its + sha256; ``released_in`` is cleared and the pin becomes ``{"manifest": null, + "releases": {}}``. The golden files are not touched. Tests that add or release on a + copy start from this state, so they keep working after the repository's own releases. + + Args: + root: Root of a copy made by ``copy_tree``; never the repository. + version: Data context version. + date: Date of the open revision, ``YYYY-MM-DD``. + """ + path = manifest_path(root, version) + manifest = read_json(path) + folder = golden_folder(root, manifest) + revision: dict[str, Any] = { + "date": date, + "release": None, + "added": {p.name: file_digest(p) for p in sorted(folder.iterdir())}, + "previous": None, + } + revision["digest"] = revision_digest(revision) + manifest["golden"]["history"] = [revision] + manifest["released_in"] = None + write_json(path, manifest) + pins = read_json(pins_path(root)) + pins[version] = {"manifest": None, "releases": {}} + write_json(pins_path(root), pins) + + +# --------------------------------------------------------------------------- +# Digests +# --------------------------------------------------------------------------- + + +def canonical_json(value: Any) -> bytes: + """The canonical JSON bytes of a value: sorted keys, no spaces.""" + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def revision_digest(revision: Mapping[str, Any]) -> str: + """The digest of a revision: sha256 of the canonical JSON of its four fields. + + Args: + revision: A revision; only ``date``, ``release``, ``added`` and ``previous`` are + read. + + Returns: + The hex digest. + """ + body = {name: revision[name] for name in REVISION_FIELDS} + return hashlib.sha256(canonical_json(body)).hexdigest() + + +def fixed_manifest_digest(manifest: Mapping[str, Any]) -> str: + """The digest pinned for a released manifest: everything but ``golden``. + + Args: + manifest: Parsed manifest. + + Returns: + sha256 of the canonical JSON of the manifest without its ``golden`` field. + """ + fixed = {name: value for name, value in manifest.items() if name != "golden"} + return hashlib.sha256(canonical_json(fixed)).hexdigest() + + +def file_digest(path: Path) -> str: + """sha256 of a file's bytes.""" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +# --------------------------------------------------------------------------- +# The folder +# --------------------------------------------------------------------------- + + +def group_files(folder: Path) -> dict[str, list[Path]]: + """The golden files of a folder by group, each group's files in counter order. + + Args: + folder: A golden folder. + + Returns: + Group name to its files. + + Raises: + GoldenLayoutError: The folder holds something that is not a ``-NN.json`` + file. + """ + files: dict[str, list[Path]] = {} + for path in sorted(folder.iterdir()): + match = FILE_NAME.match(path.name) + if match is None or not path.is_file(): + raise GoldenLayoutError( + f"{path} is not a golden file: a golden folder holds only files named " + "-NN.json" + ) + files.setdefault(match["group"], []).append(path) + return files + + +def next_file_name(name: str) -> str: + """The name of a group's next golden file, e.g. ``schema-02.json`` after ``schema-01.json``. + + Args: + name: A golden file name, ``-NN.json``. + + Returns: + The same group with the counter one higher. + + Raises: + GoldenLayoutError: ``name`` is not a golden file name. + """ + match = FILE_NAME.match(name) + if match is None: + raise GoldenLayoutError(f"{name} is not a golden file name") + return f"{match['group']}-{int(match['number']) + 1:02d}.json" + + +def load_folder(folder: Path) -> dict[str, dict[str, Any]]: + """Read every golden file of a folder. + + Args: + folder: A golden folder. + + Returns: + Group name to its entries, merged over the group's files. + + Raises: + GoldenLayoutError: A file name is not ``-NN.json``, a file is not a JSON + object, or an entry name appears in two files of the folder. + """ + values: dict[str, dict[str, Any]] = {} + where: dict[str, str] = {} + for group, paths in group_files(folder).items(): + merged = values.setdefault(group, {}) + for path in paths: + entries = read_json(path) + if not isinstance(entries, dict): + raise GoldenLayoutError(f"{path} does not hold a JSON object") + for entry, value in entries.items(): + if entry in where: + raise GoldenLayoutError( + f"golden entry {entry!r} is pinned twice, in {where[entry]} and " + f"{path.name}; an entry name appears once in a folder" + ) + where[entry] = path.name + merged[entry] = value + return values + + +# --------------------------------------------------------------------------- +# The history +# --------------------------------------------------------------------------- + + +def last_release_index(history: Sequence[Mapping[str, Any]]) -> int | None: + """Index of the last release revision, or ``None`` when nothing was released.""" + releases = [i for i, revision in enumerate(history) if revision["release"]] + return releases[-1] if releases else None + + +def released_files(history: Sequence[Mapping[str, Any]]) -> set[str]: + """Names of the files listed by revisions up to the last release revision.""" + end = last_release_index(history) + if end is None: + return set() + return {name for revision in history[: end + 1] for name in revision["added"]} + + +def rechain(history: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Recompute ``previous`` and ``digest`` of the revisions after the last release. + + The revisions up to the last release revision are returned unchanged. + + Args: + history: Revisions, oldest first. + + Returns: + A new list of revisions. + """ + end = last_release_index(history) + start = 0 if end is None else end + 1 + out = [dict(revision) for revision in history] + for i in range(start, len(out)): + out[i]["previous"] = out[i - 1]["digest"] if i else None + out[i]["digest"] = revision_digest(out[i]) + return out + + +def with_file( + history: Sequence[Mapping[str, Any]], name: str, digest: str, date: str +) -> list[dict[str, Any]]: + """Record a new or rewritten golden file in the open revision. + + The open revision is the last one when it is not a release; otherwise a new revision + is appended. A file listed by an open revision has its digest updated in place. + + Args: + history: Revisions, oldest first. + name: File name, ``-NN.json``. + digest: sha256 of the file's bytes. + date: The date of the change, ``YYYY-MM-DD``; it becomes the revision's date. + + Returns: + The new history, rechained. + + Raises: + ValueError: ``name`` is listed by a released revision. + """ + if name in released_files(history): + raise ValueError(f"{name} is listed by a released revision and never changes") + out = [dict(revision) for revision in history] + owner = next((r for r in out if name in r["added"]), None) + if owner is None: + if not out or out[-1]["release"]: + out.append({"date": date, "release": None, "added": {}, "previous": None}) + owner = out[-1] + owner["added"] = dict(sorted({**owner["added"], name: digest}.items())) + owner["date"] = date + return rechain(out) + + +def with_release( + history: Sequence[Mapping[str, Any]], release: str, date: str +) -> list[dict[str, Any]]: + """Append the release revision of a library version. + + Args: + history: Revisions, oldest first. + release: The library version, normalized. + date: The release date, ``YYYY-MM-DD``. + + Returns: + The new history. + """ + out = [dict(revision) for revision in history] + revision: dict[str, Any] = { + "date": date, + "release": release, + "added": {}, + "previous": out[-1]["digest"] if out else None, + } + revision["digest"] = revision_digest(revision) + return [*out, revision] + + +# --------------------------------------------------------------------------- +# The rules +# --------------------------------------------------------------------------- + + +def history_problems(history: Any) -> list[str]: + """What breaks the shape or the chain of a history. + + Every revision has exactly the fields ``date``, ``release``, ``added``, ``previous`` + and ``digest``; a release revision adds no file and any other revision adds at least + one; ``previous`` is the digest of the revision before (``null`` for the first); the + digest recomputes; and no library version is released twice. + + Args: + history: The ``golden.history`` value of a manifest. + + Returns: + One sentence per problem; empty when the history is sound. + """ + if not isinstance(history, list): + return ["golden.history is not a list"] + problems: list[str] = [] + expected_fields = {*REVISION_FIELDS, "digest"} + previous: str | None = None + seen_releases: set[str] = set() + for i, revision in enumerate(history): + label = f"revision {i}" + if not isinstance(revision, dict) or set(revision) != expected_fields: + problems.append( + f"{label} does not have exactly the fields {sorted(expected_fields)}" + ) + previous = None + continue + if not isinstance(revision["date"], str) or not _DATE.match(revision["date"]): + problems.append(f"{label} has no YYYY-MM-DD date") + added = revision["added"] + if not isinstance(added, dict) or not all( + isinstance(name, str) and isinstance(digest, str) + for name, digest in added.items() + ): + problems.append(f"{label}: added is not a mapping of file names to digests") + added = {} + release = revision["release"] + if release is not None: + if not isinstance(release, str) or not release: + problems.append(f"{label}: release is neither null nor a version") + elif release in seen_releases: + problems.append(f"{label}: release {release} appears twice") + else: + seen_releases.add(release) + if added: + problems.append(f"{label} is a release revision and adds files") + elif not added: + problems.append(f"{label} adds no file and is not a release revision") + if revision["previous"] != previous: + problems.append( + f"{label}: previous is not the digest of the revision before" + ) + if revision["digest"] != revision_digest(revision): + problems.append(f"{label}: digest does not recompute") + previous = revision["digest"] + return problems + + +def folder_problems(history: Sequence[Mapping[str, Any]], folder: Path) -> list[str]: + """What breaks the agreement between a history and its golden folder. + + Every listed file exists and has the listed sha256; every file in the folder is listed + by exactly one revision, is named ``-NN.json`` for a group of ``cases.GROUPS``, + and holds a JSON object; and no entry name appears in two files. + + Args: + history: A history without ``history_problems``. + folder: The golden folder the manifest names. + + Returns: + One sentence per problem. + """ + problems: list[str] = [] + listed: dict[str, int] = {} + end = last_release_index(history) + for i, revision in enumerate(history): + for name, digest in revision["added"].items(): + if name in listed: + problems.append(f"{name} is listed by revisions {listed[name]} and {i}") + continue + listed[name] = i + match = FILE_NAME.match(name) + if match is None or match["group"] not in GROUPS: + problems.append( + f"{name}, listed by revision {i}, is not a golden file name " + "(-NN.json of a known group, in the golden folder)" + ) + continue + path = folder / name + if not path.is_file(): + problems.append(f"{name}, listed by revision {i}, is missing") + elif file_digest(path) != digest: + problems.append( + f"{name} does not have the sha256 revision {i} lists: " + + ( + "a released golden file never changes" + if end is not None and i <= end + else "restore the file (git checkout); a value change no release " + "lists needs approval, then generate_golden_hashes.py --remove " + "and --add on the restored tree" + ) + ) + if not folder.is_dir(): + return [*problems, f"the golden folder {folder} does not exist"] + for path in sorted(folder.iterdir()): + if path.name not in listed: + problems.append(f"{path.name} is in {folder} but no revision lists it") + match = FILE_NAME.match(path.name) + if match is not None and match["group"] not in GROUPS: + problems.append( + f"{path.name} belongs to no known group (cases.GROUPS: {list(GROUPS)})" + ) + try: + _ = load_folder(folder) + except GoldenLayoutError as exc: + problems.append(str(exc)) + return problems + + +def pin_problems( + manifest: Mapping[str, Any], pin: Any, history: Sequence[Mapping[str, Any]] +) -> list[str]: + """What breaks the agreement between a manifest, its history and its pin. + + The pin is ``{"manifest": , "releases": {: }}``. The + pinned releases are exactly the release revisions, each with its digest; the manifest + pin is set exactly when ``released_in`` is, and then equals the digest of the manifest + without ``golden``; ``released_in`` is the first release; and a manifest with releases + is pinned. + + Args: + manifest: Parsed manifest. + pin: The version's value in ``released_manifests.json``. + history: The manifest's history, without ``history_problems``. + + Returns: + One sentence per problem. + """ + if ( + not isinstance(pin, dict) + or set(pin) != {"manifest", "releases"} + or not isinstance(pin["releases"], dict) + ): + return ['the pin is not {"manifest": ..., "releases": {...}}'] + problems: list[str] = [] + revisions = {r["release"]: r["digest"] for r in history if r["release"]} + for release, digest in pin["releases"].items(): + if release not in revisions: + problems.append(f"pinned release {release} is not a release revision") + elif revisions[release] != digest: + problems.append( + f"pinned release {release} is not the digest of its release revision" + ) + for release in revisions: + if release not in pin["releases"]: + problems.append(f"release revision {release} is not pinned") + released_in = manifest.get("released_in") + if (pin["manifest"] is None) != (released_in is None): + problems.append("released_in is set exactly when the manifest is pinned") + if (pin["manifest"] is None) != (not revisions): + problems.append("the manifest is pinned exactly when it has a release") + if pin["manifest"] is not None and pin["manifest"] != fixed_manifest_digest( + manifest + ): + problems.append( + "the manifest without golden differs from its pin: a released manifest is " + "frozen; a hashing change needs a new data context version" + ) + if released_in is not None and revisions and released_in != next(iter(revisions)): + problems.append("released_in is not the first release revision") + return problems + + +def final_release_problems(pin: Mapping[str, Any], package_version: str) -> list[str]: + """What a final release of the library lacks in its data context's pin. + + A final release (not a development, pre-release or local version) has its own + ``releases`` line and a pinned manifest. + + Args: + pin: The current data context's value in ``released_manifests.json``. + package_version: The library version, e.g. ``orcapod.__version__``. + + Returns: + One sentence per problem; empty for a version that is not a final release. + """ + version = Version(package_version) + if version.is_prerelease or version.is_devrelease or version.local is not None: + return [] + problems: list[str] = [] + if pin["manifest"] is None: + problems.append(f"release {version} ships an open manifest") + if str(version) not in pin["releases"]: + problems.append( + f"release {version} has no releases line: run " + f"generate_golden_hashes.py --release {version}" + ) + return problems + + +def normalize_release(text: str) -> str: + """The normalized public version of a release, e.g. ``0.3.0``. + + Args: + text: A version, optionally with a leading ``v``. + + Returns: + The normalized version. + + Raises: + ValueError: ``text`` is not a version, or is a local or development version. + """ + try: + version = Version(text.removeprefix("v")) + except InvalidVersion as exc: + raise ValueError(f"{text!r} is not a version") from exc + if version.local is not None or version.is_devrelease: + raise ValueError(f"{text!r} is a local or development version, never released") + return str(version) + + +def tree_problems(root: Path, version: str) -> list[str]: + """Every problem of a tree's golden history, folder and pin for one version. + + Args: + root: Project root. + version: Data context version. + + Returns: + One sentence per problem. + """ + manifest = read_json(manifest_path(root, version)) + golden = manifest.get("golden") + if not isinstance(golden, dict) or set(golden) != {"folder", "history"}: + return ['the manifest golden field is not {"folder": ..., "history": [...]}'] + history = golden["history"] + problems = history_problems(history) + if problems: + return problems + problems = folder_problems(history, golden_folder(root, manifest)) + pins = read_json(pins_path(root)) + if version not in pins: + return [*problems, f"{version} is not listed in released_manifests.json"] + return [*problems, *pin_problems(manifest, pins[version], history)] diff --git a/tests/test_hashing/golden/test_golden_hashes.py b/tests/test_hashing/golden/test_golden_hashes.py index 7c5f4acbd..54df51d2b 100644 --- a/tests/test_hashing/golden/test_golden_hashes.py +++ b/tests/test_hashing/golden/test_golden_hashes.py @@ -1,12 +1,18 @@ -"""Golden hash regression tests (ITL-626). +"""Golden hash regression tests (ITL-626, ITL-946). Every hash Orcapod stores, keys data on, or embeds in a column name or a database table path -is pinned in ``data/.json``. These tests recompute each value and compare it with the -pinned one, one test per entry, so a failure names exactly what moved. +is pinned in the golden folder of the current data context, +``data//-NN.json`` (the folder is named by the manifest's ``golden.folder``). +These tests recompute each value and compare it with the pinned one, one test per entry, so +a failure names exactly what moved. A failing entry is a BREAKING change: stored data keyed on the old value is stranded. Do not -regenerate the golden files to make a failure go away; read the header of -``generate_golden_hashes.py`` first. +edit the golden files to make a failure go away; read the header of +``generate_golden_hashes.py`` and ``README.md`` first. + +A golden entry is never skipped or expected to fail, except for the optional-package rule +(ITL-871): a group whose optional package is not installed, and an entry the installed +version of that package cannot compute. """ from __future__ import annotations @@ -25,40 +31,42 @@ from orcapod.contexts import get_default_context from orcapod.types import Schema -from tests.test_hashing.golden import cases -from tests.test_hashing.golden.generate_golden_hashes import golden_path +from tests.test_hashing.golden import cases, store +from tests.test_hashing.golden.marks import ( + disallowed_marks, + entry_marks, + golden_entry_violations, + needs, +) GENERATOR = Path(__file__).with_name("generate_golden_hashes.py") - -def _load_golden() -> dict[str, dict[str, Any]]: - golden: dict[str, dict[str, Any]] = {} - for group in cases.GROUPS: - path = golden_path(group) - golden[group] = ( - json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} - ) - return golden +MANIFEST = store.read_json( + store.manifest_path(store.PROJECT_ROOT, get_default_context().version) +) +FOLDER = store.golden_folder(store.PROJECT_ROOT, MANIFEST) -GOLDEN = _load_golden() +def _load_golden() -> tuple[dict[str, dict[str, Any]], str | None]: + """The pinned entries of every group, and the error that stopped reading them.""" + try: + loaded = store.load_folder(FOLDER) + except (store.GoldenLayoutError, OSError) as exc: + loaded, error = {}, str(exc) + else: + error = None + return {group: loaded.get(group, {}) for group in {*cases.GROUPS, *loaded}}, error -def _needs(group: str) -> Any: - """A skip mark for a group whose optional dependency is not installed (ITL-871).""" - module = cases.OPTIONAL_GROUPS.get(group) - return pytest.mark.skipif( - not cases.group_available(group), - reason=f"golden group {group!r} needs the optional package {module!r}", - ) +GOLDEN, LOAD_ERROR = _load_golden() GOLDEN_ENTRIES = [ - pytest.param(group, entry, id=f"{group}:{entry}", marks=_needs(group)) + pytest.param(group, entry, id=f"{group}:{entry}", marks=entry_marks(group)) for group in cases.GROUPS for entry in GOLDEN[group] ] -GROUP_PARAMS = [pytest.param(group, marks=_needs(group)) for group in cases.GROUPS] +GROUP_PARAMS = [pytest.param(group, marks=needs(group)) for group in cases.GROUPS] def _moved_message(group: str, entry: str, expected: Any, actual: Any) -> str: @@ -78,16 +86,30 @@ def live() -> dict[str, dict[str, Any]]: return cases.compute_all() +def test_golden_folder_reads() -> None: + """The golden folder holds only ``-NN.json`` files, no entry pinned twice.""" + assert LOAD_ERROR is None, LOAD_ERROR + + +def test_golden_folder_names_only_known_groups() -> None: + """Every golden file belongs to a group of ``cases.GROUPS``.""" + unknown = sorted(set(GOLDEN) - set(cases.GROUPS)) + assert not unknown, f"golden files of unknown groups in {FOLDER}: {unknown}" + + @pytest.mark.parametrize("group", cases.GROUPS) -def test_golden_file_present(group: str) -> None: - assert golden_path(group).exists(), ( - f"missing golden file {golden_path(group)}; run generate_golden_hashes.py" +def test_golden_group_has_a_file(group: str) -> None: + assert GOLDEN[group], ( + f"no {group}-NN.json in {FOLDER}; run generate_golden_hashes.py" ) @pytest.mark.parametrize(("group", "entry"), GOLDEN_ENTRIES) def test_golden_entry(live: dict[str, dict[str, Any]], group: str, entry: str) -> None: if entry in cases.unavailable_entries(group): + # The optional-package rule only: an entry the installed version of an optional + # package cannot compute (ITL-871). + assert group in cases.OPTIONAL_GROUPS, f"{group}:{entry} may not be skipped" pytest.skip(f"{group}:{entry} needs a newer version of an optional dependency") actual = live[group].get(entry, "") expected = GOLDEN[group][entry] @@ -99,8 +121,113 @@ def test_no_unpinned_entries(live: dict[str, dict[str, Any]], group: str) -> Non unpinned = sorted(set(live[group]) - set(GOLDEN[group])) assert not unpinned, ( f"{len(unpinned)} {group} entries are computed but not pinned (new cases need " - f"the generator run; check the diff adds entries only): {unpinned[:10]}" + f"generate_golden_hashes.py --add; check the diff adds entries only): " + f"{unpinned[:10]}" + ) + + +# --------------------------------------------------------------------------- +# No golden entry is skipped or expected to fail, but for a missing optional package +# --------------------------------------------------------------------------- + + +def test_golden_entry_params_carry_only_the_optional_package_skip() -> None: + """No parametrized golden entry is marked skip or xfail but by the ITL-871 rule.""" + found = { + param.id: disallowed_marks(str(param.values[0]), param.marks) + for param in GOLDEN_ENTRIES + } + assert not {k: v for k, v in found.items() if v} + + +def test_collected_golden_entries_carry_only_the_optional_package_skip( + request: pytest.FixtureRequest, +) -> None: + """The collected golden entry tests carry no other skip or xfail, whoever added it.""" + found = golden_entry_violations(request.session.items) + assert not found, f"golden entries marked skip or xfail: {found}" + + +class _FakeItem: + """A collected golden entry test with the given group and marks.""" + + originalname = "test_golden_entry" + + def __init__( + self, group: str, marks: list[Any], path: Path = Path(__file__) + ) -> None: + self.nodeid = f"test_golden_hashes.py::test_golden_entry[{group}:x]" + self.path = path + self.callspec = type( + "CallSpec", (), {"params": {"group": group, "entry": "x"}} + )() + self._marks = [getattr(mark, "mark", mark) for mark in marks] + + def iter_markers(self) -> list[Any]: + return self._marks + + +def test_the_collection_hook_refuses_a_skipped_golden_entry( + pytestconfig: pytest.Config, +) -> None: + """The conftest hook, outside this module, fails the collection on a forbidden mark.""" + from tests.test_hashing.golden import conftest + + skipped = _FakeItem("semantic", [pytest.mark.skip(reason="module-level skip")]) + with pytest.raises(pytest.UsageError, match="semantic:x"): + conftest.pytest_collection_modifyitems(pytestconfig, [skipped]) + + +def test_the_collection_hook_allows_the_optional_package_skip( + pytestconfig: pytest.Config, +) -> None: + """The torch group's optional-package skip passes the collection hook.""" + from tests.test_hashing.golden import conftest + + items = [_FakeItem("torch", [needs("torch")]), _FakeItem("semantic", [])] + conftest.pytest_collection_modifyitems(pytestconfig, items) + + +def test_the_collection_hook_ignores_other_modules(pytestconfig: pytest.Config) -> None: + """A test of the same name in another module is not a golden entry.""" + from tests.test_hashing.golden import conftest + + other = _FakeItem( + "semantic", + [pytest.mark.skip(reason="x")], + path=Path(__file__).with_name("x.py"), ) + conftest.pytest_collection_modifyitems(pytestconfig, [other]) + + +def test_golden_test_module_marks_nothing() -> None: + """Neither the module nor the entry test function carries a mark of its own.""" + assert "pytestmark" not in globals() + own = [ + mark.name + for mark in getattr(test_golden_entry, "pytestmark", []) + if mark.name != "parametrize" + ] + assert not own + + +@pytest.mark.parametrize( + ("group", "mark"), + [ + ("semantic", pytest.mark.xfail(reason="moved")), + ("schema", pytest.mark.skip(reason="later")), + ("torch", pytest.mark.skipif(True, reason="another reason")), + ("arrow", needs("torch")), + ], +) +def test_a_forbidden_mark_is_reported(group: str, mark: Any) -> None: + """``disallowed_marks`` names a skip or xfail other than the optional-package skip.""" + assert disallowed_marks(group, [mark.mark]) + + +def test_the_optional_package_skip_is_allowed() -> None: + """The optional-package skip of an optional group is not reported.""" + assert disallowed_marks("torch", [needs("torch").mark]) == [] GENERATOR_TIMEOUT_S = 600 @@ -396,13 +523,18 @@ def test_tensor_and_ndarray_pins_differ(name: str) -> None: OLDER_TORCH_GOLDEN = """ -import json import pathlib -import shutil import sys -from tests.test_hashing.golden import cases -from tests.test_hashing.golden.generate_golden_hashes import golden_path, main +from orcapod.contexts import get_default_context +from tests.test_hashing.golden import cases, store +from tests.test_hashing.golden.marks import ( + disallowed_marks, + entry_marks, + golden_entry_violations, + needs, +) +from tests.test_hashing.golden.generate_golden_hashes import main # Stand in for a supported PyTorch that predates float8_e8m0fnu (added in torch # 2.7). The attribute itself cannot be removed: torch.save refers to it. @@ -414,16 +546,16 @@ def test_tensor_and_ndarray_pins_differ(name: str) -> None: live = cases.compute_group("torch") print("produced:", "value.float8_e8m0fnu" in live) print("unavailable:", sorted(cases.unavailable_entries("torch"))) -out = pathlib.Path(sys.argv[1]) -shutil.copy(golden_path("torch"), out / "torch.json") -main(["--group", "torch", "--output-dir", str(out)]) -kept = json.loads((out / "torch.json").read_text()) -pinned = json.loads(golden_path("torch").read_text()) -print("pins kept:", kept == pinned) -empty = out / "empty" -main(["--group", "torch", "--output-dir", str(empty)]) -fresh = json.loads((empty / "torch.json").read_text()) -print("pins kept in an empty output dir:", fresh == pinned) +root = pathlib.Path(sys.argv[1]) +version = get_default_context().version +store.copy_tree(store.PROJECT_ROOT, root, version) +manifest = store.manifest_path(root, version) +folder = store.golden_folder(root, store.read_json(manifest)) +before = {p.name: p.read_bytes() for p in [manifest, *sorted(folder.iterdir())]} +main(["--root", str(root), "--group", "torch"]) +main(["--root", str(root), "--check", "--group", "torch"]) +after = {p.name: p.read_bytes() for p in [manifest, *sorted(folder.iterdir())]} +print("pins kept:", before == after) """ @@ -444,4 +576,3 @@ def test_torch_group_runs_on_a_torch_without_a_newer_dtype(tmp_path: Path) -> No assert "produced: False" in lines assert "unavailable: ['value.float8_e8m0fnu']" in lines assert "pins kept: True" in lines - assert "pins kept in an empty output dir: True" in lines diff --git a/tests/test_hashing/golden/test_golden_store.py b/tests/test_hashing/golden/test_golden_store.py new file mode 100644 index 000000000..c38bda799 --- /dev/null +++ b/tests/test_hashing/golden/test_golden_store.py @@ -0,0 +1,560 @@ +"""The golden folder, its history and the generator's add, check and release (ITL-946). + +The generator runs on a copy of the repository's golden tree (``--root``), never on the +repository itself. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +import pytest + +from orcapod.contexts import get_default_context +from tests.test_hashing.golden import cases, store +from tests.test_hashing.golden.generate_golden_hashes import dump, main + +VERSION = get_default_context().version + + +# The trial trees copy the repository's golden folder, so the newest file of a group and the +# next counter come from that inventory, never from fixed names. +_REPOSITORY_FILES = store.group_files( + store.golden_folder( + store.PROJECT_ROOT, + store.read_json(store.manifest_path(store.PROJECT_ROOT, VERSION)), + ) +) +NEWEST_SCHEMA = _REPOSITORY_FILES["schema"][-1].name +NEXT_SCHEMA = store.next_file_name(NEWEST_SCHEMA) + + +def _write(folder: Path, name: str, values: dict[str, Any]) -> None: + folder.mkdir(parents=True, exist_ok=True) + _ = (folder / name).write_text(dump(values), encoding="utf-8") + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + """A copy of the repository's golden tree, reopened: one open revision, no release.""" + root = tmp_path / "tree" + store.copy_tree(store.PROJECT_ROOT, root, VERSION) + store.reopen_tree(root, VERSION, "2026-11-15") + return root + + +def _manifest(root: Path) -> dict[str, Any]: + return store.read_json(store.manifest_path(root, VERSION)) + + +def _folder(root: Path) -> Path: + return store.golden_folder(root, _manifest(root)) + + +def _history(root: Path) -> list[dict[str, Any]]: + return _manifest(root)["golden"]["history"] + + +def _pin(root: Path) -> dict[str, Any]: + return store.read_json(store.pins_path(root))[VERSION] + + +def _record(root: Path, name: str, values: dict[str, Any]) -> None: + """Write an unreleased golden file and record it, as an earlier ``--add`` would.""" + folder = _folder(root) + _write(folder, name, values) + manifest = _manifest(root) + manifest["golden"]["history"] = store.with_file( + manifest["golden"]["history"], + name, + store.file_digest(folder / name), + "2026-11-15", + ) + store.write_json(store.manifest_path(root, VERSION), manifest) + + +def _drop_entries(root: Path, name: str, count: int) -> dict[str, Any]: + """Remove the last ``count`` entries of an unreleased file; return them.""" + values = store.read_json(_folder(root) / name) + dropped = dict(list(values.items())[-count:]) + _record(root, name, {k: v for k, v in values.items() if k not in dropped}) + return dropped + + +def _snapshot(folder: Path) -> dict[str, bytes]: + return { + path.name: path.read_bytes() + for path in sorted(folder.iterdir()) + if path.is_file() + } + + +# --------------------------------------------------------------------------- +# The folder +# --------------------------------------------------------------------------- + + +def test_the_repository_tree_is_sound() -> None: + """The repository's manifest, pins and golden folder agree.""" + assert store.tree_problems(store.PROJECT_ROOT, VERSION) == [] + + +@pytest.mark.parametrize( + ("first", "second"), + [("arrow-01.json", "arrow-02.json"), ("arrow-01.json", "schema-01.json")], +) +def test_an_entry_pinned_in_two_files_is_an_error( + tmp_path: Path, first: str, second: str +) -> None: + """An entry name appears once in a folder, across the files of every group.""" + _write(tmp_path, first, {"a": 1, "b": 2}) + _write(tmp_path, second, {"b": 2}) + with pytest.raises(store.GoldenLayoutError, match="'b' is pinned twice"): + _ = store.load_folder(tmp_path) + + +def test_entries_of_a_group_merge_over_its_files(tmp_path: Path) -> None: + _write(tmp_path, "arrow-01.json", {"a": 1}) + _write(tmp_path, "arrow-02.json", {"b": 2}) + _write(tmp_path, "schema-01.json", {"c": 3}) + assert store.load_folder(tmp_path) == { + "arrow": {"a": 1, "b": 2}, + "schema": {"c": 3}, + } + + +@pytest.mark.parametrize( + "name", ["arrow.json", "arrow-1.json", "arrow-001.json", "notes.txt"] +) +def test_a_file_not_named_group_and_counter_is_an_error( + tmp_path: Path, name: str +) -> None: + _ = (tmp_path / name).write_text("{}") + with pytest.raises(store.GoldenLayoutError, match="-NN.json"): + _ = store.load_folder(tmp_path) + + +def test_an_edited_listed_file_is_reported(tree: Path) -> None: + """The problem says to restore the file, never to record the edit with the generator.""" + path = _folder(tree) / "arrow-01.json" + _ = path.write_bytes(path.read_bytes().replace(b"\n", b"\r\n")) + [problem] = [ + p + for p in store.tree_problems(tree, VERSION) + if "arrow-01.json does not have the sha256" in p + ] + assert "restore" in problem + assert "--remove" in problem + assert "to record it" not in problem + + +def test_an_unlisted_file_is_reported(tree: Path) -> None: + name = store.next_file_name(_REPOSITORY_FILES["arrow"][-1].name) + _write(_folder(tree), name, {"new": 1}) + assert f"{name} is in" in " ".join(store.tree_problems(tree, VERSION)) + + +def test_a_recorded_file_repeating_an_entry_name_is_reported(tree: Path) -> None: + """A duplicate entry name in correctly recorded files is a problem, and no release.""" + entry, value = next(iter(store.read_json(_folder(tree) / NEWEST_SCHEMA).items())) + _record(tree, NEXT_SCHEMA, {entry: value}) + assert any("pinned twice" in p for p in store.tree_problems(tree, VERSION)) + with pytest.raises(SystemExit, match="pinned twice"): + main(["--root", str(tree), "--release", "0.3.0"]) + + +def test_a_recorded_file_of_an_unknown_group_is_reported(tree: Path) -> None: + """A file whose group is not one of ``cases.GROUPS`` is a problem, even when recorded.""" + _record(tree, "obsolete-01.json", {"obsolete.entry": "x"}) + assert any( + "obsolete-01.json" in p and "known group" in p + for p in store.tree_problems(tree, VERSION) + ) + + +def test_a_key_repeated_inside_one_file_is_an_error(tmp_path: Path) -> None: + """JSON keeps the last of two equal keys; the store refuses the file instead.""" + _ = (tmp_path / "arrow-01.json").write_text( + '{"example.entry": "old", "example.entry": "current"}\n' + ) + with pytest.raises(store.GoldenLayoutError, match="example.entry"): + _ = store.load_folder(tmp_path) + + +def test_a_key_repeated_inside_a_recorded_file_is_reported(tree: Path) -> None: + name = NEXT_SCHEMA + folder = _folder(tree) + _ = (folder / name).write_bytes(b'{"dup.entry": "a", "dup.entry": "b"}\n') + manifest = _manifest(tree) + manifest["golden"]["history"] = store.with_file( + manifest["golden"]["history"], + name, + store.file_digest(folder / name), + "2026-11-15", + ) + store.write_json(store.manifest_path(tree, VERSION), manifest) + assert any("dup.entry" in p for p in store.tree_problems(tree, VERSION)) + + +@pytest.mark.parametrize( + "name", ["../arrow-01.json", "sub/arrow-01.json", "obsolete-01.json", "arrow.json"] +) +def test_a_listed_name_that_is_not_a_golden_file_name_is_reported( + tree: Path, name: str +) -> None: + """A history entry names a ``-NN.json`` file of a known group, in the folder.""" + history = _history(tree) + history[0]["added"][name] = "0" * 64 + history = store.rechain(history) + problems = store.folder_problems(history, _folder(tree)) + assert any(name in p and "not a golden file name" in p for p in problems) + + +def test_a_missing_listed_file_is_reported(tree: Path) -> None: + (_folder(tree) / "torch-01.json").unlink() + assert any( + "torch-01.json" in p and "missing" in p + for p in store.tree_problems(tree, VERSION) + ) + + +def test_a_file_listed_twice_is_reported(tree: Path) -> None: + history = _history(tree) + added = history[0]["added"] + history = store.with_release(history, "0.3.0", "2026-11-16") + history.append( + {"date": "2026-11-17", "release": None, "added": dict(added), "previous": None} + ) + history = store.rechain(history) + assert store.history_problems(history) == [] + assert any( + "listed by revisions 0 and 2" in p + for p in store.folder_problems(history, _folder(tree)) + ) + + +# --------------------------------------------------------------------------- +# The history +# --------------------------------------------------------------------------- + + +def _released_history(tree: Path) -> list[dict[str, Any]]: + return store.with_release(_history(tree), "0.3.0", "2026-11-16") + + +def test_a_rechained_history_is_sound(tree: Path) -> None: + assert store.history_problems(_released_history(tree)) == [] + + +@pytest.mark.parametrize( + ("change", "problem"), + [ + ( + lambda h: h[0].update(date="2026-11-14"), + "revision 0: digest does not recompute", + ), + (lambda h: h[1].update(previous=None), "revision 1: previous is not"), + ( + lambda h: h[1].update(added={"x-01.json": "00"}), + "release revision and adds files", + ), + (lambda h: h[0].update(added={}), "adds no file and is not a release"), + (lambda h: h[1].pop("digest"), "does not have exactly the fields"), + ( + lambda h: h.append(dict(h[1], previous=h[1]["digest"])), + "release 0.3.0 appears twice", + ), + (lambda h: h[0].update(date="15 November"), "no YYYY-MM-DD date"), + ], +) +def test_a_broken_history_is_reported(tree: Path, change: Any, problem: str) -> None: + history = _released_history(tree) + change(history) + assert problem in " ".join(store.history_problems(history)) + + +def test_a_released_file_cannot_be_recorded_again(tree: Path) -> None: + with pytest.raises(ValueError, match="never changes"): + _ = store.with_file( + _released_history(tree), "arrow-01.json", "00", "2026-11-17" + ) + + +def test_rechain_leaves_released_revisions_alone(tree: Path) -> None: + history = _released_history(tree) + tampered = [dict(history[0], date="2000-01-01"), *history[1:]] + assert store.rechain(tampered)[0]["digest"] == history[0]["digest"] + + +# --------------------------------------------------------------------------- +# The pins +# --------------------------------------------------------------------------- + + +def test_a_final_release_needs_its_releases_line_and_a_pinned_manifest() -> None: + pin = {"manifest": None, "releases": {}} + problems = store.final_release_problems(pin, "0.3.0") + assert "release 0.3.0 ships an open manifest" in problems + assert any("no releases line" in p for p in problems) + + +@pytest.mark.parametrize("version", ["0.3.1.dev4+g1234abc", "0.3.0rc1", "0.3.0+local"]) +def test_a_build_that_is_not_a_final_release_needs_no_releases_line( + version: str, +) -> None: + assert ( + store.final_release_problems({"manifest": None, "releases": {}}, version) == [] + ) + + +def test_the_repository_pin_is_consistent() -> None: + """The repository's pin, manifest and history agree, open or released.""" + pin = _pin(store.PROJECT_ROOT) + manifest = _manifest(store.PROJECT_ROOT) + assert store.pin_problems(manifest, pin, manifest["golden"]["history"]) == [] + + +# --------------------------------------------------------------------------- +# The generator on a copy: add, check, release +# --------------------------------------------------------------------------- + + +def test_add_writes_new_entries_into_the_open_file_of_their_group(tree: Path) -> None: + """A group's newest file that no release lists takes the new entries.""" + dropped = _drop_entries(tree, NEWEST_SCHEMA, 3) + before = _snapshot(_folder(tree)) + main(["--root", str(tree), "--group", "schema"]) + after = _snapshot(_folder(tree)) + assert set(after) == set(before) + schema = json.loads(after[NEWEST_SCHEMA]) + assert dropped.items() <= schema.items() + assert {n: b for n, b in after.items() if n != NEWEST_SCHEMA} == { + n: b for n, b in before.items() if n != NEWEST_SCHEMA + } + assert len(_history(tree)) == 1 + assert store.tree_problems(tree, VERSION) == [] + + +def test_add_after_a_release_writes_a_new_file_and_a_new_revision(tree: Path) -> None: + """A released file never changes: new entries go to the next ``-NN.json``.""" + dropped = _drop_entries(tree, NEWEST_SCHEMA, 2) + main(["--root", str(tree), "--release", "0.3.0"]) + before = _snapshot(_folder(tree)) + main(["--root", str(tree), "--group", "schema"]) + after = _snapshot(_folder(tree)) + assert {n: after[n] for n in before} == before + assert set(after) - set(before) == {NEXT_SCHEMA} + assert json.loads(after[NEXT_SCHEMA]) == dropped + history = _history(tree) + assert [r["release"] for r in history] == [None, "0.3.0", None] + assert history[2]["previous"] == history[1]["digest"] + assert list(history[2]["added"]) == [NEXT_SCHEMA] + assert store.tree_problems(tree, VERSION) == [] + + +def test_add_changes_nothing_when_a_later_group_fails( + tree: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Every group is computed before anything is written: files and history stay in step.""" + _ = _drop_entries(tree, NEWEST_SCHEMA, 1) + before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + compute = cases.compute_group + + def failing(group: str, **kwargs: Any) -> dict[str, Any]: + if group == "semantic": + raise RuntimeError("semantic failed") + return compute(group, **kwargs) + + monkeypatch.setattr(cases, "compute_group", failing) + with pytest.raises(RuntimeError, match="semantic failed"): + main(["--root", str(tree), "--group", "schema", "--group", "semantic"]) + after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + assert after == before + assert store.tree_problems(tree, VERSION) == [] + + +def test_add_refuses_one_new_entry_name_in_two_groups( + tree: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two groups that compute the same new entry name in one run write nothing.""" + before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + compute = cases.compute_group + + def with_shared_entry(group: str, **kwargs: Any) -> dict[str, Any]: + return {**compute(group, **kwargs), "shared.new_entry": group} + + monkeypatch.setattr(cases, "compute_group", with_shared_entry) + with pytest.raises(SystemExit, match="shared.new_entry"): + main(["--root", str(tree), "--group", "schema", "--group", "arrow"]) + after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + assert after == before + + +def test_add_writes_nothing_when_every_entry_is_pinned(tree: Path) -> None: + before = _snapshot(tree / store.CONTEXTS_DIR / "data"), _snapshot(_folder(tree)) + main(["--root", str(tree), "--group", "schema"]) + assert ( + _snapshot(tree / store.CONTEXTS_DIR / "data"), + _snapshot(_folder(tree)), + ) == before + + +def test_add_refuses_a_tree_that_breaks_the_rules(tree: Path) -> None: + path = _folder(tree) / NEWEST_SCHEMA + _ = path.write_text(path.read_text() + " ") + manifest = store.manifest_path(tree, VERSION).read_bytes() + with pytest.raises(SystemExit, match="nothing written"): + main(["--root", str(tree), "--group", "schema"]) + assert store.manifest_path(tree, VERSION).read_bytes() == manifest + + +def test_remove_then_add_changes_an_unreleased_value(tree: Path) -> None: + """The approved-change procedure: ``--remove`` the entry, then ``--add`` writes it.""" + name = NEWEST_SCHEMA + values = store.read_json(_folder(tree) / name) + entry = next(iter(values)) + _record(tree, name, {**values, entry: "old value"}) + main(["--root", str(tree), "--remove", entry]) + assert entry not in store.read_json(_folder(tree) / name) + assert store.tree_problems(tree, VERSION) == [] + main(["--root", str(tree), "--group", "schema"]) + assert store.read_json(_folder(tree) / name) == values + assert len(_history(tree)) == 1 + assert store.tree_problems(tree, VERSION) == [] + + +def test_remove_refuses_an_entry_a_release_lists(tree: Path) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + entry = next(iter(store.read_json(_folder(tree) / NEWEST_SCHEMA))) + before = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") + with pytest.raises(SystemExit, match="released"): + main(["--root", str(tree), "--remove", entry]) + after = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") + assert after == before + + +def test_remove_refuses_an_unknown_entry_and_writes_nothing(tree: Path) -> None: + entry = next(iter(store.read_json(_folder(tree) / NEWEST_SCHEMA))) + before = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") + with pytest.raises(SystemExit, match="no.such.entry"): + main(["--root", str(tree), "--remove", entry, "--remove", "no.such.entry"]) + after = _snapshot(_folder(tree)), _snapshot(tree / store.CONTEXTS_DIR / "data") + assert after == before + + +def test_check_reports_a_moved_value_and_writes_nothing(tree: Path) -> None: + name = NEWEST_SCHEMA + values = store.read_json(_folder(tree) / name) + entry = next(iter(values)) + _record(tree, name, {**values, entry: "moved"}) + before = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + with pytest.raises(SystemExit, match=re.escape(f"schema:{entry} moved")): + main(["--root", str(tree), "--check", "--group", "schema"]) + after = _snapshot(_folder(tree)), store.manifest_path(tree, VERSION).read_bytes() + assert after == before + + +def test_check_reports_an_unpinned_entry(tree: Path) -> None: + dropped = _drop_entries(tree, NEWEST_SCHEMA, 1) + with pytest.raises( + SystemExit, match=re.escape(f"schema:{next(iter(dropped))} is computed") + ): + main(["--root", str(tree), "--check", "--group", "schema"]) + + +def test_check_passes_on_the_copy(tree: Path) -> None: + main(["--root", str(tree), "--check", "--group", "schema"]) + + +def test_release_freezes_the_manifest_and_pins_the_release(tree: Path) -> None: + """The freeze step, on a copy: release revision, pins and ``released_in``.""" + files = _snapshot(_folder(tree)) + main(["--root", str(tree), "--release", "v0.3.0"]) + manifest = _manifest(tree) + history = manifest["golden"]["history"] + pin = _pin(tree) + assert history[-1]["release"] == "0.3.0" + assert history[-1]["added"] == {} + assert manifest["released_in"] == "0.3.0" + assert pin == { + "manifest": store.fixed_manifest_digest(manifest), + "releases": {"0.3.0": history[-1]["digest"]}, + } + assert _snapshot(_folder(tree)) == files + assert store.tree_problems(tree, VERSION) == [] + assert store.final_release_problems(pin, "0.3.0") == [] + + +def test_a_later_release_keeps_released_in_and_the_manifest_pin(tree: Path) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + first = _pin(tree) + main(["--root", str(tree), "--release", "0.3.1"]) + pin = _pin(tree) + assert _manifest(tree)["released_in"] == "0.3.0" + assert pin["manifest"] == first["manifest"] + assert pin["releases"]["0.3.0"] == first["releases"]["0.3.0"] + assert set(pin["releases"]) == {"0.3.0", "0.3.1"} + assert store.tree_problems(tree, VERSION) == [] + + +def test_a_released_manifest_edited_afterwards_is_reported(tree: Path) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + manifest = _manifest(tree) + manifest["description"] += " (edited)" + store.write_json(store.manifest_path(tree, VERSION), manifest) + assert any("differs from its pin" in p for p in store.tree_problems(tree, VERSION)) + + +def test_a_released_golden_file_edited_afterwards_is_reported(tree: Path) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + path = _folder(tree) / "semantic-01.json" + _ = path.write_text(path.read_text().replace("semantic_v1", "semantic_v2", 1)) + assert any( + "a released golden file never changes" in p + for p in store.tree_problems(tree, VERSION) + ) + + +@pytest.mark.parametrize( + "change", + [ + lambda pin, h: pin["releases"].update({"0.3.1": h[-1]["digest"]}), + lambda pin, h: pin["releases"].clear(), + lambda pin, h: pin["releases"].update({"0.3.0": h[0]["digest"]}), + lambda pin, h: pin.update(manifest=None), + ], +) +def test_a_pin_that_disagrees_with_the_history_is_reported( + tree: Path, change: Any +) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + pins = store.read_json(store.pins_path(tree)) + change(pins[VERSION], _history(tree)) + store.write_json(store.pins_path(tree), pins) + assert store.tree_problems(tree, VERSION) + + +@pytest.mark.parametrize( + "version", ["0.3.0", "0.3.1+local", "0.3.1.dev1", "not-a-version"] +) +def test_release_refuses_a_version_it_cannot_record(tree: Path, version: str) -> None: + main(["--root", str(tree), "--release", "0.3.0"]) + before = ( + _snapshot(tree / store.CONTEXTS_DIR), + _snapshot(tree / store.CONTEXTS_DIR / "data"), + ) + with pytest.raises(SystemExit): + main(["--root", str(tree), "--release", version]) + after = ( + _snapshot(tree / store.CONTEXTS_DIR), + _snapshot(tree / store.CONTEXTS_DIR / "data"), + ) + assert after == before + + +def test_allow_errors_is_only_for_stdout(tree: Path) -> None: + with pytest.raises(SystemExit): + main(["--root", str(tree), "--allow-errors", "--group", "schema"])