From d3f1a5d6cc9fc1334baf54c45f0d9b9bd1857fc2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 17:53:55 -0800 Subject: [PATCH 1/3] fix(derived_source): tags_as_data mapping enters the content hash (ITL-1000) Co-Authored-By: Claude Sonnet 5.5 --- changelog.d/ITL-1000.md | 3 ++ design_issues/ITL-1000-DS1.md | 10 ++++ src/orcapod/core/sources/derived_source.py | 22 ++++++-- .../plans/ITL-1000-derived-source-identity.md | 14 +++++ .../test_derived_source_tags_as_data.py | 52 ++++++++++++++++++- tests/test_hashing/golden/data/schema.json | 6 +-- 6 files changed, 100 insertions(+), 7 deletions(-) create mode 100644 changelog.d/ITL-1000.md create mode 100644 design_issues/ITL-1000-DS1.md create mode 100644 superpowers/plans/ITL-1000-derived-source-identity.md diff --git a/changelog.d/ITL-1000.md b/changelog.d/ITL-1000.md new file mode 100644 index 000000000..8af148acd --- /dev/null +++ b/changelog.d/ITL-1000.md @@ -0,0 +1,3 @@ +### Breaking changes and migration + +- The `content_hash` of a `DerivedSource` built with `tags_as_data` now includes the mapping (ITL-1000). Before, two mappings that gave the same schema (two tag columns of one type copied to one name) and shared an explicit `source_id` had the same `content_hash` although the copy held different values; the sorted mapping items are now part of the identity. The `content_hash` moves whenever `tags_as_data` is given, with the default `source_id` as well as an explicit one, because the mapping is appended in both cases. The `source_id` (the default id was already built from the mapping), record IDs, source-info tokens and `pipeline_hash` do not move, and a `DerivedSource` without `tags_as_data` (a job node's stored records) is unchanged. `tags_as_data` was added in 0.3 and has not been released, so no released store holds a hash that moves. A store written by an unreleased 0.3 build with a `tags_as_data` `DerivedSource` gives that source, and the nodes downstream of it, a new node identity: those pipeline records are recomputed, and the old rows stay in place. The golden values `schema:derived_source_tags_as_data.{default_source_id,named_source_id,record_id_column}.content_hash` changed with it. diff --git a/design_issues/ITL-1000-DS1.md b/design_issues/ITL-1000-DS1.md new file mode 100644 index 000000000..e1cd42cf4 --- /dev/null +++ b/design_issues/ITL-1000-DS1.md @@ -0,0 +1,10 @@ +## `src/orcapod/core/sources/derived_source.py` — identity of a `tags_as_data` source (ITL-1000) + +### DS1 — `DerivedSource` `tags_as_data` mapping was not part of the content hash +**Status:** resolved +**Severity:** medium +**Issue:** ITL-1000 + +A `DerivedSource` with `tags_as_data` hashed `(class name, output schema, source_id)`. Two mappings that give the same schema (two tag columns of one type copied to one name) with one explicit `source_id` shared a content hash, although the copy holds different values. + +**Fix:** `DerivedSource.identity_structure` appends `("tags_as_data", sorted items)` when the option is given, as `SpiralDBTableSource` does (ITL-996). The `pipeline_hash` stays schema-only; the plain form is unchanged. Developer decision, November 12, 2026. diff --git a/src/orcapod/core/sources/derived_source.py b/src/orcapod/core/sources/derived_source.py index 1f170f6c9..f9ad67ab1 100644 --- a/src/orcapod/core/sources/derived_source.py +++ b/src/orcapod/core/sources/derived_source.py @@ -138,8 +138,9 @@ class DerivedSource(RootSource): the ``record_id_column`` when one is set. It changes when the origin's identity or one of these parameters changes. An explicit ``source_id`` replaces it. - - ``content_hash`` follows the rule of every root source: class name, - output schema and ``source_id``. + - ``content_hash`` follows the rule of every root source (class name, + output schema and ``source_id``) plus the sorted ``tags_as_data`` items, + since two mappings can give one schema and one explicit ``source_id``. - ``pipeline_hash`` is schema-only, as for every root source. Identity (plain form): @@ -261,10 +262,25 @@ def __init__( # ------------------------------------------------------------------------- def identity_structure(self) -> Any: + """Return what the content hash is built from. + + The plain form is tied to the origin node's content hash. The + ``tags_as_data`` form is the root source rule (class name, output schema, + ``source_id``) followed by the sorted mapping items: two mappings can give + the same schema (two tag columns of one type copied to one name) while the + copy holds different values, so with an explicit ``source_id`` the mapping + has to tell the sources apart (ITL-1000). + + Returns: + The identity structure. + """ if self._tags_as_data is None: # Tied precisely to the specific node's data identity return (self._origin.content_hash(),) - return super().identity_structure() + return ( + *super().identity_structure(), + ("tags_as_data", tuple(sorted(self._tags_as_data.items()))), + ) # ------------------------------------------------------------------------- # Serialization diff --git a/superpowers/plans/ITL-1000-derived-source-identity.md b/superpowers/plans/ITL-1000-derived-source-identity.md new file mode 100644 index 000000000..560c619cc --- /dev/null +++ b/superpowers/plans/ITL-1000-derived-source-identity.md @@ -0,0 +1,14 @@ +# ITL-1000 plan: DerivedSource tags_as_data mapping enters content hash + +Decision (developer, 2026-11-12): the mapping is part of identity when used; fix before the tag. + +1. Failing test (public API) in tests/test_core/sources/test_derived_source_tags_as_data.py: + two DerivedSources, same origin, same explicit source_id, mappings of two same-typed tags to + one target name -> content hashes differ; record ids, source ids, pipeline hashes equal. + Without the option: identity_structure unchanged. +2. Fix: DerivedSource.identity_structure appends ("tags_as_data", sorted items) when given + (as SpiralDBTableSource, PR #336). Pipeline hash stays schema-only. Plain form untouched. +3. Measure which hashes move (explicit id vs default id). Run golden suite; additions only. +4. Check to_config/from_config and saved-job node identities. +5. changelog.d/ITL-1000.md (Breaking changes and migration), design_issues/ITL-1000-*.md. +6. Suite 3.12/3.11/3.13, ruff, format, basedpyright. One push, PR, readiness. diff --git a/tests/test_core/sources/test_derived_source_tags_as_data.py b/tests/test_core/sources/test_derived_source_tags_as_data.py index 4816517e3..61e8a5c88 100644 --- a/tests/test_core/sources/test_derived_source_tags_as_data.py +++ b/tests/test_core/sources/test_derived_source_tags_as_data.py @@ -24,6 +24,7 @@ from orcapod.errors import InputValidationError from orcapod.operators import Join from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.types import ColumnConfig from orcapod.sources import ArrowTableSource, DictSource, SQLiteTableSource UTC = dt.timezone.utc @@ -400,14 +401,60 @@ def test_content_hash_follows_origin_and_mapping(self): ) assert base.content_hash() != _experiment().content_hash() - def test_content_hash_is_the_root_source_rule(self): + def test_content_hash_is_the_root_source_rule_plus_the_mapping(self): derived = DerivedSource(_experiment(), tags_as_data=COPY) + # identity_structure is the non-public entry point this test is about: the + # structure is the root source rule followed by the sorted mapping items. assert derived.identity_structure() == ( "DerivedSource", derived.output_schema(), derived.source_id, + ("tags_as_data", tuple(sorted(COPY.items()))), ) + def test_mappings_with_one_schema_have_different_content_hashes(self): + # ITL-1000: two tag columns of one type copied to one name give one schema + # but different values, so with one explicit source_id the mapping must tell + # the sources apart. + table = pa.table( + { + "a": pa.array([1, 2], pa.int64()), + "b": pa.array([3, 4], pa.int64()), + "x": pa.array([0.5, 1.5], pa.float64()), + } + ) + base = ArrowTableSource(table, tag_columns=["a", "b"], source_id="two_tags") + from_a = DerivedSource(base, source_id="same", tags_as_data={"a": "k"}) + from_b = DerivedSource(base, source_id="same", tags_as_data={"b": "k"}) + assert from_a.output_schema() == from_b.output_schema() + assert from_a.source_id == from_b.source_id + assert from_a.pipeline_hash() == from_b.pipeline_hash() + assert from_a.content_hash() != from_b.content_hash() + # The copies hold different values, and record ids (from the source id and + # row position) are equal. + ta, tb = from_a.as_table(), from_b.as_table() + assert ta.column("k").to_pylist() != tb.column("k").to_pylist() + ca, cb = ( + from_a.as_table(columns=ColumnConfig(system_tags=True)), + from_b.as_table(columns=ColumnConfig(system_tags=True)), + ) + (rid_a,) = _prefixed(ca, "_tag_record_id") + (rid_b,) = _prefixed(cb, "_tag_record_id") + assert ca.column(rid_a).to_pylist() == cb.column(rid_b).to_pylist() + + def test_mapping_order_does_not_change_content_hash(self): + a = DerivedSource( + _experiment(), + source_id="s", + tags_as_data={"subject_id": "p", "experiment_index": "q"}, + ) + b = DerivedSource( + _experiment(), + source_id="s", + tags_as_data={"experiment_index": "q", "subject_id": "p"}, + ) + assert a.content_hash() == b.content_hash() + # --------------------------------------------------------------------------- # Downstream use @@ -586,6 +633,8 @@ def test_saved_job_reloads_the_derived_source(self, tmp_path): assert reloaded.content_hash() == original.content_hash() assert reloaded.as_table(all_info=True).equals(original.as_table(all_info=True)) assert loaded.nodes["len"].pipeline_hash() == job.nodes["len"].pipeline_hash() + # The node identities include the mapping through the source's content hash. + assert loaded.nodes["len"].content_hash() == job.nodes["len"].content_hash() def test_config_round_trip_keeps_an_explicit_source_id(self, tmp_path): db_path = _make_probe_db(tmp_path / "edi.db") @@ -600,6 +649,7 @@ def test_config_round_trip_keeps_an_explicit_source_id(self, tmp_path): assert config["tags_as_data"] == {"name": "name_copy"} rebuilt = DerivedSource.from_config(config) assert rebuilt.source_id == "probe_named" + assert rebuilt.content_hash() == derived.content_hash() assert rebuilt.as_table(all_info=True).equals(derived.as_table(all_info=True)) def test_not_reloadable_when_the_origin_is_not(self, tmp_path): diff --git a/tests/test_hashing/golden/data/schema.json b/tests/test_hashing/golden/data/schema.json index 350866a39..510316401 100644 --- a/tests/test_hashing/golden/data/schema.json +++ b/tests/test_hashing/golden/data/schema.json @@ -1,7 +1,7 @@ { "compute_schema_hash.chars_16": "03596e8aeb60e3f5", "compute_schema_hash.full": "03596e8aeb60e3f594b596b4f5c0ab87c8e2efb2ee3bd1886775e971e8d7fc39", - "derived_source_tags_as_data.default_source_id.content_hash": "semantic_v1:c41422acd1e99d2754b7dcb1c70fc65411f922ac1bcd68d0be8d23281f78989b", + "derived_source_tags_as_data.default_source_id.content_hash": "semantic_v1:8bdcacb99dec766697c555c1631a3f96e776abd2758aeccc0dd624807742f3fa", "derived_source_tags_as_data.default_source_id.pipeline_hash": "semantic_v1:3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a", "derived_source_tags_as_data.default_source_id.row[{\"subject\":\"s1\"}]._tag_record_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a": "8be2c82e53555b479b5bd2df2b8f56ca", "derived_source_tags_as_data.default_source_id.row[{\"subject\":\"s1\"}]._tag_source_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a": "tags_as_data:semantic_v1:a255549e5d36cb6fd37496fc382dcfc776e54461313cdc1e22a1ccaee98747f6", @@ -17,7 +17,7 @@ "_tag_source_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a", "_tag_record_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a" ], - "derived_source_tags_as_data.named_source_id.content_hash": "semantic_v1:8597ec7aafe8117c96c052a35d8f102d3fa8a3ada1d5418b77380ec6dce907c9", + "derived_source_tags_as_data.named_source_id.content_hash": "semantic_v1:a02722ed9904292f268293a3bc6c2d708b9473f69ea1c4c239a6066a6fdbc0af", "derived_source_tags_as_data.named_source_id.pipeline_hash": "semantic_v1:3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a", "derived_source_tags_as_data.named_source_id.row[{\"subject\":\"s1\"}]._tag_record_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a": "aa78da3cf532571baccfbd2122decfb1", "derived_source_tags_as_data.named_source_id.row[{\"subject\":\"s1\"}]._tag_source_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a": "golden_derived", @@ -33,7 +33,7 @@ "_tag_source_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a", "_tag_record_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a" ], - "derived_source_tags_as_data.record_id_column.content_hash": "semantic_v1:1c2e6bef64bd40852e4439cd3e2025049034f6d9ae40696bae2b3c444786556d", + "derived_source_tags_as_data.record_id_column.content_hash": "semantic_v1:b681686fb59cb9eef4426db6a005fa6144efbf5ae6dbfb3d98d0707540bc2a3f", "derived_source_tags_as_data.record_id_column.pipeline_hash": "semantic_v1:3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a", "derived_source_tags_as_data.record_id_column.row[{\"subject\":\"s1\"}]._tag_record_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a": "fb17ac745786500eb610b18676bf9f81", "derived_source_tags_as_data.record_id_column.row[{\"subject\":\"s1\"}]._tag_source_id::3becdc3d3abb1d555203a086f4b02d6712e4dfead024591d1e844b519fc14c7a": "tags_as_data:semantic_v1:4bc1b45a35684cfe7142828423ebf054568796e49d6bf17ebba30a97818acf66", From 9c3a9c956299a1b671507bfe11fcdda3619f9b51 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 18:11:02 -0800 Subject: [PATCH 2/3] test(derived_source): docstrings on the identity tests (ITL-1000) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_core/sources/test_derived_source_tags_as_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_core/sources/test_derived_source_tags_as_data.py b/tests/test_core/sources/test_derived_source_tags_as_data.py index 61e8a5c88..28f8bc2d7 100644 --- a/tests/test_core/sources/test_derived_source_tags_as_data.py +++ b/tests/test_core/sources/test_derived_source_tags_as_data.py @@ -402,6 +402,7 @@ def test_content_hash_follows_origin_and_mapping(self): assert base.content_hash() != _experiment().content_hash() def test_content_hash_is_the_root_source_rule_plus_the_mapping(self): + """The identity structure is the root source rule followed by the sorted mapping.""" derived = DerivedSource(_experiment(), tags_as_data=COPY) # identity_structure is the non-public entry point this test is about: the # structure is the root source rule followed by the sorted mapping items. @@ -413,6 +414,7 @@ def test_content_hash_is_the_root_source_rule_plus_the_mapping(self): ) def test_mappings_with_one_schema_have_different_content_hashes(self): + """Two mappings with one schema and one source_id still differ in content hash (ITL-1000).""" # ITL-1000: two tag columns of one type copied to one name give one schema # but different values, so with one explicit source_id the mapping must tell # the sources apart. @@ -443,6 +445,7 @@ def test_mappings_with_one_schema_have_different_content_hashes(self): assert ca.column(rid_a).to_pylist() == cb.column(rid_b).to_pylist() def test_mapping_order_does_not_change_content_hash(self): + """The mapping enters the identity sorted, so its order does not matter.""" a = DerivedSource( _experiment(), source_id="s", From 2f8d9bcb4f3483da440de2428a3a3d63876d1be8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 18:41:49 -0800 Subject: [PATCH 3/3] docs(derived_source): mapping enters the content hash with an explicit source_id (ITL-1000) Co-Authored-By: Claude Sonnet 5.5 --- docs/concepts/sources.md | 3 ++- docs/reference/hashing.md | 2 +- .../specs/2026-11-11-itl-939-tag-and-data-same-name-design.md | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 52c596b5e..d96e54b1d 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -394,7 +394,8 @@ before. Things to know: has provenance columns of its own, like the result of handing the same rows to `ArrowTableSource`. Use it in place of the origin. Its default `source_id` is `tags_as_data:` followed by a labelled hash built from the origin's identity and the mapping, so - it changes when either changes; pass `source_id=` to choose a name. + it changes when either changes; pass `source_id=` to choose a name. The mapping is part of the + source's `content_hash` either way, so two mappings never share one. - A saved job reloads a derived source whenever its origin can be reloaded (a CSV file, a Delta table, a database table). Over an in-memory source it is saved but not reloaded, like the in-memory source itself. diff --git a/docs/reference/hashing.md b/docs/reference/hashing.md index 8fbb7a3d3..53c417e99 100644 --- a/docs/reference/hashing.md +++ b/docs/reference/hashing.md @@ -322,7 +322,7 @@ The table below shows what each class returns from `identity_structure()`. | `ArrowTableStream` | `(producer, argument_symmetry(upstreams))` | Falls back to `(class_name, as_table(all_info=True), tag_columns)` when no producer or producer is not a `PipelineElementProtocol` | | `RootSource` (`ArrowTableSource`) | `(class_name, (tag_schema, data_schema), source_id)` | `source_id` is the explicit user-supplied value; when not provided it defaults to `::` of the Arrow hash of the raw table (site 4). Data-inclusive base case of the Merkle chain. | | `DerivedSource` | Origin node's `content_hash` | Inherits its generating node's identity | -| `DerivedSource` with `tags_as_data` | `(class_name, output_schema, source_id)`; the default `source_id` is `tags_as_data: